diff options
| author | Stefan Boberg <[email protected]> | 2026-04-23 18:16:57 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2026-04-23 18:16:57 +0200 |
| commit | 0232b991cd7d8e3a2114ea30e4591dd3e7b65c36 (patch) | |
| tree | 94730e7594fd09ae1fa820391ce311f6daf13905 /src/zen | |
| parent | Fix forward declaration order for s_GotSigWinch and SigWinchHandler (diff) | |
| parent | trace: declare Region event name fields as AnsiString (#1012) (diff) | |
| download | archived-zen-sb/zen-help.tar.xz archived-zen-sb/zen-help.zip | |
Merge branch 'main' into sb/zen-helpsb/zen-help
- Combine HelpCommand (this branch) with HistoryCommand (main) in zen CLI dispatcher
- Keep filter-aware TuiPickOne rewrite; adopt main's ASCII arrow glyphs in doc comment
Diffstat (limited to 'src/zen')
71 files changed, 20510 insertions, 5398 deletions
diff --git a/src/zen/authutils.cpp b/src/zen/authutils.cpp index 922007ac8..a2af2b63e 100644 --- a/src/zen/authutils.cpp +++ b/src/zen/authutils.cpp @@ -2,13 +2,16 @@ #include "authutils.h" +#include <zencore/crypto.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/iobuffer.h> #include <zencore/logging.h> #include <zenhttp/auth/authmgr.h> #include <zenhttp/httpclient.h> #include <zenhttp/httpclientauth.h> +#include <zenutil/authutils.h> ZEN_THIRD_PARTY_INCLUDES_START #include <json11.hpp> @@ -112,77 +115,204 @@ AuthCommandLineOptions::AddOptions(cxxopts::Options& Ops) ""); }; +// Load or generate a per-install machine AES key+IV under AuthDir/machinekey.dat +// so the auth-state file is encrypted with bytes unique to this machine rather +// than a hardcoded constant. +// +// When per-user OS-protected storage is available (DPAPI on Windows) the key +// material is wrapped before it lands on disk, so a copy of the file off-machine +// or out of a backup cannot be unwrapped without also stealing the user's OS +// master key. On platforms without OS-level wrapping we fall back to persisting +// the raw bytes with restrictive file permissions (0600 on POSIX; user-only on +// Windows via inheritance from the profile dir). +// +// File format: +// [4-byte magic 'Z','E','N','\x01'] [1-byte flags] [payload] +// flags bit 0 set -> payload is OS-protected (DPAPI blob) +// flags bit 0 clear -> payload is raw KeyBytes+IvBytes bytes +// Legacy files without the magic are interpreted as raw bytes. void -AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, - const std::filesystem::path& SystemRootDir, - HttpClientSettings& ClientSettings, - std::string_view HostUrl, - std::unique_ptr<AuthMgr>& Auth, - bool Quiet, - bool Hidden, - bool Verbose) +AuthCommandLineOptions::LoadOrCreateMachineKey(const std::filesystem::path& AuthDir, bool Quiet) { - auto CreateAuthMgr = [&]() { - ZEN_ASSERT(!SystemRootDir.empty()); - if (!Auth) + constexpr size_t KeyBytes = AesKey256Bit::ByteCount; + constexpr size_t IvBytes = AesIV128Bit::ByteCount; + static constexpr std::array<uint8_t, 4> FileMagic = {'Z', 'E', 'N', 0x01}; + static constexpr uint8_t FlagProtected = 0x01; + const std::filesystem::path KeyFile = AuthDir / "machinekey.dat"; + std::array<uint8_t, KeyBytes + IvBytes> KeyMaterial{}; + bool Loaded = false; + + auto ParseFile = [&](MemoryView FileBytes) -> bool { + // Legacy: raw KeyBytes+IvBytes payload. + if (FileBytes.GetSize() == KeyMaterial.size()) + { + memcpy(KeyMaterial.data(), FileBytes.GetData(), KeyMaterial.size()); + return true; + } + if (FileBytes.GetSize() < FileMagic.size() + 1) + { + return false; + } + if (memcmp(FileBytes.GetData(), FileMagic.data(), FileMagic.size()) != 0) + { + return false; + } + const uint8_t Flags = static_cast<const uint8_t*>(FileBytes.GetData())[FileMagic.size()]; + const MemoryView Payload = FileBytes.Mid(FileMagic.size() + 1); + if (Flags & FlagProtected) { - static const std::string_view DefaultEncryptionKey("abcdefghijklmnopqrstuvxyz0123456"); - static const std::string_view DefaultEncryptionIV("0123456789abcdef"); - if (m_EncryptionKey.empty() && m_EncryptionIV.empty()) + std::vector<uint8_t> Plaintext; + if (!TryUnprotectData(Payload, Plaintext)) { - m_EncryptionKey = DefaultEncryptionKey; - m_EncryptionIV = DefaultEncryptionIV; if (!Quiet) { - ZEN_CONSOLE_WARN("Auth: Using default encryption key and initialization vector for auth storage"); + ZEN_CONSOLE_WARN("Auth: failed to unwrap OS-protected machine key at '{}', regenerating", KeyFile); } + return false; } - else + if (Plaintext.size() != KeyMaterial.size()) { - if (m_EncryptionKey.empty()) - { - m_EncryptionKey = DefaultEncryptionKey; - if (!Quiet) - { - ZEN_CONSOLE_WARN("Auth: Using default encryption key for auth storage"); - } - } - if (m_EncryptionIV.empty()) - { - m_EncryptionIV = DefaultEncryptionIV; - if (!Quiet) - { - ZEN_CONSOLE_WARN("Auth: Using default encryption initialization vector for auth storage"); - } - } + return false; } + memcpy(KeyMaterial.data(), Plaintext.data(), KeyMaterial.size()); + return true; + } + if (Payload.GetSize() != KeyMaterial.size()) + { + return false; + } + memcpy(KeyMaterial.data(), Payload.GetData(), KeyMaterial.size()); + return true; + }; - AuthConfig AuthMgrConfig = {.RootDirectory = SystemRootDir / "auth", - .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), - .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; - if (!AuthMgrConfig.EncryptionKey.IsValid()) - { - throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", m_EncryptionKey), Ops.help()); - } - if (!AuthMgrConfig.EncryptionIV.IsValid()) + std::error_code Ec; + if (std::filesystem::exists(KeyFile, Ec)) + { + IoBuffer Data = ReadFile(KeyFile).Flatten(); + if (ParseFile(Data.GetView())) + { + Loaded = true; + } + else if (!Quiet) + { + ZEN_CONSOLE_WARN("Auth: machine key file '{}' is unreadable (size {}), regenerating", KeyFile, Data.GetSize()); + } + } + + if (!Loaded) + { + CreateDirectories(AuthDir); + if (!SecureRandomBytes(MutableMemoryView(KeyMaterial.data(), KeyMaterial.size()))) + { + throw std::runtime_error("failed to obtain secure random bytes for auth machine key"); + } + + std::vector<uint8_t> FileBytes; + FileBytes.reserve(FileMagic.size() + 1 + KeyMaterial.size()); + FileBytes.insert(FileBytes.end(), FileMagic.begin(), FileMagic.end()); + + std::vector<uint8_t> Wrapped; + if (TryProtectData(MemoryView(KeyMaterial.data(), KeyMaterial.size()), Wrapped)) + { + FileBytes.push_back(FlagProtected); + FileBytes.insert(FileBytes.end(), Wrapped.begin(), Wrapped.end()); + if (!Quiet) { - throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", m_EncryptionIV), Ops.help()); + ZEN_CONSOLE_WARN("Auth: generated OS-protected machine-specific auth encryption key at '{}'", KeyFile); } - if (Verbose) + } + else + { + FileBytes.push_back(0); + FileBytes.insert(FileBytes.end(), KeyMaterial.begin(), KeyMaterial.end()); + if (!Quiet) { - ExtendableStringBuilder<128> SB; - SB << "\n RootDirectory: " << AuthMgrConfig.RootDirectory.string(); - SB << "\n EncryptionKey: " << HideSensitiveString(m_EncryptionKey); - SB << "\n EncryptionIV: " << HideSensitiveString(m_EncryptionIV); - ZEN_CONSOLE("Auth: Creating auth manager with:{}", SB.ToString()); + ZEN_CONSOLE_WARN("Auth: generated machine-specific auth encryption key at '{}' (no OS wrapping available)", KeyFile); } - Auth = AuthMgr::Create(AuthMgrConfig); } - }; + WriteFile(KeyFile, IoBufferBuilder::MakeCloneFromMemory(FileBytes.data(), FileBytes.size())); + + // Belt and suspenders: restrict access on POSIX. On Windows the + // default DACL inherited from a per-user profile dir is already + // user-only in the common case; an explicit tighten there would + // require touching the DACL which is more code than it's worth + // while DPAPI wrapping is the primary defense. +#if !ZEN_PLATFORM_WINDOWS + std::error_code PermEc; + std::filesystem::permissions(KeyFile, + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write, + std::filesystem::perm_options::replace, + PermEc); +#endif + } + + m_EncryptionKey.assign(reinterpret_cast<const char*>(KeyMaterial.data()), KeyBytes); + m_EncryptionIV.assign(reinterpret_cast<const char*>(KeyMaterial.data() + KeyBytes), IvBytes); +} + +void +AuthCommandLineOptions::CreateAuthMgr(cxxopts::Options& Ops, + const std::filesystem::path& SystemRootDir, + std::unique_ptr<AuthMgr>& InOutAuth, + bool Quiet, + bool Verbose) +{ + ZEN_ASSERT(!SystemRootDir.empty()); + if (InOutAuth) + { + return; + } + + const std::filesystem::path AuthDir = SystemRootDir / "auth"; + + if (m_EncryptionKey.empty() != m_EncryptionIV.empty()) + { + throw OptionParseException( + std::string("'--encryption-aes-key' and '--encryption-aes-iv' must be supplied together or both omitted"), + Ops.help()); + } + + if (m_EncryptionKey.empty() && m_EncryptionIV.empty()) + { + LoadOrCreateMachineKey(AuthDir, Quiet); + } + + AuthConfig AuthMgrConfig = {.RootDirectory = AuthDir, + .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), + .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; + if (!AuthMgrConfig.EncryptionKey.IsValid()) + { + throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", m_EncryptionKey), Ops.help()); + } + if (!AuthMgrConfig.EncryptionIV.IsValid()) + { + throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", m_EncryptionIV), Ops.help()); + } + if (Verbose) + { + ExtendableStringBuilder<128> SB; + SB << "\n RootDirectory: " << AuthMgrConfig.RootDirectory.string(); + SB << "\n EncryptionKey: " << HideSensitiveString(m_EncryptionKey); + SB << "\n EncryptionIV: " << HideSensitiveString(m_EncryptionIV); + ZEN_CONSOLE("Auth: Creating auth manager with:{}", SB.ToString()); + } + InOutAuth = AuthMgr::Create(AuthMgrConfig); +} + +void +AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, + const std::filesystem::path& SystemRootDir, + HttpClientSettings& ClientSettings, + std::string_view HostUrl, + std::unique_ptr<AuthMgr>& Auth, + bool Quiet, + bool Hidden, + bool Verbose) +{ if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) { - CreateAuthMgr(); + CreateAuthMgr(Ops, SystemRootDir, Auth, Quiet, Verbose); std::string ProviderName = m_OpenIdProviderName.empty() ? "Default" : m_OpenIdProviderName; if (Verbose) { @@ -249,7 +379,7 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, } else if (!m_OpenIdProviderName.empty()) { - CreateAuthMgr(); + CreateAuthMgr(Ops, SystemRootDir, Auth, Quiet, Verbose); if (!Quiet) { ZEN_CONSOLE("Auth: Using OpenId provider: {}", m_OpenIdProviderName); @@ -282,7 +412,7 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, if (!ClientSettings.AccessTokenProvider) { - CreateAuthMgr(); + CreateAuthMgr(Ops, SystemRootDir, Auth, Quiet, Verbose); if (!Quiet) { ZEN_CONSOLE("Auth: Using default Open ID provider"); diff --git a/src/zen/authutils.h b/src/zen/authutils.h index fa9670b3f..fa353ab04 100644 --- a/src/zen/authutils.h +++ b/src/zen/authutils.h @@ -3,7 +3,6 @@ #pragma once #include "zen.h" -#include "zenutil/authutils.h" namespace zen { @@ -45,6 +44,15 @@ struct AuthCommandLineOptions bool Quiet, bool Hidden, bool Verbose); + +private: + void CreateAuthMgr(cxxopts::Options& Ops, + const std::filesystem::path& SystemRootDir, + std::unique_ptr<AuthMgr>& InOutAuth, + bool Quiet, + bool Verbose); + + void LoadOrCreateMachineKey(const std::filesystem::path& AuthDir, bool Quiet); }; std::string ReadAccessTokenFromJsonFile(const std::filesystem::path& Path); diff --git a/src/zen/browser_launcher.cpp b/src/zen/browser_launcher.cpp new file mode 100644 index 000000000..a115bf46a --- /dev/null +++ b/src/zen/browser_launcher.cpp @@ -0,0 +1,71 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "browser_launcher.h" + +#include <zenbase/zenbase.h> +#include <zencore/except_fmt.h> +#include <zencore/logging.h> + +#include <stdexcept> +#include <string> + +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +# include <shellapi.h> +#else +# include <spawn.h> +# include <sys/wait.h> +extern char** environ; +#endif + +namespace zen { + +void +LaunchBrowser(std::string_view Url) +{ + if (Url.empty()) + { + throw zen::runtime_error("Cannot launch browser with empty URL"); + } + + bool Success = false; + +#if ZEN_PLATFORM_WINDOWS + std::string UrlZ(Url); + HINSTANCE Result = ShellExecuteA(nullptr, "open", UrlZ.c_str(), nullptr, nullptr, SW_SHOWNORMAL); + Success = reinterpret_cast<intptr_t>(Result) > 32; +#else +# if ZEN_PLATFORM_MAC + const char* Program = "open"; +# elif ZEN_PLATFORM_LINUX + const char* Program = "xdg-open"; +# else + ZEN_NOT_IMPLEMENTED("Browser launching not implemented on this platform"); + const char* Program = nullptr; +# endif + + // Spawn directly via posix_spawnp to avoid the shell entirely, so URL contents + // cannot be interpreted as shell syntax regardless of what characters they contain. + std::string Url_c(Url); + char* const Argv[] = {const_cast<char*>(Program), Url_c.data(), nullptr}; + pid_t Pid = 0; + const int SpawnRc = posix_spawnp(&Pid, Program, nullptr, nullptr, Argv, environ); + if (SpawnRc == 0) + { + int Status = 0; + if (waitpid(Pid, &Status, 0) == Pid) + { + Success = WIFEXITED(Status) && WEXITSTATUS(Status) == 0; + } + } +#endif + + if (!Success) + { + throw zen::runtime_error("Failed to launch browser for '{}'", Url); + } + + ZEN_CONSOLE("Web browser launched for '{}' successfully", Url); +} + +} // namespace zen diff --git a/src/zen/browser_launcher.h b/src/zen/browser_launcher.h new file mode 100644 index 000000000..1b8efcd69 --- /dev/null +++ b/src/zen/browser_launcher.h @@ -0,0 +1,14 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <string_view> + +namespace zen { + +// Opens the given URL in the user's default web browser. +// Throws zen::runtime_error on failure. On POSIX platforms the URL is +// screened for shell metacharacters to prevent command injection. +void LaunchBrowser(std::string_view Url); + +} // namespace zen diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 034d430fd..2580517fa 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -2,6 +2,8 @@ #include "admin_cmd.h" +#include "zenserviceclient.h" + #include <zencore/basicfile.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -41,14 +43,8 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); HttpClient::KeyValueMap Params{{"skipdelete", ToString(m_DryRun)}, {"skipgc", ToString(m_NoGc)}, @@ -168,12 +164,7 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); HttpClient::KeyValueMap Params; Params.Entries.insert({"smallobjects", m_SmallObjects ? "true" : "false"}); @@ -258,7 +249,7 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } Params.Entries.insert({"enablevalidation", m_EnableValidation ? "true" : "false"}); - HttpClient Http = CreateHttpClient(m_HostName); + HttpClient& Http = Service.Http(); if (HttpClient::Response Response = Http.Post("/admin/gc"sv, HttpClient::Accept(HttpContentType::kJSON), Params)) { ZEN_CONSOLE("OK: {}", Response.ToText()); @@ -290,14 +281,8 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Response = Http.Get("/admin/gc"sv, HttpClient::Accept(HttpContentType::kJSON))) { ZEN_CONSOLE("OK: {}", Response.ToText()); @@ -328,14 +313,8 @@ GcStopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Response = Http.Post("/admin/gc-stop"sv, HttpClient::Accept(HttpContentType::kJSON))) { if (Response.StatusCode == HttpResponseCode::Accepted) @@ -377,14 +356,8 @@ JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (m_Cancel) { @@ -460,14 +433,8 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); HttpClient::KeyValueMap Parameters; @@ -522,9 +489,10 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw std::runtime_error(fmt::format("Failed to retrieve {} log path", SourceName)); } - if (!CopyFile(SourcePath, TargetPath, {})) + if (std::error_code Ec = CopyFile(SourcePath, TargetPath, {}); Ec) { - throw std::runtime_error( + throw std::system_error( + Ec, fmt::format("Failed to copy {} log file {} to output file '{}'", SourceName, SourcePath, TargetPath)); } }; @@ -579,16 +547,10 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - zen::HttpClient Http = CreateHttpClient(m_HostName); - - if (zen::HttpClient::Response Response = Http.Post("/admin/flush"sv)) + if (HttpClient::Response Response = Http.Post("/admin/flush"sv)) { ZEN_CONSOLE("OK: {}", Response.ToText()); @@ -624,7 +586,10 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target) CreateDirectories(Target.parent_path()); CopyFileOptions Options; - CopyFile(Source, Target, Options); + if (std::error_code Ec = CopyFile(Source, Target, Options); Ec) + { + throw std::system_error(Ec, fmt::format("Failed to copy '{}' to '{}'", Source, Target)); + } } static bool @@ -638,7 +603,8 @@ TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target CreateDirectories(Target.parent_path()); CopyFileOptions Options; - return CopyFile(Source, Target, Options); + std::error_code Ec = CopyFile(Source, Target, Options); + return !Ec; } void diff --git a/src/zen/cmds/bench_cmd.cpp b/src/zen/cmds/bench_cmd.cpp index b1639105a..c935179e2 100644 --- a/src/zen/cmds/bench_cmd.cpp +++ b/src/zen/cmds/bench_cmd.cpp @@ -1661,7 +1661,7 @@ BenchDiskSubCmd::RunClone(const std::filesystem::path& Dir) try { std::filesystem::path DstPath = Dir / fmt::format("bench_clone_{}.tmp", FileIndex); - if (TryCloneFile(SrcPath, DstPath)) + if (std::error_code CloneEc = TryCloneFile(SrcPath, DstPath); !CloneEc) { CloneCount.fetch_add(1, std::memory_order_relaxed); } diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index cc8315e0b..84d8424aa 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -4,68 +4,34 @@ #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryfile.h> -#include <zencore/compactbinaryfmt.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/compress.h> -#include <zencore/except.h> +#include <zencore/except_fmt.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/parallelwork.h> +#include <zencore/process.h> #include <zencore/scopeguard.h> #include <zencore/session.h> -#include <zencore/stream.h> #include <zencore/string.h> -#include <zencore/trace.h> -#include <zencore/uid.h> +#include <zenhttp/auth/authmgr.h> #include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> -#include <zenremotestore/builds/buildcontent.h> -#include <zenremotestore/builds/buildmanifest.h> -#include <zenremotestore/builds/buildsavedstate.h> -#include <zenremotestore/builds/buildstoragecache.h> -#include <zenremotestore/builds/buildstorageoperations.h> -#include <zenremotestore/builds/buildstorageutil.h> +#include <zenremotestore/builds/buildinspect.h> +#include <zenremotestore/builds/buildprimecache.h> +#include <zenremotestore/builds/buildupdatefolder.h> +#include <zenremotestore/builds/builduploadfolder.h> +#include <zenremotestore/builds/buildvalidatebuildpart.h> #include <zenremotestore/builds/filebuildstorage.h> #include <zenremotestore/builds/jupiterbuildstorage.h> -#include <zenremotestore/chunking/chunkblock.h> -#include <zenremotestore/chunking/chunkedcontent.h> -#include <zenremotestore/chunking/chunkedfile.h> #include <zenremotestore/chunking/chunkingcache.h> #include <zenremotestore/chunking/chunkingcontroller.h> -#include <zenremotestore/filesystemutils.h> -#include <zenremotestore/jupiter/jupiterhost.h> -#include <zenremotestore/operationlogoutput.h> -#include <zenremotestore/transferthreadworkers.h> +#include <zenutil/filesystemutils.h> +#include <zenutil/progress.h> #include <zenutil/wildcard.h> #include <zenutil/workerpools.h> -#include <zenutil/zenserverprocess.h> - -#include "../progressbar.h" #include <signal.h> #include <memory> -#include <numeric> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <tsl/robin_map.h> -#include <tsl/robin_set.h> -#include <json11.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#else -# include <fcntl.h> -# include <sys/file.h> -# include <sys/stat.h> -# include <unistd.h> -#endif - -static const bool DoExtraContentVerify = false; namespace zen { @@ -115,14 +81,6 @@ namespace builds_impl { } }; - struct MemMap - { - void* Handle = nullptr; - void* Data = nullptr; - size_t Size = 0; - std::string Name; - }; - class ZenState { public: @@ -236,1869 +194,103 @@ namespace builds_impl { } } - const std::string ZenFolderName = ".zen"; - std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } - // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; - // } - - std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path) - { - const std::u8string LocalPathString = Path.generic_u8string(); - IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length()); - return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash); - } - - const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; - - const std::string UnsyncFolderName = ".unsync"; - - const std::string UGSFolderName = ".ugs"; - const std::string LegacyZenTempFolderName = ".zen-tmp"; - - const std::vector<std::string> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); - const std::vector<std::string> DefaultExcludeExtensions({}); - + // Debugging knobs for file build storage - always 0 in shipped builds const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const bool SingleThreaded = false; - bool UseSparseFiles = false; - - static bool IsVerbose = false; - static bool IsQuiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - -#undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ - } - - const std::string DefaultAccessTokenEnvVariableName( -#if ZEN_PLATFORM_WINDOWS - "UE-CloudDataCacheAccessToken"sv -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - "UE_CloudDataCacheAccessToken"sv -#endif - - ); - - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; - } - - class FilteredRate - { - public: - FilteredRate() {} - - void Start() - { - if (StartTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs())) - { - LastTimeUS = StartTimeUS.load(); - } - } - } - void Stop() - { - if (EndTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()); - } - } - - void Update(uint64_t Count) - { - if (LastTimeUS == (uint64_t)-1) - { - return; - } - uint64_t TimeUS = Timer.GetElapsedTimeUs(); - uint64_t TimeDeltaUS = TimeUS - LastTimeUS; - if (TimeDeltaUS >= 2000000) - { - uint64_t Delta = Count - LastCount; - uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; - - LastPerSecond = PerSecond; - - LastCount = Count; - - FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; - - LastTimeUS = TimeUS; - } - } - - uint64_t GetCurrent() const // If Stopped - return total count / total time - { - if (LastTimeUS == (uint64_t)-1) - { - return 0; - } - return FilteredPerSecond; - } - - uint64_t GetElapsedTimeUS() const - { - if (StartTimeUS == (uint64_t)-1) - { - return 0; - } - if (EndTimeUS == (uint64_t)-1) - { - return 0; - } - uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS; - return TimeDeltaUS; - } - - bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); } - - private: - Stopwatch Timer; - std::atomic<uint64_t> StartTimeUS = (uint64_t)-1; - std::atomic<uint64_t> EndTimeUS = (uint64_t)-1; - std::atomic<uint64_t> LastTimeUS = (uint64_t)-1; - uint64_t LastCount = 0; - uint64_t LastPerSecond = 0; - uint64_t FilteredPerSecond = 0; - }; - - uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) + void WriteResultObject(const std::filesystem::path& Path, const CbObject& Response) { - if (ElapsedWallTimeUS == 0) - { - return 0; - } - return Count * 1000000 / ElapsedWallTimeUS; - } - - bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory) - { - return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory); - } - - void ValidateBuildPart(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - BuildStorageBase& Storage, - const Oid& BuildId, - Oid BuildPartId, - const std::string_view BuildPartName) - { - ZEN_TRACE_CPU("ValidateBuildPart"); - - ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); - - BuildsOperationValidateBuildPart ValidateOp(Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - BuildPartId, - BuildPartName, - BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose}); - - ValidateOp.Execute(); - - const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount; - const uint64_t DownloadedByteCount = - ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount; - ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}", - DownloadedCount, - NiceBytes(DownloadedByteCount), - NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), - NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); - } - - struct UploadFolderOptions - { - std::filesystem::path TempDir; - uint64_t FindBlockMaxCount; - uint8_t BlockReuseMinPercentLimit; - bool AllowMultiparts; - bool CreateBuild; - bool IgnoreExistingBlocks; - bool UploadToZenCache; - const std::vector<std::string>& ExcludeFolders = DefaultExcludeFolders; - const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions; - }; - - std::vector<std::pair<Oid, std::string>> UploadFolder(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath, - const CbObject& MetaData, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const UploadFolderOptions& Options) - { - ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); - - Stopwatch UploadTimer; - - BuildsOperationUploadFolder UploadOp( - Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - Options.CreateBuild, - std::move(MetaData), - BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .DoExtraContentValidation = DoExtraContentVerify, - .FindBlockMaxCount = Options.FindBlockMaxCount, - .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit, - .AllowMultiparts = Options.AllowMultiparts, - .IgnoreExistingBlocks = Options.IgnoreExistingBlocks, - .TempDir = Options.TempDir, - .ExcludeFolders = Options.ExcludeFolders, - .ExcludeExtensions = Options.ExcludeExtensions, - .ZenExcludeManifestName = ZenExcludeManifestName, - .NonCompressableExtensions = DefaultSplitOnlyExtensions, - .PopulateCache = Options.UploadToZenCache}); - - std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache); - if (AbortFlag) - { - return {}; - } - - ZEN_CONSOLE_VERBOSE( - "Folder scanning stats:" - "\n FoundFileCount: {}" - "\n FoundFileByteCount: {}" - "\n AcceptedFileCount: {}" - "\n AcceptedFileByteCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), - NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Chunking stats:" - "\n FilesProcessed: {}" - "\n FilesChunked: {}" - "\n BytesHashed: {}" - "\n UniqueChunksFound: {}" - "\n UniqueSequencesFound: {}" - "\n UniqueBytesFound: {}" - "\n FilesFoundInCache: {}" - "\n ChunksFoundInCache: {}" - "\n FilesStoredInCache: {}" - "\n ChunksStoredInCache: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_ChunkingStats.FilesProcessed.load(), - UploadOp.m_ChunkingStats.FilesChunked.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), - UploadOp.m_ChunkingStats.UniqueChunksFound.load(), - UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), - NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), - UploadOp.m_ChunkingStats.FilesFoundInCache.load(), - UploadOp.m_ChunkingStats.ChunksFoundInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()), - UploadOp.m_ChunkingStats.FilesStoredInCache.load(), - UploadOp.m_ChunkingStats.ChunksStoredInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()), - NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Find block stats:" - "\n FindBlockTimeMS: {}" - "\n PotentialChunkCount: {}" - "\n PotentialChunkByteCount: {}" - "\n FoundBlockCount: {}" - "\n FoundBlockChunkCount: {}" - "\n FoundBlockByteCount: {}" - "\n AcceptedBlockCount: {}" - "\n NewBlocksCount: {}" - "\n NewBlocksChunkCount: {}" - "\n NewBlocksChunkByteCount: {}", - NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), - UploadOp.m_FindBlocksStats.PotentialChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), - UploadOp.m_FindBlocksStats.FoundBlockCount, - UploadOp.m_FindBlocksStats.FoundBlockChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), - UploadOp.m_FindBlocksStats.AcceptedBlockCount, - UploadOp.m_FindBlocksStats.NewBlocksCount, - UploadOp.m_FindBlocksStats.NewBlocksChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Reuse block stats:" - "\n AcceptedChunkCount: {}" - "\n AcceptedByteCount: {}" - "\n AcceptedRawByteCount: {}" - "\n RejectedBlockCount: {}" - "\n RejectedChunkCount: {}" - "\n RejectedByteCount: {}" - "\n AcceptedReduntantChunkCount: {}" - "\n AcceptedReduntantByteCount: {}", - UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), - UploadOp.m_ReuseBlocksStats.RejectedBlockCount, - UploadOp.m_ReuseBlocksStats.RejectedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), - UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n GeneratedBlockByteCount: {}" - "\n GeneratedBlockCount: {}" - "\n GenerateBlocksElapsedWallTimeUS: {}", - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n ChunkCount: {}" - "\n ChunkByteCount: {}" - "\n CompressedChunkCount: {}" - "\n CompressChunksElapsedWallTimeUS: {}", - UploadOp.m_LooseChunksStats.ChunkCount, - NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Disk stats:" - "\n OpenReadCount: {}" - "\n OpenWriteCount: {}" - "\n ReadCount: {}" - "\n ReadByteCount: {}" - "\n WriteCount: {} ({} cloned)" - "\n WriteByteCount: {} ({} cloned)" - "\n CurrentOpenFileCount: {}", - UploadOp.m_DiskStats.OpenReadCount.load(), - UploadOp.m_DiskStats.OpenWriteCount.load(), - UploadOp.m_DiskStats.ReadCount.load(), - NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), - UploadOp.m_DiskStats.WriteCount.load(), - UploadOp.m_DiskStats.CloneCount.load(), - NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), - NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), - UploadOp.m_DiskStats.CurrentOpenFileCount.load()); - - ZEN_CONSOLE_VERBOSE( - "Upload stats:" - "\n BlockCount: {}" - "\n BlocksBytes: {}" - "\n ChunkCount: {}" - "\n ChunksBytes: {}" - "\n ReadFromDiskBytes: {}" - "\n MultipartAttachmentCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), - UploadOp.m_UploadStats.MultipartAttachmentCount.load(), - NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); - - const double DeltaByteCountPercent = - UploadOp.m_ChunkingStats.BytesHashed > 0 - ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / - (UploadOp.m_ChunkingStats.BytesHashed) - : 0.0; - - const std::string MultipartAttachmentStats = - Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; - - if (!IsQuiet) - { - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), - - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), - NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, - UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)), - NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, - (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - MultipartAttachmentStats); - } - return UploadedParts; - } - - struct VerifyFolderStatistics - { - std::atomic<uint64_t> FilesVerified = 0; - std::atomic<uint64_t> FilesFailed = 0; - std::atomic<uint64_t> ReadBytes = 0; - uint64_t VerifyElapsedWallTimeUs = 0; - }; - - void VerifyFolder(TransferThreadWorkers& Workers, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::filesystem::path& Path, - const std::vector<std::string>& ExcludeFolders, - bool VerifyFileHash, - VerifyFolderStatistics& VerifyFolderStats) - { - ZEN_TRACE_CPU("VerifyFolder"); - - Stopwatch Timer; - - ProgressBar ProgressBar(ProgressMode, "Verify Files"); - - WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool(); - - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size()); - - RwLock ErrorLock; - std::vector<std::string> Errors; - - auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++) - { - if (Work.IsAborted()) - { - break; - } - - Work.ScheduleWork( - VerifyPool, - [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex]( - std::atomic<bool>&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("VerifyFile_work"); - - // TODO: Convert ScheduleWork body to function - - const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); - if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) - { - const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!IsFile(TargetPath)) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); - }); - VerifyFolderStats.FilesFailed++; - } - else - { - std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec)); - if (Ec) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back( - fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk < ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > 0 && VerifyFileHash) - { - const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; - IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); - IoHash RawHash = IoHash::HashBuffer(Buffer); - if (RawHash != ExpectedRawHash) - { - uint64_t FileOffset = 0; - const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash); - const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex]; - for (uint32_t OrderIndex = OrderOffset; - OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex]; - OrderIndex++) - { - uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; - uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; - IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize); - if (IoHash::HashBuffer(FileChunk) != ChunkHash) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format( - "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. " - "Mismatch at chunk {}", - TargetPath, - ExpectedRawHash, - RawHash, - OrderIndex - OrderOffset)); - }); - break; - } - FileOffset += ChunkSize; - } - VerifyFolderStats.FilesFailed++; - } - VerifyFolderStats.ReadBytes += SizeOnDisk; - } - } - } - VerifyFolderStats.FilesVerified++; - } - }, - [&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) { - std::string Description; - try - { - std::rethrow_exception(Ex); - } - catch (const std::exception& Ex) - { - Description = Ex.what(); - } - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", - (Path / Content.Paths[PathIndex]).make_preferred(), - Description)); - }); - VerifyFolderStats.FilesFailed++; - }); - } - - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", - VerifyFolderStats.FilesVerified.load(), - PathCount, - NiceBytes(VerifyFolderStats.ReadBytes.load()), - VerifyFolderStats.FilesFailed.load()); - ProgressBar.UpdateState({.Task = "Verifying files ", - .Details = Details, - .TotalCount = gsl::narrow<uint64_t>(PathCount), - .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - - ProgressBar.Finish(); - if (AbortFlag) - { - return; - } - - for (const std::string& Error : Errors) - { - ZEN_CONSOLE_ERROR("{}", Error); - } - if (!Errors.empty()) - { - throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size())); - } - } - - CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId) - { - Stopwatch GetBuildTimer; - CbObject BuildObject = Storage.GetBuild(BuildId); - if (!IsQuiet) - { - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", - NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["name"sv].AsString(), - NiceBytes(BuildObject.GetSize())); - - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); - } - return BuildObject; - } - - std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths, - const std::span<const std::filesystem::path> Paths) - { - tsl::robin_set<std::string> KnownPathsSet; - KnownPathsSet.reserve(KnownPaths.size()); - for (const std::filesystem::path& LocalPath : KnownPaths) - { - KnownPathsSet.insert(LocalPath.generic_string()); - } - - std::vector<std::filesystem::path> NewPaths; - for (const std::filesystem::path& UntrackedPath : Paths) - { - if (!KnownPathsSet.contains(UntrackedPath.generic_string())) - { - NewPaths.push_back(UntrackedPath); - } - } - return NewPaths; - } - - BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - std::span<const std::filesystem::path> PathsToCheck) - { - FolderContent FolderState; - ChunkedFolderContent ChunkedContent; - { - ProgressBar ProgressBar(ProgressMode, "Check Files"); - FolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - PathsToCheck, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - - if (FolderState.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : FolderState.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - FolderState, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - FolderState.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - ChunkingStats += LocalChunkingStats; - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - } - - return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)}, - .FolderState = FolderState, - .LocalPath = Path}; - } - - BuildSaveState GetLocalContent(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch ReadStateTimer; - bool FileExists = IsFile(StateFilePath); - if (!FileExists) - { - ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path); - return {}; - } - - BuildSaveState SavedLocalState; - try - { - SavedLocalState = ReadBuildSaveStateFile(StateFilePath); - if (!IsQuiet) - { - ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - } - } - catch (const std::exception& Ex) + const MemoryView ResponseView = Response.GetView(); + if (ToLower(Path.extension().string()) == ".cbo") { - ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); - return {}; - } - - FolderContent CurrentLocalFolderState; - { - ProgressBar ProgressBar(ProgressMode, "Check Known Files"); - CurrentLocalFolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - SavedLocalState.FolderState.Paths, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - if (AbortFlag) - { - return {}; - } - - if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState)) - { - const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size(); - std::vector<std::filesystem::path> DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths); - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - - if (!IsQuiet) - { - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocalStatePathCount); - } - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Known Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - UpdatedContent, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - - ChunkingStats += LocalChunkingStats; - - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - if (AbortFlag) - { - return {}; - } - SavedLocalState.State.ChunkedContent = - MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}}); - } - } - else - { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set<std::string> LocalFolderPaths; - LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector<std::filesystem::path> DeletedPaths; - for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths) - { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } - } - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - } - - SavedLocalState.FolderState = CurrentLocalFolderState; - - return SavedLocalState; - } - - ChunkedFolderContent ScanAndChunkFolder( - TransferThreadWorkers& Workers, - GetFolderContentStatistics& GetFolderContentStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder, - std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch Timer; - - ZEN_TRACE_CPU("ScanAndChunkFolder"); - - FolderContent Content = GetFolderContent( - GetFolderContentStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - Workers.GetIOWorkerPool(), - GetUpdateDelayMS(ProgressMode), - [](bool, std::ptrdiff_t) {}, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - BuildState LocalContent = GetLocalContent(Workers, - GetFolderContentStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - ChunkController, - ChunkCache) - .State; - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths); - - BuildState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State; - - ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent}); - - const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); - const uint64_t ChunkedRawSize = - std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Result.Paths.size(), - NiceBytes(TotalRawSize), - Result.ChunkedContent.ChunkHashes.size(), - NiceBytes(ChunkedRawSize), - Path, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - } - return Result; - }; - - struct DownloadOptions - { - std::filesystem::path SystemRootDir; - std::filesystem::path ZenFolderPath; - bool AllowMultiparts = true; - EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; - bool CleanTargetFolder = false; - bool PostDownloadVerify = false; - bool PrimeCacheOnly = false; - bool EnableOtherDownloadsScavenging = true; - bool EnableTargetFolderScavenging = true; - bool AllowFileClone = true; - std::vector<std::string> IncludeWildcards; - std::vector<std::string> ExcludeWildcards; - uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; - bool PopulateCache = true; - bool AppendNewContent = false; - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; - }; - - void DownloadFolder(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const BuildStorageCache::Statistics& StorageCacheStats, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - const std::filesystem::path& DownloadSpecPath, - const std::filesystem::path& Path, - const DownloadOptions& Options) - { - ZEN_TRACE_CPU("DownloadFolder"); - - ProgressBar::SetLogOperationName(ProgressMode, "Download Folder"); - - enum TaskSteps : uint32_t - { - CheckState, - CompareState, - Download, - Verify, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ZEN_ASSERT((!Options.PrimeCacheOnly) || - (Options.PrimeCacheOnly && (Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off))); - - Stopwatch DownloadTimer; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); - - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath); - CreateDirectories(ZenTempFolder); - - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - BuildManifest Manifest; - if (!DownloadSpecPath.empty()) - { - const std::filesystem::path AbsoluteDownloadSpecPath = - DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath); - Manifest = ParseBuildManifest(DownloadSpecPath); - } - - std::vector<ChunkedFolderContent> PartContents; - - std::unique_ptr<ChunkingController> ChunkController; - - std::vector<ChunkBlockDescription> BlockDescriptions; - std::vector<IoHash> LooseChunkHashes; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); - - ChunkedFolderContent RemoteContent = GetRemoteContent(Output, - Storage, - BuildId, - AllBuildParts, - Manifest, - Options.IncludeWildcards, - Options.ExcludeWildcards, - ChunkController, - PartContents, - BlockDescriptions, - LooseChunkHashes, - IsQuiet, - IsVerbose, - DoExtraContentVerify); - - const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - GetFolderContentStatistics LocalFolderScanStats; - ChunkingStatistics ChunkingStats; - - BuildSaveState LocalState; - - if (!Options.PrimeCacheOnly) - { - if (IsDir(Path)) - { - if (!ChunkController && !IsQuiet) - { - ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); - ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - } - std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache()); - - LocalState = GetLocalContent(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - *ChunkController, - *ChunkCache); - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths); - - BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - *ChunkController, - *ChunkCache, - UntrackedPaths); - - if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty()) - { - LocalState.State.ChunkedContent = - MergeChunkedFolderContents(LocalState.State.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent}); - - // TODO: Helper - LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.end()); - LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.end()); - LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.end()); - LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.end()); - } - - if (Options.AppendNewContent) - { - RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent, - RemoteContent, - Options.IncludeWildcards, - Options.ExcludeWildcards); - } -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(RemoteContent, - BlockDescriptions, - LooseChunkHashes, - Options.IncludeWildcards, - Options.ExcludeWildcards); -#endif // ZEN_BUILD_DEBUG - } - else - { - CreateDirectories(Path); - } - } - if (AbortFlag) - { - return; - } - - LocalState.LocalPath = Path; - - { - BuildsSelection::Build RemoteBuildState = {.Id = BuildId, - .IncludeWildcards = Options.IncludeWildcards, - .ExcludeWildcards = Options.ExcludeWildcards}; - RemoteBuildState.Parts.reserve(BuildPartIds.size()); - for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++) - { - RemoteBuildState.Parts.push_back( - {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex], - .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}}); - } - - if (Options.AppendNewContent) - { - LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState)); - } - else - { - LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)}; - } - } - - if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder && - CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent)) - { - if (!IsQuiet) - { - ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); - } - - Stopwatch WriteStateTimer; - - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { - ExtendableStringBuilder<128> BuildPartString; - for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts) - { - BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); - } - - uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); - } - - Stopwatch IndexTimer; - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); - const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - - if (!IsQuiet) - { - ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - - BuildsOperationUpdateFolder Updater( - Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - LocalState.State.ChunkedContent, - LocalLookup, - RemoteContent, - RemoteLookup, - BlockDescriptions, - LooseChunkHashes, - BuildsOperationUpdateFolder::Options{ - .IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .AllowFileClone = Options.AllowFileClone, - .UseSparseFiles = UseSparseFiles, - .SystemRootDir = Options.SystemRootDir, - .ZenFolderPath = Options.ZenFolderPath, - .LargeAttachmentSize = LargeAttachmentSize, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .PartialBlockRequestMode = Options.PartialBlockRequestMode, - .WipeTargetFolder = Options.CleanTargetFolder, - .PrimeCacheOnly = Options.PrimeCacheOnly, - .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, - .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, - .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = Options.ExcludeFolders, - .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, - .PopulateCache = Options.PopulateCache}); - { - ProgressBar::PushLogOperation(ProgressMode, "Download"); - auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); }); - FolderContent UpdatedLocalFolderState; - Updater.Execute(UpdatedLocalFolderState); - - LocalState.State.ChunkedContent = RemoteContent; - LocalState.FolderState = std::move(UpdatedLocalFolderState); - } - - VerifyFolderStatistics VerifyFolderStats; - if (!AbortFlag) - { - if (!Options.PrimeCacheOnly) - { - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - - VerifyFolder(Workers, - RemoteContent, - RemoteLookup, - Path, - Options.ExcludeFolders, - Options.PostDownloadVerify, - VerifyFolderStats); - - Stopwatch WriteStateTimer; - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - -#if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); -#endif // 0 - } - const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() + - Updater.m_DownloadStats.DownloadedBlockCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockCount.load(); - const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() + - Updater.m_DownloadStats.DownloadedBlockByteCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load(); - const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); - - if (!IsQuiet) - { - std::string CloneInfo; - if (Updater.m_DiskStats.CloneByteCount > 0) - { - CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load())); - } - - std::string DownloadDetails; - { - ExtendableStringBuilder<128> SB; - BuildStorageBase::ExtendedStatistics ExtendedDownloadStats; - if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats)) - { - if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty()) - { - for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append(It.first); - SB.Append(": "sv); - SB.Append(NiceBytes(It.second)); - } - } - } - if (Storage.CacheStorage) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append("Cache: "); - SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load())); - } - if (SB.Size() > 0) - { - DownloadDetails = fmt::format(" ({})", SB.ToView()); - } - } - - ZEN_CONSOLE( - "Downloaded build {}, parts:{} in {}\n" - " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n" - " Download: {} ({}) {}bits/s{}\n" - " Write: {} ({}) {}B/s{}\n" - " Clean: {}\n" - " Finalize: {}\n" - " Verify: {}", - BuildId, - BuildPartString.ToView(), - NiceTimeSpanMs(DownloadTimeMs), - - NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) / - 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000), - - DownloadCount, - NiceBytes(DownloadByteCount), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - DownloadDetails, - - Updater.m_DiskStats.WriteCount.load(), - NiceBytes(Updater.m_WrittenChunkByteCount.load()), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())), - CloneInfo, - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); - } - } - } - if (Options.PrimeCacheOnly) - { - if (Storage.CacheStorage) - { - Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) { - if (!IsQuiet) - { - if (Remaining == 0) - { - ZEN_CONSOLE("Build cache upload complete"); - } - else - { - ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); - } - } - return !AbortFlag; - }); - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); - - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder); - } - - void ListBuild(StorageInstance& Storage, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - CbObjectWriter* OptionalStructuredOutput) - { - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId); - OptionalStructuredOutput->AddObject("build"sv, BuildObject); - } - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - if (!AllBuildParts.empty()) - { - Stopwatch GetBuildPartTimer; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("parts"sv); - } - - for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) - { - const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; - const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId); - OptionalStructuredOutput->AddString("partName"sv, BuildPartName); - } - { - if (OptionalStructuredOutput != nullptr) - { - } - else if (!IsQuiet) - { - ZEN_CONSOLE("{}Part: {} ('{}'):\n", - BuildPartIndex > 0 ? "\n" : "", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - } - - std::vector<std::filesystem::path> Paths; - std::vector<IoHash> RawHashes; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - - SourcePlatform Platform; - std::vector<IoHash> SequenceRawHashes; - std::vector<uint32_t> ChunkCounts; - std::vector<uint32_t> AbsoluteChunkOrders; - std::vector<IoHash> LooseChunkHashes; - std::vector<uint64_t> LooseChunkRawSizes; - std::vector<IoHash> BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - Platform, - Paths, - RawHashes, - RawSizes, - Attributes, - SequenceRawHashes, - ChunkCounts, - AbsoluteChunkOrders, - LooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - std::vector<size_t> Order(Paths.size()); - std::iota(Order.begin(), Order.end(), 0); - - std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { - const std::filesystem::path& LhsPath = Paths[Lhs]; - const std::filesystem::path& RhsPath = Paths[Rhs]; - return LhsPath < RhsPath; - }); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("files"sv); - } - { - for (size_t Index : Order) - { - const std::filesystem::path& Path = Paths[Index]; - if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true)) - { - const IoHash& RawHash = RawHashes[Index]; - const uint64_t RawSize = RawSizes[Index]; - const uint32_t Attribute = Attributes[Index]; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - { - OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path)); - OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize); - OptionalStructuredOutput->AddHash("rawHash"sv, RawHash); - switch (Platform) - { - case SourcePlatform::Windows: - OptionalStructuredOutput->AddInteger("attributes"sv, Attribute); - break; - case SourcePlatform::MacOS: - case SourcePlatform::Linux: - OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute)); - break; - default: - throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform)); - } - } - OptionalStructuredOutput->EndObject(); - } - else - { - ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); - } - } - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // "files" - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndObject(); - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // parts - } - } - } - - void DiffFolders(TransferThreadWorkers& Workers, - const std::filesystem::path& BasePath, - const std::filesystem::path& ComparePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const std::vector<std::string>& ExcludeFolders, - const std::vector<std::string>& ExcludeExtensions) - { - ZEN_TRACE_CPU("DiffFolders"); - - ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders"); - - enum TaskSteps : uint32_t - { - CheckBase, - CheckCompare, - Diff, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ChunkedFolderContent BaseFolderContent; - ChunkedFolderContent CompareFolderContent; - - { - auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string& ExcludeExtension : ExcludeExtensions) - { - if (RelativePath.ends_with(ExcludeExtension)) - { - return false; - } - } - return true; - }; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount); - - GetFolderContentStatistics BaseGetFolderContentStats; - ChunkingStatistics BaseChunkingStats; - BaseFolderContent = ScanAndChunkFolder(Workers, - BaseGetFolderContentStats, - BaseChunkingStats, - BasePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - if (AbortFlag) - { - return; - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount); - - GetFolderContentStatistics CompareGetFolderContentStats; - ChunkingStatistics CompareChunkingStats; - CompareFolderContent = ScanAndChunkFolder(Workers, - CompareGetFolderContentStats, - CompareChunkingStats, - ComparePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - - if (AbortFlag) - { - return; - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount); - - std::vector<IoHash> AddedHashes; - std::vector<IoHash> RemovedHashes; - uint64_t RemovedSize = 0; - uint64_t AddedSize = 0; - - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup; - for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - BaseRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup; - for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex]; - if (!BaseRawHashLookup.contains(RawHash)) - { - AddedHashes.push_back(RawHash); - AddedSize += CompareFolderContent.RawSizes[PathIndex]; - } - CompareRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - if (!CompareRawHashLookup.contains(RawHash)) - { - RemovedHashes.push_back(RawHash); - RemovedSize += BaseFolderContent.RawSizes[PathIndex]; - } - } - - uint64_t BaseTotalRawSize = 0; - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex]; - } - - double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; - - ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", - RemovedHashes.size(), - NiceBytes(RemovedSize), - AddedHashes.size(), - NiceBytes(AddedSize), - BaseFolderContent.Paths.size() - RemovedHashes.size(), - NiceBytes(BaseTotalRawSize - RemovedSize), - KeptPercent); - - uint64_t CompareTotalRawSize = 0; - - uint64_t FoundChunkCount = 0; - uint64_t FoundChunkSize = 0; - uint64_t NewChunkCount = 0; - uint64_t NewChunkSize = 0; - const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent); - for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) - { - const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex]; - if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash)) - { - FoundChunkCount++; - FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - else - { - NewChunkCount++; - NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(ResponseView, SB); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - - double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; - double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; - - ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", - FoundChunkCount, - NiceBytes(FoundChunkSize), - FoundPercent, - CompareFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(CompareTotalRawSize), - BaseFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(BaseTotalRawSize), - NewChunkCount, - NiceBytes(NewChunkSize), - NewPercent); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } } // namespace builds_impl -////////////////////////////////////////////////////////////////////////////////////////////////////// -// BuildsCommand — Option-adding helpers -// +////////////////////////////////////////////////////////////////////////// void -BuildsCommand::AddSystemOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddSystemOptions(cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(SystemRootDir), "<systemdir>"); Ops.add_option("", "", "use-sparse-files", "Enable use of sparse files when writing large files. Defaults to true.", - cxxopts::value(m_UseSparseFiles), + cxxopts::value(UseSparseFiles), "<usesparsefiles>"); } void -BuildsCommand::AddCloudOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCloudOptions(cxxopts::Options& Ops) { - m_AuthOptions.AddOptions(Ops); + AuthOptions.AddOptions(Ops); - Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); - Ops.add_option("cloud build", - "", - "url", - "Cloud Builds host url (legacy - use --override-host)", - cxxopts::value(m_OverrideHost), - "<url>"); - Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>"); - Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(OverrideHost), "<override-host>"); + Ops.add_option("cloud build", "", "url", "Cloud Builds host url (legacy - use --override-host)", cxxopts::value(OverrideHost), "<url>"); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(Url), "<cloud-url>"); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(Host), "<host>"); Ops.add_option("cloud build", "", "assume-http2", "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_AssumeHttp2), + cxxopts::value(AssumeHttp2), "<assumehttp2>"); Ops.add_option("cloud build", "", "verbose-http", "Enable verbose option for http client", - cxxopts::value(m_VerboseHttp), + cxxopts::value(VerboseHttp), "<verbosehttp>"); - Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>"); - Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>"); - Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>"); + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(Namespace), "<namespace>"); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(Bucket), "<bucket>"); + Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(AllowRedirect), "<allow-redirect>"); } void -BuildsCommand::AddFileOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddFileOptions(cxxopts::Options& Ops) { - Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>"); + Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(StoragePath), "<storagepath>"); Ops.add_option("filestorage", "", "json-metadata", "Write build, part and block metadata as .json files in addition to .cb files", - cxxopts::value(m_WriteMetadataAsJson), + cxxopts::value(WriteMetadataAsJson), "<jsonmetadata>"); } void -BuildsCommand::AddCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCacheOptions(cxxopts::Options& Ops) { - Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(ZenCacheHost), "<zenhost>"); } void -BuildsCommand::AddOutputOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddOutputOptions(cxxopts::Options& Ops) { - Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>"); - Ops.add_option("output", - "", - "log-progress", - "Write @progress style progress to output", - cxxopts::value(m_LogProgress), - "<logprogress>"); - Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); - Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); + Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(PlainProgress), "<plainprogress>"); + Ops.add_option("output", "", "log-progress", "Write @progress style progress to output", cxxopts::value(LogProgress), "<logprogress>"); + Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(Verbose), "<verbose>"); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(Quiet), "<quiet>"); } void -BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWorkerOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "boost-worker-count", "Increase the number of worker threads - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkerCount), + cxxopts::value(BoostWorkerCount), "<boostworkercount>"); Ops.add_option("", @@ -2106,47 +298,47 @@ BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) "boost-worker-memory", "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to " "be less responsive due to high memory usage", - cxxopts::value(m_BoostWorkerMemory), + cxxopts::value(BoostWorkerMemory), "<boostworkermemory>"); Ops.add_option("", "", "boost-workers", "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkers), + cxxopts::value(BoostWorkers), "<boostworkermemory>"); } void -BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddZenFolderOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "zen-folder-path", - fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName), - cxxopts::value(m_ZenFolderPath), - "<boostworkers>"); + "Path to the zen state and temp folder used by this command. Overrides the command-specific default.", + cxxopts::value(ZenFolderPath), + "<path>"); } void -BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddChunkingCacheOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "chunking-cache-path", "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", - cxxopts::value(m_ChunkingCachePath), + cxxopts::value(ChunkingCachePath), "<chunkingcachepath>"); } void -BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWildcardOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;", - cxxopts::value(m_IncludeWildcard), + cxxopts::value(IncludeWildcard), "<wildcard>"); Ops.add_option("", @@ -2154,46 +346,46 @@ BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) "exclude-wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard " "include filter", - cxxopts::value(m_ExcludeWildcard), + cxxopts::value(ExcludeWildcard), "<excludewildcard>"); } void -BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeFolderOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-folders", "Names of folders to exclude, separated by ;", - cxxopts::value(m_ExcludeFolders), + cxxopts::value(ExcludeFolders), "<excludefolders>"); } void -BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeExtensionsOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-extensions", "Extensions to exclude, separated by ;" "include filter", - cxxopts::value(m_ExcludeExtensions), + cxxopts::value(ExcludeExtensions), "<excludeextensions>"); } void -BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddMultipartOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "allow-multipart", "Allow large attachments to be transfered using multipart protocol. Defaults to true.", - cxxopts::value(m_AllowMultiparts), + cxxopts::value(AllowMultiparts), "<allowmultipart>"); } void -BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddPartialBlockRequestOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2206,12 +398,12 @@ BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) "allowed to host\n" " true = multiple partial block ranges requests per block allowed to zen cache and host\n" "Defaults to 'mixed'.", - cxxopts::value(m_AllowPartialBlockRequests), + cxxopts::value(AllowPartialBlockRequests), "<allowpartialblockrequests>"); } void -BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddAppendNewContentOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2220,26 +412,26 @@ BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) " false = the local content will be replaced by the remote content\n" " true = the remote data will be overlayed on top of local data\n" "Defaults to false.", - cxxopts::value(m_AppendNewContent), + cxxopts::value(AppendNewContent), "<append>"); } BuildsCommand::BuildsCommand() -: m_ListNamespacesSubCmd(*this) -, m_ListSubCmd(*this) -, m_ListBlocksSubCmd(*this) -, m_UploadSubCmd(*this) -, m_DownloadSubCmd(*this) -, m_LsSubCmd(*this) -, m_DiffSubCmd(*this) -, m_FetchBlobSubCmd(*this) -, m_PrimeCacheSubCmd(*this) -, m_PauseSubCmd(*this) -, m_ResumeSubCmd(*this) -, m_AbortSubCmd(*this) -, m_ValidatePartSubCmd(*this) -, m_TestSubCmd(*this) -, m_MultiTestDownloadSubCmd(*this) +: m_ListNamespacesSubCmd(m_Configuration) +, m_ListSubCmd(m_Configuration) +, m_ListBlocksSubCmd(m_Configuration) +, m_UploadSubCmd(m_Configuration) +, m_DownloadSubCmd(m_Configuration) +, m_LsSubCmd(m_Configuration) +, m_DiffSubCmd(m_Configuration) +, m_FetchBlobSubCmd(m_Configuration) +, m_PrimeCacheSubCmd(m_Configuration) +, m_PauseSubCmd(m_Configuration) +, m_ResumeSubCmd(m_Configuration) +, m_AbortSubCmd(m_Configuration) +, m_ValidatePartSubCmd(m_Configuration) +, m_TestSubCmd(m_Configuration) +, m_MultiTestDownloadSubCmd(m_Configuration) { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); @@ -2269,174 +461,250 @@ BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/) { using namespace builds_impl; - signal(SIGINT, SignalCallbackHandler); + // Held in member variables so the handlers stay installed through the + // subcommand's Run() and are restored when BuildsCommand is destroyed. + m_SigIntGuard.emplace(SIGINT, SignalCallbackHandler); #if ZEN_PLATFORM_WINDOWS - signal(SIGBREAK, SignalCallbackHandler); -#endif // ZEN_PLATFORM_WINDOWS + m_SigBreakGuard.emplace(SIGBREAK, SignalCallbackHandler); +#endif // Validate output options - if (m_Verbose && m_Quiet) + if (m_Configuration.Verbose && m_Configuration.Quiet) { throw OptionParseException("'--verbose' conflicts with '--quiet'", {}); } - if (m_LogProgress && m_PlainProgress) + if (m_Configuration.LogProgress && m_Configuration.PlainProgress) { throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {}); } - if (m_LogProgress && m_Quiet) + if (m_Configuration.LogProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--log-progress'", {}); } - if (m_PlainProgress && m_Quiet) + if (m_Configuration.PlainProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {}); } - IsVerbose = m_Verbose; - IsQuiet = m_Quiet; - if (m_LogProgress) - { - ProgressMode = ProgressBar::Mode::Log; - } - else if (m_PlainProgress) + if (m_Configuration.LogProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Log; } - else if (m_Verbose) + else if (m_Configuration.PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Plain; } - else if (IsQuiet) + else if (m_Configuration.Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + m_Configuration.ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + m_Configuration.ProgressMode = ConsoleProgressMode::Pretty; } - if (m_BoostWorkers) + if (m_Configuration.BoostWorkers) { - m_BoostWorkerCount = true; - m_BoostWorkerMemory = true; + m_Configuration.BoostWorkerCount = true; + m_Configuration.BoostWorkerMemory = true; } // Parse system options - if (m_SystemRootDir.empty()) + if (m_Configuration.SystemRootDir.empty()) { - m_SystemRootDir = PickDefaultSystemRootDirectory(); + m_Configuration.SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathInPlace(m_SystemRootDir); - - UseSparseFiles = m_UseSparseFiles; + MakeSafeAbsolutePathInPlace(m_Configuration.SystemRootDir); + MakeSafeAbsolutePathInPlace(m_Configuration.ChunkingCachePath); return true; } +std::atomic<bool>& +BuildsSubCmdBase::AbortFlag() const +{ + return builds_impl::AbortFlag; +} + +std::atomic<bool>& +BuildsSubCmdBase::PauseFlag() const +{ + return builds_impl::PauseFlag; +} + +std::unique_ptr<ProgressBase> +BuildsSubCmdBase::CreateProgress() const +{ + return std::unique_ptr<ProgressBase>(CreateConsoleProgress(m_Config.ProgressMode)); +} + +////////////////////////////////////////////////////////////////////////// + +void +BuildsSubCmdBase::LogBanner() +{ + if (!m_Config.Quiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } +} + void -BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts) +BuildsSubCmdBase::LogWorkersInfo(const TransferThreadWorkers& Workers) { - if (!m_Url.empty()) + if (!m_Config.Quiet) { - if (!m_Host.empty()) + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } +} + +void +BuildsSubCmdBase::EnsureZenFolderExists() +{ + m_CreatedZenFolder = CreateDirectories(GetZenFolderPath()); +} + +void +BuildsSubCmdBase::CleanZenFolder() +{ + using namespace builds_impl; + WorkerThreadPool& Pool = GetSmallWorkerPool(EWorkloadType::Burst); + if (m_CreatedZenFolder) + { + CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), GetZenFolderPath()); + } + else + { + CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), ZenTempFolderPath(GetZenFolderPath())); + } +} + +////////////////////////////////////////////////////////////////////////// + +BuildsSubCmdBase::ResolvedStorage +BuildsSubCmdBase::ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride, + const std::filesystem::path& StoragePathOverride) +{ + ResolvedStorage Resolved{.SystemRootDir = SystemRootDirOverride.empty() ? m_Config.SystemRootDir : SystemRootDirOverride, + .Host = m_Config.Host, + .Namespace = m_Config.Namespace, + .Bucket = m_Config.Bucket, + .StoragePath = StoragePathOverride.empty() ? m_Config.StoragePath : StoragePathOverride}; + + if (!m_Config.Url.empty()) + { + if (!Resolved.Host.empty()) { - throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", Resolved.Host, m_Config.Url), + SubOpts.help()); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", Resolved.Bucket, m_Config.Url), + SubOpts.help()); } if (!BuildId.empty()) { - throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Config.Url), + SubOpts.help()); } - if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId)) + if (!ParseBuildStorageUrl(m_Config.Url, Resolved.Host, Resolved.Namespace, Resolved.Bucket, BuildId)) { - throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help()); + throw OptionParseException( + fmt::format("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", m_Config.Url), + SubOpts.help()); } } - if (!m_OverrideHost.empty() || !m_Host.empty()) + if (!m_Config.OverrideHost.empty() || !Resolved.Host.empty()) { - if (!m_StoragePath.empty()) + if (!Resolved.StoragePath.empty()) { throw OptionParseException( - fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath), + fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", Resolved.StoragePath), SubOpts.help()); } - if (RequireNamespace && m_Namespace.empty()) + if (Options.RequireNamespace && Resolved.Namespace.empty()) { throw OptionParseException("'--namespace' is required", SubOpts.help()); } - if (RequireBucket && m_Bucket.empty()) + if (Options.RequireBucket && Resolved.Bucket.empty()) { throw OptionParseException("'--bucket' is required", SubOpts.help()); } } - else if (m_StoragePath.empty()) + else if (Resolved.StoragePath.empty()) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - MakeSafeAbsolutePathInPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(Resolved.StoragePath); + return Resolved; } StorageInstance -BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr<AuthMgr>& Auth, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::CreateBuildStorage(const std::filesystem::path& ZenFolder, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + std::unique_ptr<AuthMgr>& Auth, + const ResolvedStorage& Resolved) { using namespace builds_impl; - ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts); + // Empty ZenFolder => operate without a temp directory and keep all downloads in memory. + const bool HasZenFolder = !ZenFolder.empty(); + const std::filesystem::path StorageTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "storage" : std::filesystem::path{}; + const std::filesystem::path CacheTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "zencache" : std::filesystem::path{}; - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 2, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; + HttpClientSettings ClientSettings{ + .LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_Config.AssumeHttp2, + .AllowResume = true, + .RetryCount = 2, + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = HasZenFolder ? GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory) + : std::numeric_limits<uint64_t>::max()}; std::string StorageDescription; std::string CacheDescription; StorageInstance Result; - if (!m_Host.empty() || !m_OverrideHost.empty()) + if (!Resolved.Host.empty() || !m_Config.OverrideHost.empty()) { - m_AuthOptions.ParseOptions(SubOpts, - m_SystemRootDir, - ClientSettings, - m_Host.empty() ? m_OverrideHost : m_Host, - Auth, - IsQuiet, - /*Hidden*/ false, - m_Verbose); - - BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*CreateConsoleLogOutput(ProgressMode), + AuthCommandLineOptions AuthOpts = m_Config.AuthOptions; + AuthOpts.ParseOptions(SubOpts, + Resolved.SystemRootDir, + ClientSettings, + Resolved.Host.empty() ? m_Config.OverrideHost : Resolved.Host, + Auth, + m_Config.Quiet, + /*Hidden*/ false, + m_Config.Verbose); + + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(), ClientSettings, - m_Host, - m_OverrideHost, - m_ZenCacheHost, + Resolved.Host, + m_Config.OverrideHost, + m_Config.ZenCacheHost, ZenCacheResolveMode::All, - m_Verbose); + m_Config.Verbose); if (!ResolveRes.Cloud.Address.empty()) { ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, [this]() { return AbortFlag().load(); }); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, - m_Namespace, - m_Bucket, - m_AllowRedirect, - TempPath / "storage"); + Resolved.Namespace, + Resolved.Bucket, + m_Config.AllowRedirect, + StorageTempPath); Result.BuildStorageHost = ResolveRes.Cloud; uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; @@ -2446,8 +714,8 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, ResolveRes.Cloud.Name, (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, NiceLatencyNs(HostLatencyNs)); if (!ResolveRes.Cache.Address.empty()) @@ -2461,19 +729,21 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); + Resolved.Namespace, + Resolved.Bucket, + CacheTempPath, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); Result.CacheHost = ResolveRes.Cache; + Result.SetupCacheSession(ResolveRes.Cache.Address, fmt::format("builds {}", m_SubOptions.program()), GetSessionId()); + uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; CacheDescription = @@ -2483,69 +753,71 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, Result.CacheHttp->GetSessionId(), NiceLatencyNs(CacheLatencyNs)); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } } - else if (!m_StoragePath.empty()) + else if (!Resolved.StoragePath.empty()) { - StorageDescription = fmt::format("folder {}", m_StoragePath); - Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageDescription = fmt::format("folder {}", Resolved.StoragePath); + Result.BuildStorage = CreateFileBuildStorage(Resolved.StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = Resolved.StoragePath.generic_string(), .Name = "Disk", .LatencySec = 1.0 / 100000, // 1 us .Caps = {.MaxRangeCountPerRequest = 2048u}}; - if (!m_ZenCacheHost.empty()) + if (!m_Config.ZenCacheHost.empty()) { - ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_Config.ZenCacheHost, m_Config.AssumeHttp2, m_Config.VerboseHttp); if (TestResult.Success) { Result.CacheHttp = std::make_unique<HttpClient>( - m_ZenCacheHost, + m_Config.ZenCacheHost, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, + .AssumeHttp2 = m_Config.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, - .Name = m_ZenCacheHost, - .AssumeHttp2 = m_AssumeHttp2, + Resolved.Namespace, + Resolved.Bucket, + CacheTempPath, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_Config.ZenCacheHost, + .Name = m_Config.ZenCacheHost, + .AssumeHttp2 = m_Config.AssumeHttp2, .LatencySec = TestResult.LatencySeconds, .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; + Result.SetupCacheSession(m_Config.ZenCacheHost, fmt::format("builds {}", m_SubOptions.program()), GetSessionId()); + CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } @@ -2555,7 +827,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); if (!Result.CacheHost.Name.empty()) @@ -2567,45 +839,33 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, } Oid -BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) { - if (BuildIdStr.length() != Oid::StringLength) + const Oid BuildId = Oid::TryFromHexString(BuildIdStr); + if (BuildId == Oid::Zero) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildIdStr, Oid::StringLength), + fmt::format("'--build-id' ('{}') is malformed, expected {} hex characters", BuildIdStr, Oid::StringLength), SubOpts.help()); } - else if (Oid BuildId = Oid::FromHexString(BuildIdStr); BuildId == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", BuildIdStr), SubOpts.help()); - } - else - { - return BuildId; - } + return BuildId; } Oid -BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) { - if (BuildPartIdStr.length() != Oid::StringLength) + const Oid BuildPartId = Oid::TryFromHexString(BuildPartIdStr); + if (BuildPartId == Oid::Zero) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildPartIdStr, Oid::StringLength), + fmt::format("'--build-part-id' ('{}') is malformed, expected {} hex characters", BuildPartIdStr, Oid::StringLength), SubOpts.help()); } - else if (Oid BuildPartId = Oid::FromHexString(BuildPartIdStr); BuildPartId == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildPartIdStr), SubOpts.help()); - } - else - { - return BuildPartId; - } + return BuildPartId; } std::vector<Oid> -BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) { std::vector<Oid> BuildPartIds; for (const std::string& BuildPartId : BuildPartIdStrs) @@ -2620,7 +880,7 @@ BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs } std::vector<std::string> -BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) { std::vector<std::string> BuildPartNames; for (const std::string& BuildPartName : BuildPartNameStrs) @@ -2635,10 +895,10 @@ BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartName } CbObject -BuildsCommand::ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts) { if (CreateBuild) { @@ -2697,7 +957,7 @@ BuildsCommand::ParseBuildMetadata(bool CreateBuild, } void -BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) { if (Path.empty()) { @@ -2707,7 +967,7 @@ BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) } IoHash -BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) { if (BlobHashStr.empty()) { @@ -2731,7 +991,7 @@ BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& S } void -BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) +BuildsSubCmdBase::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) { auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) { ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { @@ -2757,12 +1017,13 @@ BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, s }); }; - SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); - SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); + SplitAndAppendWildcard(m_Config.IncludeWildcard, OutIncludeWildcards); + SplitAndAppendWildcard(m_Config.ExcludeWildcard, OutExcludeWildcards); } void -BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) +BuildsSubCmdBase::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, + std::vector<std::string>& OutExcludeExtensions) { auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) { ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { @@ -2779,38 +1040,50 @@ BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExclu }); }; - SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); - SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); + SplitAndAppendExclusion(m_Config.ExcludeFolders, OutExcludeFolders); + SplitAndAppendExclusion(m_Config.ExcludeExtensions, OutExcludeExtensions); +} + +void +BuildsSubCmdBase::SetZenFolderPath(const std::filesystem::path& Fallback) +{ + m_ResolvedZenFolderPath = m_Config.ZenFolderPath.empty() ? Fallback : m_Config.ZenFolderPath; + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } void -BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) +BuildsSubCmdBase::ResolveZenFolderPath(const std::filesystem::path& LocalPath) { - if (m_ZenFolderPath.empty()) + using namespace builds_impl; + if (!m_Config.ZenFolderPath.empty()) + { + m_ResolvedZenFolderPath = m_Config.ZenFolderPath; + } + else if (!LocalPath.empty()) + { + m_ResolvedZenFolderPath = LocalPath / ZenFolderName; + } + else { - m_ZenFolderPath = DefaultPath; + m_ResolvedZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } EPartialBlockRequestMode -BuildsCommand::ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts) { - if (PrimeCacheOnly) - { - return EPartialBlockRequestMode::Off; - } - EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_Config.AllowPartialBlockRequests); if (Mode == EPartialBlockRequestMode::Invalid) { - throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_Config.AllowPartialBlockRequests), SubOpts.help()); } return Mode; } void -BuildsCommand::ParseZenProcessId(int& ZenProcessId) +BuildsSubCmdBase::ParseZenProcessId(int& ZenProcessId) { if (ZenProcessId == -1) { @@ -2831,20 +1104,15 @@ BuildsCommand::ParseZenProcessId(int& ZenProcessId) ////////////////////////////////////////////////////////////////////////// -// --------------------------------------------------------------------------- -// Subcommand implementations -// --------------------------------------------------------------------------- - -BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets") -, m_Parent(Parent) +BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-namespaces", "List all namespaces and optionally their buckets") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>"); Opts.add_option("", "", @@ -2859,36 +1127,20 @@ BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) void BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); + cxxopts::Options& Opts = SubOptions(); - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ false, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + std::string DummyBuildId; + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireNamespace = false, .RequireBucket = false}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -2900,29 +1152,20 @@ BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath); - if (ToLower(ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(MakeSafeAbsolutePath(m_ResultPath), Response); } } -BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListSubCmd::BuildsListSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "list", "List builds matching a query") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "query-path", @@ -2942,19 +1185,16 @@ BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list" void BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + if (!m_ResultPath.empty()) + { + LogBanner(); + } + + cxxopts::Options& Opts = SubOptions(); MakeSafeAbsolutePathInPlace(m_QueryPath); MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) - { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - } std::string JsonQuery; if (m_QueryPath.empty()) { @@ -2991,25 +1231,13 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } } + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireBucket = false}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -3021,28 +1249,20 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-blocks", "List blocks for a build") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-blocks", "List blocks for a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -3058,50 +1278,34 @@ BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) void BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } + cxxopts::Options& Opts = SubOptions(); + + MakeSafeAbsolutePathInPlace(m_ResultPath); + if (m_MaxCount == 0) { throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help()); } BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response); - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size()); } @@ -3116,35 +1320,25 @@ BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("upload", "Upload a folder to build storage") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "upload", "Upload a folder to build storage") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", @@ -3192,7 +1386,7 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); Opts.add_option("", "", @@ -3217,50 +1411,31 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) void BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - ZenState InstanceState; + ParsePath(m_Path, Opts); - m_Parent.ParsePath(m_Path, Opts); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); if (m_BuildPartName.empty() && m_ManifestPath.empty()) { m_BuildPartName = m_Path.filename().string(); } - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(m_BuildId, Opts); if (m_BuildId.empty()) { m_BuildId = BuildId.ToString(); @@ -3269,28 +1444,31 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Oid BuildPartId; if (!m_BuildPartId.empty()) { - BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + BuildPartId = ParseBuildPartId(m_BuildPartId, Opts); } - CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); + CbObject MetaData = ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); - const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath()); + const std::filesystem::path TempDir = ZenTempFolderPath(GetZenFolderPath()); std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -3302,89 +1480,89 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = TempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = m_CreateBuild, .IgnoreExistingBlocks = m_Clean, .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, .ExcludeFolders = ExcludeFolders, .ExcludeExtensions = ExcludeExtensions}); - if (!AbortFlag) + if (!AbortFlag()) { if (m_PostUploadVerify) { for (const auto& Part : UploadedParts) { - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + Part.first, + Part.second); } } } - if (true) + if (!m_Config.Quiet) { - if (!IsQuiet) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.BuildStorageHost.Name, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } + const BuildStorageResolveResult::Host& Host = Storage.BuildStorageHost; + const BuildStorageBase::Statistics& Stats = StorageStats; + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Host.Name, + NiceBytes(Stats.TotalBytesRead.load()), + NiceBytes(Stats.TotalBytesWritten.load()), + Stats.TotalRequestCount.load(), + Stats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(Stats.TotalExecutionTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0, + Stats.TotalRequestCount.load() > 0 ? NiceTimeSpanMs(Stats.TotalRequestTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Upload aborted"); } } -BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("download", "Download a build to a local folder") -, m_Parent(Parent) -{ - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddExcludeFolderOption(Opts); +////////////////////////////////////////////////////////////////////////// - Opts.add_option("cache", - "", - "cache-prime-only", - "Only download blobs missing in cache and upload to cache", - cxxopts::value(m_PrimeCacheOnly), - "<cacheprimeonly>"); +BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "download", "Download a build to a local folder") +{ + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddExcludeFolderOption(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); Opts.add_option("", @@ -3405,9 +1583,9 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) "Upload data downloaded from remote host to zen cache", cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); Opts.add_option( "", @@ -3437,139 +1615,107 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) void BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - ZenState InstanceState; - - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - if (m_PostDownloadVerify && m_PrimeCacheOnly) - { - throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", Opts.help()); - } + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Preserve zen folder on exit - <zen>/current_state.cbo tracks what has been downloaded into + // this folder and is consulted by later download operations against the same target. + ResolveZenFolderPath(m_Path); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); - if (m_Clean && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled"); - } - - if (m_Force && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled"); - } - - if (m_Parent.m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled"); - } + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(m_PrimeCacheOnly, Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); - if (m_Parent.m_AppendNewContent && m_Clean) + if (m_Config.AppendNewContent && m_Clean) { throw OptionParseException("'--append' conflicts with '--clean'", Opts.help()); } std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); DownloadFolder( - *Output, + ConsoleLog(), + *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, BuildPartIds, BuildPartNames, m_DownloadSpecPath, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_Config.SystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = m_Clean, .PostDownloadVerify = m_PostDownloadVerify, - .PrimeCacheOnly = m_PrimeCacheOnly, .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force, .EnableTargetFolderScavenging = !m_Force, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory), + .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory), .PopulateCache = m_UploadToZenCache, - .AppendNewContent = m_Parent.m_AppendNewContent, + .AppendNewContent = m_Config.AppendNewContent, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles, .ExcludeFolders = ExcludeFolders}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Download aborted"); } } -BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsLsSubCmd::BuildsLsSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "ls", "List files in a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); @@ -3594,43 +1740,31 @@ BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "Lis void BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); + + builds_impl::ZenState InstanceState; std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - - m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context + ParseFileFilters(IncludeWildcards, ExcludeWildcards); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); std::unique_ptr<CbObjectWriter> StructuredOutput; if (!m_ResultPath.empty()) @@ -3639,38 +1773,30 @@ BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) StructuredOutput = std::make_unique<CbObjectWriter>(); } - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); + ListBuild(m_Config.Quiet, Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); if (StructuredOutput) { CbObject Response = StructuredOutput->Save(); - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("List build aborted"); } } -BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "diff", "Diff two local folders") { - auto& Opts = SubOptions(); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); Opts.add_option("", @@ -3686,38 +1812,28 @@ BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff" void BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); if (m_DiffPath.empty()) { throw OptionParseException("'--compare-path' is required", Opts.help()); } MakeSafeAbsolutePathInPlace(m_DiffPath); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); StandardChunkingControllerSettings ChunkingSettings; std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); if (m_OnlyChunked) { @@ -3729,24 +1845,37 @@ BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ChunkingSettings.SplitAndCompressExtensions.end()); } - DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); - if (AbortFlag) + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + + DiffFolders(*Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + Workers, + m_Path, + m_DiffPath, + *ChunkController, + *ChunkCache, + ExcludeFolders, + ExcludeExtensions); + if (AbortFlag()) { throw std::runtime_error("Diff folders aborted"); } } -BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "fetch-blob", "Fetch and validate a specific blob") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>"); Opts.parse_positional({"build-id", "blob-hash"}); @@ -3756,80 +1885,55 @@ BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) void BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - const Oid BuildId = Oid::FromHexString(m_BuildId); + IoHash BlobHash = ParseBlobHash(m_BlobHash, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) + ValidateBlob(AbortFlag(), *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag()) { throw std::runtime_error("Fetch blob aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); } } -BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "prime-cache", "Prime the zen cache with build data") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); Opts.add_option("", @@ -3845,47 +1949,31 @@ BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) void BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = + CreateBuildStorage(GetZenFolderPath(), {.BoostCacheBackgroundWorkers = true}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ true, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, m_Config.Quiet); std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); @@ -3897,100 +1985,101 @@ BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) AllBuildPartIds.push_back(BuildPart.first); } - ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + Progress->SetLogOperationName("Prime Cache"); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - - BuildsOperationPrimeCache PrimeOp(*Output, + BuildsOperationPrimeCache PrimeOp(ConsoleLog(), + *Progress, Storage, - AbortFlag, - PauseFlag, + AbortFlag(), + PauseFlag(), Workers.GetNetworkPool(), BuildId, AllBuildPartIds, - BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .ZenFolderPath = m_Parent.GetZenFolderPath(), + BuildsOperationPrimeCache::Options{.IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .ZenFolderPath = GetZenFolderPath(), .LargeAttachmentSize = PreferredMultipartChunkSize * 4u, .PreferredMultipartChunkSize = PreferredMultipartChunkSize, .ForceUpload = m_Force}, - StorageCacheStats); + CacheStats); PrimeOp.Execute(); - if (!IsQuiet) + if (!m_Config.Quiet && Storage.CacheStorage) { - if (Storage.CacheStorage) - { - ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", - StorageCacheStats.PutBlobCount.load(), - NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheHost.Name); - } + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", + CacheStats.PutBlobCount.load(), + NiceBytes(CacheStats.PutBlobByteCount), + Storage.CacheHost.Name); } } -BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +namespace { + void AddProcessIdOption(cxxopts::Options& Opts, int& ZenProcessId) + { + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(ZenProcessId), "<pid>"); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); + } +} // namespace + +BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "pause", "Pause a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(1); } -BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "resume", "Resume a paused zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(false); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(0); } -BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "abort", "Abort a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Abort.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(1); } -BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("validate-part", "Validate a build part") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "validate-part", "Validate a build part") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -4011,42 +2100,24 @@ BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) void BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + builds_impl::ZenState InstanceState; - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + Oid BuildId = ParseBuildId(m_BuildId, Opts); if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { @@ -4055,33 +2126,46 @@ BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Opts.help()); } - const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(m_BuildPartId, Opts); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Validate build part failed"); } } -BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsTestSubCmd::BuildsTestSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "test", "Run an upload/download test cycle") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); - Parent.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddChunkingCacheOptions(Opts); + Config.AddMultipartOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "", "enable-scavenge", @@ -4101,38 +2185,35 @@ BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test" void BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + const bool CreatedTempStorage = m_Config.OverrideHost.empty() && m_Config.StoragePath.empty(); + if (CreatedTempStorage) { - m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath); - CreateDirectories(m_Parent.m_StoragePath); - m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string(); + m_TestStoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), m_TestStoragePath); + CreateDirectories(m_TestStoragePath); + m_TestStoragePath = m_TestStoragePath.generic_string(); } - auto StorageGuard = MakeGuard([this]() { - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + auto StorageGuard = MakeGuard([this, CreatedTempStorage]() { + if (CreatedTempStorage) { - DeleteDirectories(m_Parent.m_StoragePath); + DeleteDirectories(m_TestStoragePath); } }); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -4143,37 +2224,35 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3"); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); - auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() { - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + auto DownloadGuard = MakeGuard([this, &Workers, DownloadPath, DownloadPath2, DownloadPath3]() { + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); }); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::unique_ptr<AuthMgr> Auth; std::string TestBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - TestBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - m_BuildId = Oid::NewOid().ToString(); - m_BuildPartId = Oid::NewOid().ToString(); - m_CreateBuild = true; - - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + const ResolvedStorage Resolved = ParseStorageOptions(TestBuildId, {}, Opts, m_TestSystemRootDir, m_TestStoragePath); + // Place scratch next to the folder under test so upload does not self-include, and so the + // scratch area shares the volume of the source data. + { + const std::u8string PathStr = m_Path.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length()); + SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash)); + } + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, StorageCacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); }); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + m_BuildId = BuildId.ToString(); + m_BuildPartId = BuildPartId.ToString(); + m_CreateBuild = true; auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -4196,18 +2275,21 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); } - const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + const std::filesystem::path UploadTempDir = ZenTempFolderPath(GetZenFolderPath()); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -4219,12 +2301,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload build)"); } @@ -4232,9 +2316,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path); - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), Oid::NewOid(), Oid::NewOid(), m_BuildPartName, @@ -4246,52 +2333,70 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); } } - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); - if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty()) + if (!m_Config.IncludeWildcard.empty() || !m_Config.ExcludeWildcard.empty()) { - auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); }); + auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); }); ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4301,66 +2406,79 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = ExcludeWildcards, .ExcludeWildcards = IncludeWildcards, - .AppendNewContent = true}); - if (AbortFlag) + .AppendNewContent = true, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = {}, .ExcludeWildcards = {}, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4368,25 +2486,30 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4394,17 +2517,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download identical target)"); } @@ -4436,7 +2561,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) return true; }; - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + ParallelWork Work(AbortFlag(), PauseFlag(), WorkerThreadPool::EMode::EnableBacklog); uint32_t Randomizer = 0; auto FileSizeIt = DownloadContent.FileSizes.begin(); @@ -4454,8 +2579,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { Work.ScheduleWork( Workers.GetIOWorkerPool(), - [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { - if (!AbortFlag) + [this, SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { + if (!AbortFlag()) { bool WasReadOnly = SetFileReadOnly(FilePath, false); { @@ -4483,7 +2608,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) case 1: { (void)SetFileReadOnly(FilePath, false); - (void)RemoveFile(FilePath); + std::error_code Ec; + const bool Removed = RemoveFile(FilePath, Ec); + if (!Removed || Ec) + { + throw zen::runtime_error("ScrambleDir: failed to delete '{}'. Reason: '{}'", + FilePath, + Ec ? Ec.message() : "file no longer present"); + } } break; default: @@ -4496,15 +2628,18 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_UNUSED(IsAborted, IsPaused); ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); }); - ZEN_ASSERT(!AbortFlag.load()); + ZEN_ASSERT(!AbortFlag().load()); ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }; ScrambleDir(DownloadPath); ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4512,17 +2647,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled target)"); } @@ -4539,9 +2676,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId2, BuildPartId2, m_BuildPartName, @@ -4553,22 +2693,38 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload scrambled)"); } - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4576,133 +2732,156 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download scrambled)"); } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath2, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath2 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath3, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath3 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } } -BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "multi-test-download", "Download multiple builds sequentially as a test") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>"); Opts.add_option("", @@ -4724,42 +2903,37 @@ BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Pare void BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); - - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); + ParsePath(m_Path, Opts); + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {}, Opts, m_TestSystemRootDir); + // Place scratch next to the download target so the scratch area shares its volume. + { + const std::u8string PathStr = m_Path.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length()); + SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash)); + } + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); }); - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); + + std::unique_ptr<ProgressBase> Progress = CreateProgress(); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -4769,35 +2943,40 @@ BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), Opts.help()); } - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, /*BuildPartIds,*/ {}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = BuildIdString == m_BuildIds.front(), .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Multitest aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("\n"); } } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 7ef71e176..338318757 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -2,65 +2,227 @@ #pragma once -#include "../authutils.h" -#include "../zen.h" - -#include <zenhttp/auth/authmgr.h> -#include <zenhttp/httpclientauth.h> +#include <zencore/scopeguard.h> #include <zenremotestore/builds/buildstoragecache.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/partialblockrequestmode.h> -#include <filesystem> +#include <zenremotestore/transferthreadworkers.h> + +#include "authutils.h" +#include "consoleprogress.h" + +#include <optional> namespace zen { -class BuildsCommand; +class ProgressBase; +class AuthMgr; + +struct CreateBuildStorageOptions +{ + bool RequireNamespace = true; + bool RequireBucket = true; + bool BoostCacheBackgroundWorkers = false; +}; + +////////////////////////////////////////////////////////////////////////// -class BuildsListNamespacesSubCmd : public ZenSubCmdBase +struct BuildsConfiguration +{ + std::filesystem::path SystemRootDir; + bool UseSparseFiles = true; + + bool PlainProgress = false; + bool LogProgress = false; + bool Verbose = false; + bool Quiet = false; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + bool BoostWorkerCount = false; + bool BoostWorkerMemory = false; + bool BoostWorkers = false; + + std::string OverrideHost; + std::string Host; + std::string Url; + bool AssumeHttp2 = false; + bool VerboseHttp = false; + bool AllowRedirect = false; + std::string Namespace; + std::string Bucket; + + std::filesystem::path StoragePath; + bool WriteMetadataAsJson = false; + + std::string ZenCacheHost; + + AuthCommandLineOptions AuthOptions; + + std::string IncludeWildcard; + std::string ExcludeWildcard; + std::string ExcludeFolders; + std::string ExcludeExtensions; + + std::filesystem::path ChunkingCachePath; + + bool AllowMultiparts = true; + std::string AllowPartialBlockRequests = "true"; + + bool AppendNewContent = false; + + std::filesystem::path ZenFolderPath; + + void AddSystemOptions(cxxopts::Options& Ops); + void AddCloudOptions(cxxopts::Options& Ops); + void AddFileOptions(cxxopts::Options& Ops); + void AddCacheOptions(cxxopts::Options& Ops); + void AddOutputOptions(cxxopts::Options& Ops); + void AddWorkerOptions(cxxopts::Options& Ops); + void AddZenFolderOptions(cxxopts::Options& Ops); + void AddChunkingCacheOptions(cxxopts::Options& Ops); + void AddWildcardOptions(cxxopts::Options& Ops); + void AddExcludeFolderOption(cxxopts::Options& Ops); + void AddExcludeExtensionsOption(cxxopts::Options& Ops); + void AddMultipartOptions(cxxopts::Options& Ops); + void AddPartialBlockRequestOptions(cxxopts::Options& Ops); + void AddAppendNewContentOptions(cxxopts::Options& Ops); +}; + +////////////////////////////////////////////////////////////////////////// + +class BuildsSubCmdBase : public ZenSubCmdBase { public: - explicit BuildsListNamespacesSubCmd(BuildsCommand& Parent); + BuildsSubCmdBase(BuildsConfiguration& Config, std::string_view Name, std::string_view Description) + : ZenSubCmdBase(Name, Description) + , m_Config(Config) + { + } + +protected: + const BuildsConfiguration& m_Config; + std::filesystem::path m_ResolvedZenFolderPath; + // Set by EnsureZenFolderExists: true if this command newly created the zen folder + // (so CleanZenFolder may remove the whole folder), false if it already existed + // (in which case only the temp subfolder is wiped, to preserve any pre-existing state). + bool m_CreatedZenFolder = false; + + struct ResolvedStorage + { + std::filesystem::path SystemRootDir; + std::string Host; + std::string Namespace; + std::string Bucket; + std::filesystem::path StoragePath; + }; + + void LogBanner(); + void LogWorkersInfo(const TransferThreadWorkers& Workers); + + // SystemRootDirOverride / StoragePathOverride: empty = fall back to m_Config. + ResolvedStorage ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride = {}, + const std::filesystem::path& StoragePathOverride = {}); + + // Builds the storage instance using the supplied Resolved values. ZenFolder is the final zen folder + // path (caller resolves; pass GetZenFolderPath() after ResolveZenFolderPath). If ZenFolder is empty + // the storage runs without a temp directory and keeps all downloads in memory. + // Caller owns Stats/Auth lifetime. Stats must outlive the returned StorageInstance. + StorageInstance CreateBuildStorage(const std::filesystem::path& ZenFolder, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& OutStorageStats, + BuildStorageCache::Statistics& OutCacheStats, + std::unique_ptr<AuthMgr>& OutAuth, + const ResolvedStorage& Resolved); + Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); + Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); + std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts); + std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts); + CbObject ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts); + void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); + IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); + EPartialBlockRequestMode ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts); + void ParseZenProcessId(int& ZenProcessId); + void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards); + void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions); + + // Resolves the zen folder using this chain: --zen-folder-path; else LocalPath/ZenFolderName + // if LocalPath is non-empty; else cwd/ZenFolderName. Read-only commands that do not need a + // zen folder should skip this and pass an empty path to CreateBuildStorage. + void ResolveZenFolderPath(const std::filesystem::path& LocalPath); + // Assigns the zen folder to --zen-folder-path if given, else to Fallback directly (no + // ZenFolderName appended). For commands that want a specific scratch location rather than + // the standard LocalPath/.zen chain. + void SetZenFolderPath(const std::filesystem::path& Fallback); + const std::filesystem::path& GetZenFolderPath() const { return m_ResolvedZenFolderPath; } + // Creates the resolved zen folder and records whether it had to be created. + // Use together with CleanZenFolder via MakeGuard. + void EnsureZenFolderExists(); + + std::atomic<bool>& AbortFlag() const; + std::atomic<bool>& PauseFlag() const; + std::unique_ptr<ProgressBase> CreateProgress() const; + + // Wipes temp state produced by the command. If EnsureZenFolderExists created the zen folder + // it is removed outright; otherwise only the temp subfolder is wiped so pre-existing state + // (e.g. current_state.cbo from a prior download in the same folder) is preserved. + void CleanZenFolder(); +}; + +////////////////////////////////////////////////////////////////////////// + +class BuildsListNamespacesSubCmd : public BuildsSubCmdBase +{ +public: + explicit BuildsListNamespacesSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; bool m_Recursive = false; std::filesystem::path m_ResultPath; }; -class BuildsListSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsListSubCmd : public BuildsSubCmdBase { public: - explicit BuildsListSubCmd(BuildsCommand& Parent); + explicit BuildsListSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_QueryPath; std::filesystem::path m_ResultPath; }; -class BuildsListBlocksSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsListBlocksSubCmd : public BuildsSubCmdBase { public: - explicit BuildsListBlocksSubCmd(BuildsCommand& Parent); + explicit BuildsListBlocksSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::filesystem::path m_ResultPath; uint32_t m_MaxCount = 16; }; -class BuildsUploadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsUploadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsUploadSubCmd(BuildsCommand& Parent); + explicit BuildsUploadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::string m_BuildId; std::string m_BuildPartId; @@ -76,14 +238,15 @@ private: bool m_UploadToZenCache = true; }; -class BuildsDownloadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsDownloadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsDownloadSubCmd(BuildsCommand& Parent); + explicit BuildsDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::string m_BuildId; std::vector<std::string> m_BuildPartIds; @@ -94,117 +257,129 @@ private: bool m_EnableScavenging = true; std::filesystem::path m_DownloadSpecPath; bool m_UploadToZenCache = true; - bool m_PrimeCacheOnly = false; bool m_AllowFileClone = true; }; -class BuildsLsSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsLsSubCmd : public BuildsSubCmdBase { public: - explicit BuildsLsSubCmd(BuildsCommand& Parent); + explicit BuildsLsSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::vector<std::string> m_BuildPartIds; std::vector<std::string> m_BuildPartNames; std::filesystem::path m_ResultPath; }; -class BuildsDiffSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsDiffSubCmd : public BuildsSubCmdBase { public: - explicit BuildsDiffSubCmd(BuildsCommand& Parent); + explicit BuildsDiffSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::filesystem::path m_DiffPath; bool m_OnlyChunked = false; }; -class BuildsFetchBlobSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsFetchBlobSubCmd : public BuildsSubCmdBase { public: - explicit BuildsFetchBlobSubCmd(BuildsCommand& Parent); + explicit BuildsFetchBlobSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - std::string m_BuildId; - std::string m_BlobHash; + std::string m_BuildId; + std::string m_BlobHash; }; -class BuildsPrimeCacheSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsPrimeCacheSubCmd : public BuildsSubCmdBase { public: - explicit BuildsPrimeCacheSubCmd(BuildsCommand& Parent); + explicit BuildsPrimeCacheSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::vector<std::string> m_BuildPartIds; std::vector<std::string> m_BuildPartNames; bool m_Force = false; }; -class BuildsPauseSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsPauseSubCmd : public BuildsSubCmdBase { public: - explicit BuildsPauseSubCmd(BuildsCommand& Parent); + explicit BuildsPauseSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsResumeSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsResumeSubCmd : public BuildsSubCmdBase { public: - explicit BuildsResumeSubCmd(BuildsCommand& Parent); + explicit BuildsResumeSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsAbortSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsAbortSubCmd : public BuildsSubCmdBase { public: - explicit BuildsAbortSubCmd(BuildsCommand& Parent); + explicit BuildsAbortSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsValidatePartSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsValidatePartSubCmd : public BuildsSubCmdBase { public: - explicit BuildsValidatePartSubCmd(BuildsCommand& Parent); + explicit BuildsValidatePartSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - std::string m_BuildId; - std::string m_BuildPartId; - std::string m_BuildPartName; + std::string m_BuildId; + std::string m_BuildPartId; + std::string m_BuildPartName; }; -class BuildsTestSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsTestSubCmd : public BuildsSubCmdBase { public: - explicit BuildsTestSubCmd(BuildsCommand& Parent); + explicit BuildsTestSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; + // Fixture-only overrides of SystemRootDir/StoragePath; passed to ParseStorageOptions explicitly. + std::filesystem::path m_TestSystemRootDir; + std::filesystem::path m_TestStoragePath; + std::filesystem::path m_Path; std::string m_BuildPartName; std::string m_BuildId; @@ -217,21 +392,27 @@ private: bool m_AllowFileClone = true; }; -class BuildsMultiTestDownloadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsMultiTestDownloadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent); + explicit BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; + // Fixture-only override of SystemRootDir; passed to CreateStorage explicitly. + std::filesystem::path m_TestSystemRootDir; + std::filesystem::path m_Path; std::vector<std::string> m_BuildIds; bool m_EnableScavenging = true; bool m_AllowFileClone = true; }; -class BuildsCommand : public CacheStoreCmdWithSubCommands +////////////////////////////////////////////////////////////////////////// + +class BuildsCommand : public ZenCmdWithSubCommands { public: static constexpr char Name[] = "builds"; @@ -243,99 +424,14 @@ public: cxxopts::Options& Options() override { return m_Options; } - // Option-adding helpers (called by subcommand constructors) - void AddSystemOptions(cxxopts::Options& Ops); - void AddCloudOptions(cxxopts::Options& Ops); - void AddFileOptions(cxxopts::Options& Ops); - void AddCacheOptions(cxxopts::Options& Ops); - void AddOutputOptions(cxxopts::Options& Ops); - void AddWorkerOptions(cxxopts::Options& Ops); - void AddZenFolderOptions(cxxopts::Options& Ops); - void AddChunkingCacheOptions(cxxopts::Options& Ops); - void AddWildcardOptions(cxxopts::Options& Ops); - void AddExcludeFolderOption(cxxopts::Options& Ops); - void AddExcludeExtensionsOption(cxxopts::Options& Ops); - void AddMultipartOptions(cxxopts::Options& Ops); - void AddPartialBlockRequestOptions(cxxopts::Options& Ops); - void AddAppendNewContentOptions(cxxopts::Options& Ops); - - // Shared parsing/factory methods used by subcommand Run() implementations - void ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts); - StorageInstance CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr<AuthMgr>& Auth, - cxxopts::Options& SubOpts); - Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); - Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); - std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts); - std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts); - CbObject ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts); - void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); - IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); - EPartialBlockRequestMode ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts); - void ParseZenProcessId(int& ZenProcessId); - void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards); - void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions); - - void ResolveZenFolderPath(const std::filesystem::path& DefaultPath); - const std::filesystem::path& GetZenFolderPath() const { return m_ZenFolderPath; } - - cxxopts::Options m_Options{Name, Description}; - std::string m_SubCommand; - - // Shared state populated by AddXxxOptions helpers (bound via cxxopts::value references) - std::filesystem::path m_SystemRootDir; - bool m_UseSparseFiles = true; - - bool m_PlainProgress = false; - bool m_LogProgress = false; - bool m_Verbose = false; - bool m_Quiet = false; - bool m_BoostWorkerCount = false; - bool m_BoostWorkerMemory = false; - bool m_BoostWorkers = false; - - // cloud builds - std::string m_OverrideHost; - std::string m_Host; - std::string m_Url; - bool m_AssumeHttp2 = false; - bool m_VerboseHttp = false; - bool m_AllowRedirect = false; - std::string m_Namespace; - std::string m_Bucket; - - std::filesystem::path m_StoragePath; - bool m_WriteMetadataAsJson = false; - - std::string m_ZenCacheHost; - - AuthCommandLineOptions m_AuthOptions; - - std::string m_IncludeWildcard; - std::string m_ExcludeWildcard; - std::string m_ExcludeFolders; - std::string m_ExcludeExtensions; - - std::filesystem::path m_ChunkingCachePath; - - bool m_AllowMultiparts = true; - std::string m_AllowPartialBlockRequests = "true"; - - bool m_AppendNewContent = false; + BuildsConfiguration& GetConfiguration() { return m_Configuration; } + const BuildsConfiguration& GetConfiguration() const { return m_Configuration; } private: - std::filesystem::path m_ZenFolderPath; + cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + BuildsConfiguration m_Configuration; -protected: BuildsListNamespacesSubCmd m_ListNamespacesSubCmd; BuildsListSubCmd m_ListSubCmd; BuildsListBlocksSubCmd m_ListBlocksSubCmd; @@ -352,6 +448,11 @@ protected: BuildsTestSubCmd m_TestSubCmd; BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd; + std::optional<ScopedSignalHandler> m_SigIntGuard; +#if ZEN_PLATFORM_WINDOWS + std::optional<ScopedSignalHandler> m_SigBreakGuard; +#endif + bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override; }; diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index a8c15f119..f93a5318c 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -2,25 +2,40 @@ #include "cache_cmd.h" +#include "zenserviceclient.h" + #include <zencore/compactbinarybuilder.h> #include <zencore/compress.h> #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/process.h> #include <zencore/scopeguard.h> +#include <zencore/session.h> +#include <zencore/stream.h> #include <zencore/thread.h> +#include <zencore/timer.h> #include <zencore/workthreadpool.h> +#include <zenhttp/formatters.h> #include <zenhttp/httpclient.h> #include <zenhttp/httpcommon.h> #include <zenhttp/packageformat.h> #include <zenstore/cache/cachepolicy.h> +#include <zenutil/rpcrecording.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END #include <memory> #include <random> namespace zen { +using namespace std::literals; + namespace { IoBuffer CreateRandomBlob(uint64_t Size) { @@ -56,37 +71,112 @@ namespace { } } // namespace -DropCommand::DropCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheCommand + +CacheCommand::CacheCommand() { m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); - m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); - m_Options.parse_positional({"namespace", "bucket"}); + + AddSubCommand(m_DetailsSubCmd); + AddSubCommand(m_DropSubCmd); + AddSubCommand(m_GenSubCmd); + AddSubCommand(m_GetSubCmd); + AddSubCommand(m_InfoSubCmd); + AddSubCommand(m_RecordSubCmd); + AddSubCommand(m_ReplaySubCmd); + AddSubCommand(m_StatsSubCmd); } -DropCommand::~DropCommand() = default; +CacheCommand::~CacheCommand() = default; -void -DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +//////////////////////////////////////////////////////////////////////////////// +// CacheSubCmdBase + +CacheSubCmdBase::CacheSubCmdBase(std::string_view Name, std::string_view Description) : ZenSubCmdBase(Name, Description) { - ZEN_UNUSED(GlobalOptions); + m_SubOptions.add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} - if (!ParseOptions(argc, argv)) +void +CacheSubCmdBase::ResolveHost() +{ + m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); + if (m_HostName.empty()) { - return; + throw OptionParseException("Unable to resolve server specification", m_SubOptions.help()); } +} - m_HostName = ResolveTargetHostSpec(m_HostName); +//////////////////////////////////////////////////////////////////////////////// +// Legacy shim dispatcher - if (m_HostName.empty()) +namespace cache_legacy_shim { + static void Dispatch(std::span<const std::string_view> Injected, const ZenCliOptions& GlobalOptions, int argc, char** argv) { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); + // cxxopts treats argv as writable char** in the style of C main(argv). + // Stage the injected tokens in writable std::string storage so we never + // hand out pointers to string literals. + std::vector<std::string> Storage; + Storage.reserve(Injected.size()); + for (std::string_view Token : Injected) + { + Storage.emplace_back(Token); + } + + std::vector<char*> NewArgv; + NewArgv.reserve(static_cast<size_t>(argc) + Storage.size()); + NewArgv.push_back(argv[0]); + for (std::string& Token : Storage) + { + NewArgv.push_back(Token.data()); + } + for (int i = 1; i < argc; ++i) + { + NewArgv.push_back(argv[i]); + } + + CacheCommand Impl; + Impl.Run(GlobalOptions, static_cast<int>(NewArgv.size()), NewArgv.data()); } + void RunAs(const char* SubCommandName, const ZenCliOptions& GlobalOptions, int argc, char** argv) + { + const std::string_view Tokens[] = {std::string_view(SubCommandName)}; + Dispatch(Tokens, GlobalOptions, argc, argv); + } +} // namespace cache_legacy_shim + +// RpcStopRecordingCommand is unique among legacy shims in that it needs to +// inject two tokens ("record" and "stop") rather than a single subcommand name. +void +RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + const std::string_view Tokens[] = {"record"sv, "stop"sv}; + cache_legacy_shim::Dispatch(Tokens, GlobalOptions, argc, argv); +} + +//////////////////////////////////////////////////////////////////////////////// +// CacheDropSubCmd + +CacheDropSubCmd::CacheDropSubCmd() : CacheSubCmdBase("drop", "Drop cache namespace or bucket") +{ + m_SubOptions.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); + m_SubOptions.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); + m_SubOptions.parse_positional({"namespace", "bucket"}); +} + +void +CacheDropSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "drop"}); + HttpClient& Http = Service.Http(); + if (m_NamespaceName.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } std::string Url; @@ -94,18 +184,16 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BucketName.empty()) { - DropDescription = fmt::format("cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); + DropDescription = fmt::format("cache namespace '{}' from '{}'", m_NamespaceName, Service.HostSpec()); Url = fmt::format("/z$/{}", m_NamespaceName); } else { - DropDescription = fmt::format("cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); + DropDescription = fmt::format("cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, Service.HostSpec()); Url = fmt::format("/z$/{}/{}", m_NamespaceName, m_BucketName); } ZEN_CONSOLE("Dropping {}", DropDescription); - - HttpClient Http = CreateHttpClient(m_HostName); if (HttpClient::Response Response = Http.Delete(Url)) { ZEN_CONSOLE("{}", Response.ToText()); @@ -116,54 +204,43 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } -CacheInfoCommand::CacheInfoCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheInfoSubCmd + +CacheInfoSubCmd::CacheInfoSubCmd() : CacheSubCmdBase("info", "Info on cache, namespace or bucket") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); - m_Options.add_option("", - "", - "bucketsizes", - "Comma delimited list of bucket names to get size info from, * to get info on all buckets", - cxxopts::value(m_SizeInfoBucketNames), - "<bucketnames>"); - m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); - m_Options.add_option("", "", "bucketsize", "Show detailed bucket size info", cxxopts::value(m_BucketSizeInfo), "<bucketsize>"); - - m_Options.parse_positional({"namespace", "bucket"}); + m_SubOptions.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); + m_SubOptions.add_option("", + "", + "bucketsizes", + "Comma delimited list of bucket names to get size info from, * to get info on all buckets", + cxxopts::value(m_SizeInfoBucketNames), + "<bucketnames>"); + m_SubOptions.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); + m_SubOptions.add_option("", "", "bucketsize", "Show detailed bucket size info", cxxopts::value(m_BucketSizeInfo), "<bucketsize>"); + m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>"); + m_SubOptions.parse_positional({"namespace", "bucket"}); } -CacheInfoCommand::~CacheInfoCommand() = default; - void -CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheInfoSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "info"}); + HttpClient& Http = Service.Http(); std::string Url; - if (m_HostName.empty()) + if (m_NamespaceName.empty()) { if (!m_SizeInfoBucketNames.empty()) { - throw OptionParseException("'--bucketsizes' requires '--namespace'", m_Options.help()); + throw OptionParseException("'--bucketsizes' requires '--namespace'", m_SubOptions.help()); } if (m_BucketSizeInfo) { - throw OptionParseException("'--bucketsize' requires '--namespace' and '--bucket'", m_Options.help()); + throw OptionParseException("'--bucketsize' requires '--namespace' and '--bucket'", m_SubOptions.help()); } - ZEN_CONSOLE("Info on cache from '{}'", m_HostName); + ZEN_CONSOLE("Info on cache from '{}'", Service.HostSpec()); Url = "/z$"; } else if (m_BucketName.empty()) @@ -171,18 +248,18 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BucketSizeInfo) { throw OptionParseException(fmt::format("'--bucketsize' requires '--namespace' and '--bucket' ('{}')", m_BucketName), - m_Options.help()); + m_SubOptions.help()); } - ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); + ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, Service.HostSpec()); Url = fmt::format("/z$/{}", m_NamespaceName); } else { if (!m_SizeInfoBucketNames.empty()) { - throw OptionParseException("'--bucketsizes' conflicts with '--bucket'", m_Options.help()); + throw OptionParseException("'--bucketsizes' conflicts with '--bucket'", m_SubOptions.help()); } - ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); + ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, Service.HostSpec()); Url = fmt::format("/z$/{}/{}", m_NamespaceName, m_BucketName); } @@ -196,8 +273,9 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Parameters.Entries.insert({"bucketsize", "true"}); } - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON), Parameters)) + const ZenContentType AcceptType = m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON; + + if (HttpClient::Response Response = Http.Get(Url, HttpClient::Accept(AcceptType), Parameters)) { ZEN_CONSOLE("{}", Response.ToText()); } @@ -207,76 +285,62 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } -CacheStatsCommand::CacheStatsCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheStatsSubCmd + +CacheStatsSubCmd::CacheStatsSubCmd() : CacheSubCmdBase("stats", "Stats on cache") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>"); } -CacheStatsCommand::~CacheStatsCommand() = default; - void -CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheStatsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "stats"}); + HttpClient& Http = Service.Http(); - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + const ZenContentType AcceptType = m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON; - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = Http.Get("/stats/z$", HttpClient::Accept(ZenContentType::kJSON))) + if (HttpClient::Response Response = Http.Get("/stats/z$", HttpClient::Accept(AcceptType))) { ZEN_CONSOLE("{}", Response.ToText()); } else { - Response.ThrowError("Info failed"); + Response.ThrowError("Stats failed"); } } -CacheDetailsCommand::CacheDetailsCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheDetailsSubCmd + +CacheDetailsSubCmd::CacheDetailsSubCmd() : CacheSubCmdBase("details", "Details on cache") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "c", "csv", "Info on csv format", cxxopts::value(m_CSV), "<csv>"); - m_Options.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "<details>"); - m_Options.add_option("", - "a", - "attachmentdetails", - "Get detailed information about attachments", - cxxopts::value(m_AttachmentDetails), - "<attachmentdetails>"); - m_Options.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), "<namespace>"); - m_Options.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), "<bucket>"); - m_Options.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), "<valuekey>"); + m_SubOptions.add_option("", "c", "csv", "Output as CSV instead of JSON", cxxopts::value(m_CSV), "<csv>"); + m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>"); + m_SubOptions.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "<details>"); + m_SubOptions.add_option("", + "a", + "attachmentdetails", + "Get detailed information about attachments", + cxxopts::value(m_AttachmentDetails), + "<attachmentdetails>"); + m_SubOptions.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), "<namespace>"); + m_SubOptions.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), "<bucket>"); + m_SubOptions.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), "<valuekey>"); } -CacheDetailsCommand::~CacheDetailsCommand() = default; - void -CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheDetailsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "details"}); + HttpClient& Http = Service.Http(); - if (!ParseOptions(argc, argv)) + if (m_CSV && m_YAML) { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); + throw OptionParseException("'--csv' conflicts with '--yaml'", m_SubOptions.help()); } HttpClient::KeyValueMap Parameters; @@ -296,7 +360,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar } else { - Headers = HttpClient::Accept(ZenContentType::kJSON); + Headers = HttpClient::Accept(m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON); } std::string Url; @@ -304,11 +368,11 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar { if (m_Namespace.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } if (m_Bucket.empty()) { - throw OptionParseException("'--bucket' is required", m_Options.help()); + throw OptionParseException("'--bucket' is required", m_SubOptions.help()); } Url = fmt::format("/z$/details$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey); } @@ -316,7 +380,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar { if (m_Namespace.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } Url = fmt::format("/z$/details$/{}/{}", m_Namespace, m_Bucket); } @@ -329,61 +393,49 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar Url = "/z$/details$"; } - HttpClient Http = CreateHttpClient(m_HostName); if (HttpClient::Response Response = Http.Get(Url, Headers, Parameters)) { ZEN_CONSOLE("{}", Response.ToText()); } else { - Response.ThrowError("Info failed"); + Response.ThrowError("Details failed"); } } -CacheGenerateCommand::CacheGenerateCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheGenSubCmd + +CacheGenSubCmd::CacheGenSubCmd() : CacheSubCmdBase("gen", "Generates cache values into a bucket") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options + m_SubOptions .add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), "<namespace>"); - m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>"); - m_Options.add_option("", "", "count", "Number of cache values/records to generate", cxxopts::value(m_Count), "<count>"); - m_Options.add_option("", "", "min-size", "Minimum size of cache value/attachments", cxxopts::value(m_MinSize), "<min>"); - m_Options.add_option("", "", "max-size", "Maximum size of cache value/attachments", cxxopts::value(m_MaxSize), "<max>"); - m_Options.add_option("", - "", - "min-attachments", - "Minimum number of attachments when creating record based values", - cxxopts::value(m_MinAttachmentCount), - "<minattachments>"); - m_Options.add_option("", - "", - "max-attachments", - "Minimum number of attachments when creating record based values, 0 to only create cache values", - cxxopts::value(m_MaxAttachmentCount), - "<maxattachments>"); - m_Options.parse_positional({"namespace", "bucket", "count"}); - m_Options.positional_help("namespace bucket count"); + m_SubOptions.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>"); + m_SubOptions.add_option("", "", "count", "Number of cache values/records to generate", cxxopts::value(m_Count), "<count>"); + m_SubOptions.add_option("", "", "min-size", "Minimum size of cache value/attachments", cxxopts::value(m_MinSize), "<min>"); + m_SubOptions.add_option("", "", "max-size", "Maximum size of cache value/attachments", cxxopts::value(m_MaxSize), "<max>"); + m_SubOptions.add_option("", + "", + "min-attachments", + "Minimum number of attachments when creating record based values", + cxxopts::value(m_MinAttachmentCount), + "<minattachments>"); + m_SubOptions.add_option("", + "", + "max-attachments", + "Minimum number of attachments when creating record based values, 0 to only create cache values", + cxxopts::value(m_MaxAttachmentCount), + "<maxattachments>"); + m_SubOptions.parse_positional({"namespace", "bucket", "count"}); + m_SubOptions.positional_help("namespace bucket count"); } -CacheGenerateCommand::~CacheGenerateCommand() = default; - void -CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheGenSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "gen"}); + HttpClient& Http = Service.Http(); if (m_MaxSize == 0 && m_MinSize == 0) { @@ -400,6 +452,16 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } } + // The size-range expansion below requires MinSize >= 1 (it uses + // `MinSize - 1` as a uniform distribution upper bound, which would + // underflow on an unsigned zero) and MaxSize >= MinSize. + if (m_MinSize == 0 || m_MaxSize < m_MinSize) + { + throw OptionParseException( + fmt::format("'--min-size' ({}) must be >= 1 and '--max-size' ({}) must be >= '--min-size'", m_MinSize, m_MaxSize), + m_SubOptions.help()); + } + std::vector<std::uniform_int_distribution<uint64_t>> Variations; std::vector<size_t> SizeRanges; SizeRanges.push_back(m_MinSize); @@ -431,8 +493,6 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a std::uniform_int_distribution<uint64_t> KeyDistribution; - HttpClient Http = CreateHttpClient(m_HostName); - auto GeneratePutCacheValueRequest( [this, &KeyDistribution, &Generator](std::span<std::uint64_t> BatchSizes, uint64_t RequestIndex) -> CbPackage { CbPackage Package; @@ -583,68 +643,56 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } } -CacheGetCommand::CacheGetCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheGetSubCmd + +CacheGetSubCmd::CacheGetSubCmd() : CacheSubCmdBase("get", "Get cache values/records or attachments from a bucket") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options - .add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), "<namespace>"); - m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>"); - m_Options.add_option("", "v", "valuekey", "Cache entry iohash id", cxxopts::value(m_ValueKey), "<valuekey>"); - m_Options.add_option("", - "a", - "attachmenthash", - "For a cache entry record, get a particular attachment based on the 'RawHash'", - cxxopts::value(m_AttachmentHash), - "<attachmenthash>"); - m_Options.add_option("", "o", "output-path", "File path for output data", cxxopts::value(m_OutputPath), "<path>"); - m_Options.add_option("", "t", "text", "Ouput content of cache entry record as text", cxxopts::value(m_AsText), "<text>"); - m_Options + m_SubOptions.add_option("", "n", "namespace", "Namespace of the cache entry", cxxopts::value(m_Namespace), "<namespace>"); + m_SubOptions.add_option("", "b", "bucket", "Bucket of the cache entry", cxxopts::value(m_Bucket), "<bucket>"); + m_SubOptions.add_option("", "v", "valuekey", "Cache entry iohash id", cxxopts::value(m_ValueKey), "<valuekey>"); + m_SubOptions.add_option("", + "a", + "attachmenthash", + "For a cache entry record, get a particular attachment based on the 'RawHash'", + cxxopts::value(m_AttachmentHash), + "<attachmenthash>"); + m_SubOptions.add_option("", "o", "output-path", "File path for output data", cxxopts::value(m_OutputPath), "<path>"); + m_SubOptions.add_option("", "t", "text", "Output content of cache entry record as text", cxxopts::value(m_AsText), "<text>"); + m_SubOptions .add_option("", "d", "decompress", "Decompress data when applicable. Default = true", cxxopts::value(m_Decompress), "<decompress>"); - m_Options.parse_positional({"namespace", "bucket", "valuekey", "attachmenthash"}); - m_Options.positional_help("namespace bucket valuekey attachmenthash"); + m_SubOptions.parse_positional({"namespace", "bucket", "valuekey", "attachmenthash"}); + m_SubOptions.positional_help("namespace bucket valuekey attachmenthash"); } -CacheGetCommand::~CacheGetCommand() = default; - void -CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheGetSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); - using namespace std::literals; - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "get"}); + HttpClient& Http = Service.Http(); if (m_Namespace.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } if (m_Bucket.empty()) { - throw OptionParseException("'--bucket' is required", m_Options.help()); + throw OptionParseException("'--bucket' is required", m_SubOptions.help()); } if (m_ValueKey.empty()) { - throw OptionParseException("'--valuekey' is required", m_Options.help()); + throw OptionParseException("'--valuekey' is required", m_SubOptions.help()); } IoHash ValueId; if (!IoHash::TryParse(m_ValueKey, ValueId)) { - throw OptionParseException(fmt::format("'--value-key' ('{}') is malformed", m_ValueKey), m_Options.help()); + throw OptionParseException(fmt::format("'--valuekey' ('{}') is malformed", m_ValueKey), m_SubOptions.help()); } IoHash AttachmentHash; @@ -652,11 +700,14 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!IoHash::TryParse(m_AttachmentHash, AttachmentHash)) { - throw OptionParseException(fmt::format("'--attachmenthash' ('{}') is malformed", m_AttachmentHash), m_Options.help()); + throw OptionParseException(fmt::format("'--attachmenthash' ('{}') is malformed", m_AttachmentHash), m_SubOptions.help()); } } - HttpClient Http = CreateHttpClient(m_HostName); + if (m_OutputPath.empty() && !m_AsText) + { + throw OptionParseException("'--output-path' is required (or pass '--as-text' to print to stdout)", m_SubOptions.help()); + } if (!m_OutputPath.empty()) { @@ -669,18 +720,19 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CreateDirectories(m_OutputPath.parent_path()); } } - if (m_OutputPath.empty()) - { - m_OutputPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); - } - std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey); + std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, ValueId); if (AttachmentHash != IoHash::Zero) { Url = fmt::format("{}/{}", Url, AttachmentHash); } if (HttpClient::Response Result = Http.Download(Url, std::filesystem::temp_directory_path()); Result) { + // `Http.Download` parks the payload in the system temp dir and returns + // a buffer that already has delete-on-close set, so every exit path + // (exception, fallback WriteFile, `--as-text` console print) reaps it. + // A successful MoveToFile below clears the flag so the payload's + // handle-close doesn't delete the caller's output afterwards. auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer { IoHash RawHash; uint64_t RawSize; @@ -707,7 +759,17 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - if (!MoveToFile(m_OutputPath, ChunkData)) + if (std::error_code MoveEc = MoveToFile(m_OutputPath, ChunkData); MoveEc) + { + // The file was renamed into place; clearing DeleteOnClose prevents + // the move'd-out file at m_OutputPath from being deleted when the + // payload's handle closes. When m_Decompress is false ChunkData + // shares a core with ResponsePayload so either clear suffices; + // when decompressed ChunkData is in-memory and MoveToFile would + // have failed, so we don't reach this branch. + Result.ResponsePayload.SetDeleteOnClose(false); + } + else { WriteFile(m_OutputPath, ChunkData); } @@ -720,4 +782,428 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } +//////////////////////////////////////////////////////////////////////////////// +// CacheRecordSubCmd + +CacheRecordSubCmd::CacheRecordSubCmd() +: CacheSubCmdBase("record", "Start recording cache rpc requests ('cache record <path>'), or stop ('cache record stop')") +{ + m_SubOptions.add_option("", "p", "path", "Recording file path, or 'stop' to stop recording", cxxopts::value(m_Path), "<path>"); + m_SubOptions.parse_positional("path"); + m_SubOptions.positional_help("<path>|stop"); +} + +void +CacheRecordSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record"}); + HttpClient& Http = Service.Http(); + + if (m_Path == "stop") + { + if (HttpClient::Response Response = Http.Post("/z$/exec$/stop-recording"sv)) + { + ZEN_CONSOLE("{}", Response.ToText()); + } + else + { + Response.ThrowError("Failed to stop recording"); + } + return; + } + + if (m_Path.empty()) + { + throw OptionParseException("recording path is required (use '<path>' to start, 'stop' to stop)", m_SubOptions.help()); + } + + if (HttpClient::Response Response = + Http.Post("/z$/exec$/start-recording"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap({{"path", m_Path}}))) + { + ZEN_CONSOLE("{}", Response.ToText()); + } + else + { + Response.ThrowError("Failed to start recording"); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// CacheReplaySubCmd + +CacheReplaySubCmd::CacheReplaySubCmd() : CacheSubCmdBase("replay", "Replays a previously recorded session of rpc requests") +{ + m_SubOptions.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>"); + m_SubOptions.add_option("", "", "dry", "Do a dry run", cxxopts::value(m_DryRun), "<enable>"); + m_SubOptions.add_option("", + "w", + "numthreads", + "Number of worker threads per process", + cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", GetHardwareConcurrency())), + "<count>"); + m_SubOptions.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), "<onhost>"); + m_SubOptions.add_option("", + "", + "showmethodstats", + "Show statistics of which RPC methods are used", + cxxopts::value(m_ShowMethodStats), + "<showmethodstats>"); + m_SubOptions.add_option("", + "", + "offset", + "Offset into request recording to start replay", + cxxopts::value(m_Offset)->default_value("0"), + "<offset>"); + m_SubOptions.add_option("", + "", + "stride", + "Stride for request recording when replaying requests", + cxxopts::value(m_Stride)->default_value("1"), + "<stride>"); + m_SubOptions.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), "<count>"); + m_SubOptions.add_option("", + "", + "forceallowlocalrefs", + "Force enable local refs in requests", + cxxopts::value(m_ForceAllowLocalRefs), + "<enable>"); + m_SubOptions + .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), "<enable>"); + m_SubOptions.add_option("", + "", + "forceallowlocalhandlerefs", + "Force enable local refs as handles in requests", + cxxopts::value(m_ForceAllowLocalHandleRef), + "<enable>"); + m_SubOptions.add_option("", + "", + "disablelocalhandlerefs", + "Force disable local refs as handles in requests", + cxxopts::value(m_DisableLocalHandleRefs), + "<enable>"); + m_SubOptions.add_option("", + "", + "forceallowpartiallocalrefs", + "Force enable local refs for all sizes", + cxxopts::value(m_ForceAllowPartialLocalRefs), + "<enable>"); + m_SubOptions.add_option("", + "", + "disablepartiallocalrefs", + "Force disable local refs for all sizes", + cxxopts::value(m_DisablePartialLocalRefs), + "<enable>"); + m_SubOptions.parse_positional("path"); +} + +void +CacheReplaySubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + if (m_RecordingPath.empty()) + { + throw OptionParseException("'--path' is required", m_SubOptions.help()); + } + + if (!IsDir(m_RecordingPath)) + { + throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); + } + + if (m_Stride == 0) + { + throw OptionParseException("'--stride' must be >= 1", m_SubOptions.help()); + } + + m_ThreadCount = Max(m_ThreadCount, 1); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "replay"}); + m_HostName = Service.HostSpec(); + + ZEN_CONSOLE("Replay '{}' (start offset {}, stride {}) to '{}', {} threads", + m_RecordingPath, + m_Offset, + m_Stride, + m_HostName, + m_ThreadCount); + + Stopwatch TotalTimer; + + if (m_OnHost) + { + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = + Http.Post("/z$/exec$/replay-recording"sv, + HttpClient::KeyValueMap{}, + HttpClient::KeyValueMap({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}}))) + { + ZEN_CONSOLE("{}", Response.ToText()); + + return; + } + else + { + Response.ThrowError("Failed to start replay"); + } + } + + std::unique_ptr<cache::IRpcRequestReplayer> Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true); + uint64_t EntryCount = Replayer->GetRequestCount(); + + if (m_Offset >= EntryCount) + { + ZEN_CONSOLE("Offset {} is at or past the end of the recording ({} entries); nothing to replay", m_Offset, EntryCount); + return; + } + + std::atomic_uint64_t EntryOffset = m_Offset; + std::atomic_uint64_t BytesSent = 0; + std::atomic_uint64_t BytesReceived = 0; + + Stopwatch Timer; + + // The subcommand API does not receive argv, so look the zen executable path + // up from the current process to spawn child workers. + const std::filesystem::path SelfExePath = GetRunningExecutablePath(); + + if (m_ProcessCount > 1) + { + std::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses; + WorkerProcesses.resize(m_ProcessCount); + + ProcessMonitor Monitor; + for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex) + { + std::string CommandLine = + fmt::format("{} cache replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv, + SelfExePath.string(), + m_HostName, + m_RecordingPath, + m_Stride == 1 ? 0 : m_Offset + ProcessIndex, + m_Stride, + m_ThreadCount, + 1); + CreateProcResult Result(CreateProc(SelfExePath, CommandLine)); + WorkerProcesses[ProcessIndex] = std::make_unique<ProcessHandle>(); + WorkerProcesses[ProcessIndex]->Initialize(Result); + Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid()); + } + while (Monitor.IsRunning()) + { + ZEN_CONSOLE("Waiting for worker processes..."); + Sleep(1000); + } + return; + } + else + { + std::map<std::string, size_t> MethodTypes; + RwLock MethodTypesLock; + + WorkerThreadPool WorkerPool(m_ThreadCount); + + Latch WorkLatch(m_ThreadCount); + for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex) + { + WorkerPool.ScheduleWork( + [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + + std::map<std::string, size_t> LocalMethodTypes; + + auto ReduceTypes = MakeGuard([&] { + RwLock::ExclusiveLockScope __(MethodTypesLock); + + for (auto& Entry : LocalMethodTypes) + { + MethodTypes[Entry.first] += Entry.second; + } + }); + + HttpClient Http = CacheCommand::CreateHttpClient(m_HostName); + + uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride); + while (EntryIndex < EntryCount) + { + IoBuffer Payload; + const zen::cache::RecordedRequestInfo RequestInfo = Replayer->GetRequest(EntryIndex, /* out */ Payload); + + if (RequestInfo != zen::cache::RecordedRequestInfo::NullRequest) + { + CbPackage RequestPackage; + CbObject Request; + + switch (RequestInfo.ContentType) + { + case ZenContentType::kCbPackage: + { + if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage)) + { + Request = RequestPackage.GetObject(); + } + } + break; + case ZenContentType::kCbObject: + { + Request = LoadCompactBinaryObject(Payload); + } + break; + } + + RpcAcceptOptions OriginalAcceptOptions = static_cast<RpcAcceptOptions>(Request["AcceptFlags"sv].AsUInt16(0u)); + int OriginalProcessPid = Request["Pid"sv].AsInt32(0); + + int AdjustedPid = 0; + RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone; + + if (!m_DisableLocalRefs) + { + if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) || + m_ForceAllowLocalRefs) + { + AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences; + if (!m_DisablePartialLocalRefs) + { + if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) || + m_ForceAllowPartialLocalRefs) + { + AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences; + } + } + if (!m_DisableLocalHandleRefs) + { + if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef) + { + AdjustedPid = GetCurrentProcessId(); + } + } + } + } + + if (m_ShowMethodStats) + { + std::string MethodName = std::string(Request["Method"sv].AsString()); + if (auto It = LocalMethodTypes.find(MethodName); It != LocalMethodTypes.end()) + { + It->second++; + } + else + { + LocalMethodTypes[MethodName] = 1; + } + } + + if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid) + { + CbObjectWriter RequestCopyWriter; + for (const CbFieldView& Field : Request) + { + if (!Field.HasName()) + { + RequestCopyWriter.AddField(Field); + continue; + } + std::string_view FieldName = Field.GetName(); + if (FieldName == "Pid"sv) + { + continue; + } + if (FieldName == "AcceptFlags"sv) + { + continue; + } + RequestCopyWriter.AddField(FieldName, Field); + } + if (AdjustedPid != 0) + { + RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid); + } + if (AdjustedAcceptOptions != RpcAcceptOptions::kNone) + { + RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast<uint16_t>(AdjustedAcceptOptions)); + } + + if (RequestInfo.ContentType == ZenContentType::kCbPackage) + { + RequestPackage.SetObject(RequestCopyWriter.Save()); + std::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage); + std::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end()); + Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer(); + } + else + { + RequestCopyWriter.Finalize(); + Payload = IoBuffer(RequestCopyWriter.GetSaveSize()); + RequestCopyWriter.Save(Payload.GetMutableView()); + } + } + + if (!m_DryRun) + { + Http.SetSessionId(RequestInfo.SessionId); + Payload.SetContentType(RequestInfo.ContentType); + + HttpClient::Response Response = + Http.Post("/z$/$rpc", Payload, {HttpClient::Accept(RequestInfo.AcceptType)}); + + BytesSent.fetch_add(Payload.GetSize()); + if (!Response) + { + ZEN_CONSOLE_ERROR("{}", Response); + break; + } + BytesReceived.fetch_add(Response.DownloadedBytes); + } + } + + EntryIndex = EntryOffset.fetch_add(m_Stride); + } + }, + WorkerThreadPool::EMode::EnableBacklog); + } + + while (!WorkLatch.Wait(1000)) + { + // EntryCount > m_Offset is guaranteed by the early-return above. + // EntryOffset atomically overshoots EntryCount (fetch_add past the + // end) when the workload finishes, so clamp before subtracting. + const uint64_t RequestsTotal = (EntryCount - m_Offset) / m_Stride; + const uint64_t CurrentOffset = EntryOffset.load(); + const uint64_t RequestsRemaining = CurrentOffset < EntryCount ? (EntryCount - CurrentOffset) / m_Stride : 0; + const uint64_t PercentDone = RequestsTotal > 0 ? (RequestsTotal - RequestsRemaining) * 100 / RequestsTotal : 100; + + ZEN_CONSOLE("[{:3}%] [{}] {} requests, {} remaining (sent {}, received {})", + PercentDone, + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + RequestsTotal, + RequestsRemaining, + NiceBytes(BytesSent.load()), + NiceBytes(BytesReceived.load())); + } + + if (m_ShowMethodStats) + { + for (const auto& It : MethodTypes) + { + ZEN_CONSOLE("{:18}: {:10}", It.first, It.second); + } + } + } + + const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride; + const uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); + const uint64_t Sent = BytesSent.load(); + const uint64_t Received = BytesReceived.load(); + + ZEN_CONSOLE("Processed requests: {} ({}), payloads sent {} ({}), payloads received {} ({}) in {}.\nTotal runtime: {}", + RequestsSent, + NiceRate(RequestsSent, ElapsedMS, "req"), + NiceBytes(Sent), + NiceByteRate(Sent, ElapsedMS), + NiceBytes(Received), + NiceByteRate(Received, ElapsedMS), + NiceTimeSpanMs(ElapsedMS), + NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs())); +} + } // namespace zen diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h index 4f5b90f4d..a2834f73d 100644 --- a/src/zen/cmds/cache_cmd.h +++ b/src/zen/cmds/cache_cmd.h @@ -4,131 +4,319 @@ #include "../zen.h" +#include <filesystem> + namespace zen { -class DropCommand : public CacheStoreCommand +// Base for `cache` subcommands. Registers the shared --hosturl option and +// exposes ResolveHost() which subcommands must call before issuing HTTP +// requests (it normalises m_HostName and throws if no host could be resolved). +class CacheSubCmdBase : public ZenSubCmdBase { public: - static constexpr char Name[] = "drop"; - static constexpr char Description[] = "Drop cache namespace or bucket"; + CacheSubCmdBase(std::string_view Name, std::string_view Description); + +protected: + void ResolveHost(); + + std::string m_HostName; +}; + +class CacheDetailsSubCmd : public CacheSubCmdBase +{ +public: + CacheDetailsSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + bool m_CSV = false; + bool m_YAML = false; + bool m_Details = false; + bool m_AttachmentDetails = false; + std::string m_Namespace; + std::string m_Bucket; + std::string m_ValueKey; +}; + +class CacheDropSubCmd : public CacheSubCmdBase +{ +public: + CacheDropSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_NamespaceName; + std::string m_BucketName; +}; + +class CacheGenSubCmd : public CacheSubCmdBase +{ +public: + CacheGenSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_Namespace; + std::string m_Bucket; + uint64_t m_Count = 1; + uint64_t m_MinSize = 0; + uint64_t m_MaxSize = 0; + uint32_t m_MinAttachmentCount = 0; + uint32_t m_MaxAttachmentCount = 0; +}; + +class CacheGetSubCmd : public CacheSubCmdBase +{ +public: + CacheGetSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_Namespace; + std::string m_Bucket; + std::string m_ValueKey; + std::string m_AttachmentHash; + std::filesystem::path m_OutputPath; + bool m_AsText = false; + bool m_Decompress = true; +}; + +class CacheInfoSubCmd : public CacheSubCmdBase +{ +public: + CacheInfoSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_NamespaceName; + std::string m_SizeInfoBucketNames; + bool m_BucketSizeInfo = false; + bool m_YAML = false; + std::string m_BucketName; +}; - DropCommand(); - ~DropCommand(); +class CacheRecordSubCmd : public CacheSubCmdBase +{ +public: + CacheRecordSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_Path; +}; + +class CacheReplaySubCmd : public CacheSubCmdBase +{ +public: + CacheReplaySubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } +private: + std::string m_RecordingPath; + bool m_OnHost = false; + bool m_ShowMethodStats = false; + int m_ProcessCount = 1; + int m_ThreadCount = 0; + uint64_t m_Offset = 0; + uint64_t m_Stride = 1; + bool m_ForceAllowLocalRefs = false; + bool m_DisableLocalRefs = false; + bool m_ForceAllowLocalHandleRef = false; + bool m_DisableLocalHandleRefs = false; + bool m_ForceAllowPartialLocalRefs = false; + bool m_DisablePartialLocalRefs = false; + bool m_DryRun = false; +}; + +class CacheStatsSubCmd : public CacheSubCmdBase +{ +public: + CacheStatsSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + bool m_YAML = false; +}; + +class CacheCommand : public CacheStoreCmdWithSubCommands +{ +public: + static constexpr char Name[] = "cache"; + static constexpr char Description[] = "Manage cache - info, stats, details, get, gen, drop, record, replay"; + + CacheCommand(); + ~CacheCommand(); + + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + + CacheDetailsSubCmd m_DetailsSubCmd; + CacheDropSubCmd m_DropSubCmd; + CacheGenSubCmd m_GenSubCmd; + CacheGetSubCmd m_GetSubCmd; + CacheInfoSubCmd m_InfoSubCmd; + CacheRecordSubCmd m_RecordSubCmd; + CacheReplaySubCmd m_ReplaySubCmd; + CacheStatsSubCmd m_StatsSubCmd; +}; + +// --------------------------------------------------------------------------- +// Deprecated legacy top-level commands. These forward to the corresponding +// 'cache <sub>' subcommand so that existing scripts keep working. They are +// hidden from the top-level `zen --help` listing; `zen cache --help` is the +// canonical discovery surface now. + +namespace cache_legacy_shim { + void RunAs(const char* SubCommandName, const ZenCliOptions& GlobalOptions, int argc, char** argv); +} + +class DeprecatedCacheStoreCommand : public CacheStoreCommand +{ +public: + bool IsHidden() const override { return true; } +}; + +class DropCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "drop"; + static constexpr char Description[] = "(deprecated, use 'cache drop') Drop cache namespace or bucket"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("drop", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_NamespaceName; - std::string m_BucketName; }; -class CacheInfoCommand : public CacheStoreCommand +class CacheInfoCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-info"; - static constexpr char Description[] = "Info on cache, namespace or bucket"; + static constexpr char Description[] = "(deprecated, use 'cache info') Info on cache, namespace or bucket"; - CacheInfoCommand(); - ~CacheInfoCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("info", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_NamespaceName; - std::string m_SizeInfoBucketNames; - bool m_BucketSizeInfo = false; - std::string m_BucketName; }; -class CacheStatsCommand : public CacheStoreCommand +class CacheStatsCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-stats"; - static constexpr char Description[] = "Stats on cache"; + static constexpr char Description[] = "(deprecated, use 'cache stats') Stats on cache"; - CacheStatsCommand(); - ~CacheStatsCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("stats", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; }; -class CacheDetailsCommand : public CacheStoreCommand +class CacheDetailsCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-details"; - static constexpr char Description[] = "Details on cache"; + static constexpr char Description[] = "(deprecated, use 'cache details') Details on cache"; - CacheDetailsCommand(); - ~CacheDetailsCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("details", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - bool m_CSV = false; - bool m_Details = false; - bool m_AttachmentDetails = false; - std::string m_Namespace; - std::string m_Bucket; - std::string m_ValueKey; }; -class CacheGenerateCommand : public CacheStoreCommand +class CacheGenerateCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-gen"; - static constexpr char Description[] = "Generates cache values into a bucket"; + static constexpr char Description[] = "(deprecated, use 'cache gen') Generates cache values into a bucket"; - CacheGenerateCommand(); - ~CacheGenerateCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("gen", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_Namespace; - std::string m_Bucket; - uint64_t m_Count = 1; - - uint64_t m_MinSize = 0; - uint64_t m_MaxSize = 0; - uint32_t m_MinAttachmentCount = 0; - uint32_t m_MaxAttachmentCount = 0; }; -class CacheGetCommand : public CacheStoreCommand +class CacheGetCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-get"; - static constexpr char Description[] = "Get cache values/records or attachments from a bucket"; + static constexpr char Description[] = "(deprecated, use 'cache get') Get cache values/records or attachments from a bucket"; - CacheGetCommand(); - ~CacheGetCommand(); + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("get", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } +private: + cxxopts::Options m_Options{Name, Description}; +}; + +class RpcStartRecordingCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "rpc-record-start"; + static constexpr char Description[] = "(deprecated, use 'cache record <path>') Starts recording of cache rpc requests on a host"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("record", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_Namespace; - std::string m_Bucket; - std::string m_ValueKey; - std::string m_AttachmentHash; - std::filesystem::path m_OutputPath; - bool m_AsText = false; - bool m_Decompress = true; + cxxopts::Options m_Options{Name, Description}; +}; + +class RpcStopRecordingCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "rpc-record-stop"; + static constexpr char Description[] = "(deprecated, use 'cache record stop') Stops recording of cache rpc requests on a host"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; +}; + +class RpcReplayCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "rpc-record-replay"; + static constexpr char Description[] = "(deprecated, use 'cache replay') Replays a previously recorded session of rpc requests"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("replay", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; }; } // namespace zen diff --git a/src/zen/cmds/compute_cmd.cpp b/src/zen/cmds/compute_cmd.cpp new file mode 100644 index 000000000..9a350c69c --- /dev/null +++ b/src/zen/cmds/compute_cmd.cpp @@ -0,0 +1,88 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "compute_cmd.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "zenserviceclient.h" + +# include <zencore/compactbinary.h> +# include <zencore/logging.h> +# include <zenhttp/httpclient.h> + +using namespace std::literals; + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +// ComputeRecordStartSubCmd + +ComputeRecordStartSubCmd::ComputeRecordStartSubCmd() : ZenSubCmdBase("record-start", "Start recording compute actions") +{ + SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} + +void +ComputeRecordStartSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record-start"}); + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = Http.Post("/compute/record/start"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) + { + CbObject Obj = Response.AsObject(); + std::string_view Path = Obj["path"sv].AsString(); + ZEN_CONSOLE("recording started: " ZEN_BRIGHT_GREEN("{}"), Path); + } + else + { + Response.ThrowError("Failed to start recording"); + } +} + +////////////////////////////////////////////////////////////////////////// +// ComputeRecordStopSubCmd + +ComputeRecordStopSubCmd::ComputeRecordStopSubCmd() : ZenSubCmdBase("record-stop", "Stop recording compute actions") +{ + SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} + +void +ComputeRecordStopSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record-stop"}); + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = Http.Post("/compute/record/stop"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) + { + CbObject Obj = Response.AsObject(); + std::string_view Path = Obj["path"sv].AsString(); + ZEN_CONSOLE("recording stopped: " ZEN_BRIGHT_GREEN("{}"), Path); + } + else + { + Response.ThrowError("Failed to stop recording"); + } +} + +////////////////////////////////////////////////////////////////////////// +// ComputeCommand + +ComputeCommand::ComputeCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); + m_Options.parse_positional({"subcommand"}); + + AddSubCommand(m_RecordStartSubCmd); + AddSubCommand(m_RecordStopSubCmd); +} + +ComputeCommand::~ComputeCommand() = default; + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/cmds/compute_cmd.h b/src/zen/cmds/compute_cmd.h new file mode 100644 index 000000000..b26f639c4 --- /dev/null +++ b/src/zen/cmds/compute_cmd.h @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +#include <string> + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen { + +class ComputeRecordStartSubCmd : public ZenSubCmdBase +{ +public: + ComputeRecordStartSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_HostName; +}; + +class ComputeRecordStopSubCmd : public ZenSubCmdBase +{ +public: + ComputeRecordStopSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_HostName; +}; + +class ComputeCommand : public ZenCmdWithSubCommands +{ +public: + static constexpr char Name[] = "compute"; + static constexpr char Description[] = "Compute service operations"; + + ComputeCommand(); + ~ComputeCommand(); + + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + ComputeRecordStartSubCmd m_RecordStartSubCmd; + ComputeRecordStopSubCmd m_RecordStopSubCmd; +}; + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp deleted file mode 100644 index 530661607..000000000 --- a/src/zen/cmds/copy_cmd.cpp +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "copy_cmd.h" - -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/string.h> -#include <zencore/timer.h> - -namespace zen { - -CopyCommand::CopyCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_options()("no-clone", "Do not perform block clone", cxxopts::value(m_NoClone)->default_value("false")); - m_Options.add_options()("must-clone", - "Always perform block clone (fails if clone is not possible)", - cxxopts::value(m_MustClone)->default_value("false")); - m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_CopySource), "<file/directory>"); - m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_CopyTarget), "<file/directory>"); - m_Options.parse_positional({"source", "target"}); -} - -CopyCommand::~CopyCommand() = default; - -void -CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - // Validate arguments - - if (m_CopySource.empty()) - throw OptionParseException("'--source' is required", m_Options.help()); - - if (m_CopyTarget.empty()) - throw OptionParseException("'--target' is required", m_Options.help()); - - std::filesystem::path FromPath = m_CopySource; - std::filesystem::path ToPath = m_CopyTarget; - - std::error_code Ec; - std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec); - - if (!Ec) - { - std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); - - if (!Ec) - { - if (FromCanonical == ToCanonical) - { - throw std::runtime_error("Target and source must be distinct files or directories"); - } - } - } - - const bool IsFileCopy = IsFile(m_CopySource); - const bool IsDirCopy = IsDir(m_CopySource); - - if (!IsFileCopy && !IsDirCopy) - { - throw std::runtime_error("Invalid source specification (neither directory nor file)"); - } - - if (IsFileCopy && IsDirCopy) - { - throw std::runtime_error("Invalid source specification (both directory AND file!?)"); - } - - if (IsDirCopy) - { - if (IsFile(ToPath)) - { - throw std::runtime_error("Attempted copy of directory into file"); - } - - if (!IsDir(ToPath)) - { - CreateDirectories(ToPath); - } - - std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); - - if (!Ec) - { - if (ToCanonical.generic_string().starts_with(FromCanonical.generic_string()) || - FromCanonical.generic_string().starts_with(ToCanonical.generic_string())) - { - throw std::runtime_error("Invalid parent/child relationship for source/target directories"); - } - } - - // Multi file copy - - ZEN_CONSOLE("copying {} -> {}", FromPath, ToPath); - - zen::Stopwatch Timer; - - struct CopyVisitor : public FileSystemTraversal::TreeVisitor - { - CopyVisitor(std::filesystem::path InBasePath, zen::CopyFileOptions InCopyOptions) - : BasePath(InBasePath) - , CopyOptions(InCopyOptions) - { - } - - virtual void VisitFile(const std::filesystem::path& Parent, - const path_view& File, - uint64_t FileSize, - uint32_t, - uint64_t) override - { - ZEN_UNUSED(FileSize); - std::error_code Ec; - const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec); - - if (Ec) - { - FailedFileCount++; - } - else - { - const std::filesystem::path FromPath = Parent / File; - const std::filesystem::path ToPath = TargetPath / Relative / File; - - try - { - zen::CreateDirectories(TargetPath / Relative); - if (zen::CopyFile(FromPath, ToPath, CopyOptions)) - { - ++FileCount; - ByteCount += FileSize; - } - else - { - throw std::logic_error("CopyFile failed in an unexpected way"); - } - } - catch (const std::exception& Ex) - { - ++FailedFileCount; - - ZEN_CONSOLE_ERROR("Failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); - } - } - } - - virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; } - - std::filesystem::path BasePath; - std::filesystem::path TargetPath; - zen::CopyFileOptions CopyOptions; - int FileCount = 0; - uint64_t ByteCount = 0; - int FailedFileCount = 0; - }; - - zen::CopyFileOptions CopyOptions; - CopyOptions.EnableClone = !m_NoClone; - CopyOptions.MustClone = m_MustClone; - - CopyVisitor Visitor{FromPath, CopyOptions}; - Visitor.TargetPath = ToPath; - - FileSystemTraversal Traversal; - Traversal.TraverseFileSystem(FromPath, Visitor); - - ZEN_CONSOLE("Copy of {} files ({}) completed in {} ({})", - Visitor.FileCount, - NiceBytes(Visitor.ByteCount), - zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - zen::NiceRate(Visitor.ByteCount, (uint32_t)Timer.GetElapsedTimeMs())); - - if (Visitor.FailedFileCount) - { - throw std::runtime_error(fmt::format("{} file copy operations FAILED", Visitor.FailedFileCount)); - } - } - else - { - // Single file copy - - zen::Stopwatch Timer; - - zen::CopyFileOptions CopyOptions; - CopyOptions.EnableClone = !m_NoClone; - - zen::CreateDirectories(ToPath.parent_path()); - if (zen::CopyFile(FromPath, ToPath, CopyOptions)) - { - ZEN_CONSOLE("Copy completed in {}", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - else - { - throw std::runtime_error(fmt::format("Failed to copy '{}' to '{}'", FromPath, ToPath)); - } - } -} - -} // namespace zen diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h deleted file mode 100644 index 757a8e691..000000000 --- a/src/zen/cmds/copy_cmd.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** Copy files, possibly using block cloning - */ -class CopyCommand : public ZenCmdBase -{ -public: - static constexpr char Name[] = "copy"; - static constexpr char Description[] = "Copy file(s)"; - - CopyCommand(); - ~CopyCommand(); - - virtual cxxopts::Options& Options() override { return m_Options; } - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::filesystem::path m_CopySource; - std::filesystem::path m_CopyTarget; - bool m_NoClone = false; - bool m_MustClone = false; -}; - -} // namespace zen diff --git a/src/zen/cmds/dedup_cmd.cpp b/src/zen/cmds/dedup_cmd.cpp index 9ef50a97d..18ad56aec 100644 --- a/src/zen/cmds/dedup_cmd.cpp +++ b/src/zen/cmds/dedup_cmd.cpp @@ -240,7 +240,12 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { zen::BLAKE3Stream b3s; - zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); }); + if (std::error_code ScanEc = + zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); }); + ScanEc) + { + throw std::system_error(ScanEc, fmt::format("Failed to scan file '{}'", Entry.path())); + } Hash = b3s.GetHash(); } @@ -279,7 +284,11 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Options.EnableClone = true; Options.MustClone = true; - zen::CopyFile(Dupe->path(), Entry.path(), Options); + if (std::error_code Ec = zen::CopyFile(Dupe->path(), Entry.path(), Options); Ec) + { + ZEN_ERROR("Failed to clone '{}' to '{}': {}", Dupe->path(), Entry.path(), Ec.message()); + continue; + } DupeBytes += Entry.file_size(); } diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp index 9719fce77..60968b521 100644 --- a/src/zen/cmds/exec_cmd.cpp +++ b/src/zen/cmds/exec_cmd.cpp @@ -2,6 +2,8 @@ #include "exec_cmd.h" +#include "zenserviceclient.h" + #include <zencompute/computeservice.h> #include <zencompute/recordingreader.h> #include <zencore/compactbinary.h> @@ -23,6 +25,8 @@ #include <zenhttp/httpclient.h> #include <zenhttp/packageformat.h> +#include "consoleprogress.h" + #include <EASTL/hash_map.h> #include <EASTL/hash_set.h> #include <EASTL/map.h> @@ -114,7 +118,7 @@ namespace { } // namespace ////////////////////////////////////////////////////////////////////////// -// ExecSessionConfig — read-only configuration for a session run +// ExecSessionConfig - read-only configuration for a session run struct ExecSessionConfig { @@ -124,17 +128,18 @@ struct ExecSessionConfig std::vector<ExecFunctionDefinition>& FunctionList; // mutable for EmitFunctionListOnce std::string_view OrchestratorUrl; const std::filesystem::path& OutputPath; - int Offset = 0; - int Stride = 1; - int Limit = 0; - bool Verbose = false; - bool Quiet = false; - bool DumpActions = false; - bool Binary = false; + int Offset = 0; + int Stride = 1; + int Limit = 0; + bool Verbose = false; + bool Quiet = false; + bool DumpActions = false; + bool Binary = false; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; }; ////////////////////////////////////////////////////////////////////////// -// ExecSessionRunner — owns per-run state, drives the session lifecycle +// ExecSessionRunner - owns per-run state, drives the session lifecycle class ExecSessionRunner { @@ -345,8 +350,6 @@ ExecSessionRunner::DrainCompletedJobs() } m_PendingJobs.Remove(CompleteLsn); - - ZEN_CONSOLE("completed: LSN {} ({} still pending)", CompleteLsn, m_PendingJobs.GetSize()); } } } @@ -897,17 +900,22 @@ ExecSessionRunner::Run() // Then submit work items - int FailedWorkCounter = 0; - size_t RemainingWorkItems = m_Config.RecordingReader.GetActionCount(); - int SubmittedWorkItems = 0; + std::atomic<int> FailedWorkCounter{0}; + std::atomic<size_t> RemainingWorkItems{m_Config.RecordingReader.GetActionCount()}; + std::atomic<int> SubmittedWorkItems{0}; + size_t TotalWorkItems = RemainingWorkItems.load(); - ZEN_CONSOLE("submitting {} work items", RemainingWorkItems); + std::unique_ptr<ProgressBase> ProgressOwner(CreateConsoleProgress(m_Config.ProgressMode)); + std::unique_ptr<ProgressBase::ProgressBar> SubmitProgress = ProgressOwner->CreateProgressBar("Submit"); + SubmitProgress->UpdateState( + {.Task = "Submitting work items", .TotalCount = TotalWorkItems, .RemainingCount = RemainingWorkItems.load()}, + false); int OffsetCounter = m_Config.Offset; int StrideCounter = m_Config.Stride; auto ShouldSchedule = [&]() -> bool { - if (m_Config.Limit && SubmittedWorkItems >= m_Config.Limit) + if (m_Config.Limit && SubmittedWorkItems.load() >= m_Config.Limit) { // Limit reached, ignore @@ -1005,17 +1013,14 @@ ExecSessionRunner::Run() { const int32_t LsnField = EnqueueResult.Lsn; - --RemainingWorkItems; - ++SubmittedWorkItems; + size_t Remaining = --RemainingWorkItems; + int Submitted = ++SubmittedWorkItems; - if (!m_Config.Quiet) - { - ZEN_CONSOLE("submitted work item #{} - LSN {} - {}. {} remaining", - SubmittedWorkItems, - LsnField, - NiceTimeSpanMs(SubmitTimer.GetElapsedTimeMs()), - RemainingWorkItems); - } + SubmitProgress->UpdateState({.Task = "Submitting work items", + .Details = fmt::format("#{} LSN {}", Submitted, LsnField), + .TotalCount = TotalWorkItems, + .RemainingCount = Remaining}, + false); if (!m_Config.OutputPath.empty()) { @@ -1055,22 +1060,37 @@ ExecSessionRunner::Run() }, TargetParallelism); + SubmitProgress->Finish(); + // Wait until all pending work is complete + size_t TotalPendingJobs = m_PendingJobs.GetSize(); + + std::unique_ptr<ProgressBase::ProgressBar> CompletionProgress = ProgressOwner->CreateProgressBar("Execute"); + while (!m_PendingJobs.IsEmpty()) { - // TODO: improve this logic - zen::Sleep(500); + size_t PendingCount = m_PendingJobs.GetSize(); + CompletionProgress->UpdateState( + {.Task = "Executing work items", + .Details = fmt::format("{} completed, {} remaining", TotalPendingJobs - PendingCount, PendingCount), + .TotalCount = TotalPendingJobs, + .RemainingCount = PendingCount}, + false); + + zen::Sleep(ProgressOwner->GetProgressUpdateDelayMS()); DrainCompletedJobs(); SendOrchestratorHeartbeat(); } + CompletionProgress->Finish(); + // Write summary files WriteSummaryFiles(); - if (FailedWorkCounter) + if (FailedWorkCounter.load()) { return 1; } @@ -1089,7 +1109,7 @@ ExecHttpSubCmd::ExecHttpSubCmd(ExecCommand& Parent) : ZenSubCmdBase("http", "For void ExecHttpSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = ExecCommand::Name}); ZEN_ASSERT(m_Parent.m_ChunkResolver); ChunkResolver& Resolver = *m_Parent.m_ChunkResolver; @@ -1097,7 +1117,7 @@ ExecHttpSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); zen::compute::ComputeServiceSession ComputeSession(Resolver); - ComputeSession.AddRemoteRunner(Resolver, TempPath, m_HostName); + ComputeSession.AddRemoteRunner(Resolver, TempPath, Service.HostSpec()); Stopwatch ExecTimer; int ReturnValue = m_Parent.RunSession(ComputeSession); @@ -1423,6 +1443,12 @@ ExecCommand::OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) int ExecCommand::RunSession(zen::compute::ComputeServiceSession& ComputeSession, std::string_view OrchestratorUrl) { + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + if (m_QuietLogging) + { + ProgressMode = ConsoleProgressMode::Quiet; + } + ExecSessionConfig Config{ .Resolver = *m_ChunkResolver, .RecordingReader = *m_RecordingReader, @@ -1437,6 +1463,7 @@ ExecCommand::RunSession(zen::compute::ComputeServiceSession& ComputeSession, std .Quiet = m_QuietLogging, .DumpActions = m_DumpActions, .Binary = m_Binary, + .ProgressMode = ProgressMode, }; ExecSessionRunner Runner(ComputeSession, Config); diff --git a/src/zen/cmds/history_cmd.cpp b/src/zen/cmds/history_cmd.cpp new file mode 100644 index 000000000..27faae1eb --- /dev/null +++ b/src/zen/cmds/history_cmd.cpp @@ -0,0 +1,228 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "history_cmd.h" + +#include <zenbase/zenbase.h> +#include <zencore/except_fmt.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/process.h> +#include <zenutil/consoletui.h> +#include <zenutil/invocationhistory.h> + +#include <algorithm> + +namespace zen { + +HistoryCommand::HistoryCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_options()("filter", + "Filter by executable ('zen' or 'zenserver' / 'server')", + cxxopts::value<std::string>(m_Filter)->default_value("")); + m_Options.add_options()("print", "Print the selected command line instead of running it", cxxopts::value<bool>(m_Print)); + m_Options.add_options()("l,list", "List all invocations to stdout instead of showing the picker", cxxopts::value<bool>(m_List)); +} + +HistoryCommand::~HistoryCommand() +{ +} + +namespace { + + bool KeepRecord(const HistoryRecord& Rec, std::string_view Filter) + { + if (Filter.empty()) + { + return true; + } + if (Filter == "zen") + { + return Rec.Exe == "zen"; + } + if (Filter == "zenserver" || Filter == "server") + { + return Rec.Exe == "zenserver"; + } + return true; + } + + std::string BuildLabel(const HistoryRecord& Rec, int32_t TerminalCols) + { + constexpr int32_t kIndicator = 3; // " > " or " " prefix from TuiPickOne + constexpr int32_t kEllipsis = 3; // "..." + + std::string Exe = Rec.Exe.empty() ? std::string("?") : Rec.Exe; + std::string Lbl = fmt::format("{} {:<9} pid {:<7}", Rec.Ts, Exe, Rec.Pid); + + if (!Rec.CmdLine.empty()) + { + int32_t Available = TerminalCols - kIndicator - 2 - static_cast<int32_t>(Lbl.size()); + if (Available > kEllipsis) + { + Lbl += " "; + if (static_cast<int32_t>(Rec.CmdLine.size()) <= Available) + { + Lbl += Rec.CmdLine; + } + else + { + Lbl.append(Rec.CmdLine, 0, static_cast<size_t>(Available - kEllipsis)); + Lbl += "..."; + } + } + } + return Lbl; + } + + int SpawnZen(const HistoryRecord& Rec) + { + const std::filesystem::path TargetPath = GetRunningExecutablePath(); + ZEN_CONSOLE("Running: {}", Rec.CmdLine); + + CreateProcOptions Opts{}; + CreateProcResult Result = CreateProc(TargetPath, Rec.CmdLine, Opts); + if (!Result) + { + throw zen::runtime_error("failed to launch '{}'", TargetPath); + } + + ProcessHandle Proc; + Proc.Initialize(Result); + return Proc.WaitExitCode(); + } + + void SpawnZenServerDetached(const HistoryRecord& Rec) + { + const std::filesystem::path TargetPath = GetRunningExecutablePath().parent_path() / ("zenserver" ZEN_EXE_SUFFIX_LITERAL); + + ZEN_CONSOLE("Launching detached: {}", Rec.CmdLine); + + CreateProcOptions Opts{}; + Opts.Flags = CreateProcOptions::Flag_NoConsole | CreateProcOptions::Flag_NewProcessGroup; + CreateProcResult Result = CreateProc(TargetPath, Rec.CmdLine, Opts); + if (!Result) + { + throw zen::runtime_error("failed to launch '{}'", TargetPath); + } + +#if ZEN_PLATFORM_WINDOWS + // Take ownership of the handle just to release it; we do not wait. + ProcessHandle Proc; + Proc.Initialize(Result); + ZEN_CONSOLE("Launched {} (pid {})", TargetPath, Proc.Pid()); +#else + ZEN_CONSOLE("Launched {} (pid {})", TargetPath, static_cast<int>(Result)); +#endif + } + + void PrintPlainTable(const std::vector<HistoryRecord>& Records) + { + if (Records.empty()) + { + ZEN_CONSOLE("No invocation history available."); + return; + } + for (const HistoryRecord& Rec : Records) + { + std::string Line = fmt::format("{} {:<9} pid {:<7}", Rec.Ts, Rec.Exe, Rec.Pid); + if (!Rec.CmdLine.empty()) + { + Line += " "; + Line += Rec.CmdLine; + } + ZEN_CONSOLE("{}", Line); + } + } + +} // namespace + +void +HistoryCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return; + } + + if (m_List && m_Print) + { + throw OptionParseException("'--list' conflicts with '--print'", m_Options.help()); + } + + std::vector<HistoryRecord> All = ReadInvocationHistory(); + + std::vector<HistoryRecord> Filtered; + Filtered.reserve(All.size()); + for (HistoryRecord& Rec : All) + { + if (KeepRecord(Rec, m_Filter)) + { + Filtered.push_back(std::move(Rec)); + } + } + + std::reverse(Filtered.begin(), Filtered.end()); + + if (Filtered.empty()) + { + ZEN_CONSOLE("No invocation history available{}.", m_Filter.empty() ? "" : " for this filter"); + return; + } + + if (m_List || !IsTuiAvailable()) + { + PrintPlainTable(Filtered); + return; + } + + const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns()); + std::vector<std::string> Labels; + Labels.reserve(Filtered.size()); + for (const HistoryRecord& Rec : Filtered) + { + Labels.push_back(BuildLabel(Rec, Cols)); + } + + int Selected = TuiPickOne("Recent invocations. Select one to re-run:", Labels); + if (Selected < 0) + { + return; + } + + const HistoryRecord& Pick = Filtered[Selected]; + + if (m_Print) + { + ZEN_CONSOLE("{}", Pick.CmdLine); + return; + } + + if (Pick.CmdLine.empty()) + { + throw zen::runtime_error("selected record has no command line"); + } + + if (Pick.Exe == "zenserver") + { + SpawnZenServerDetached(Pick); + return; + } + + if (Pick.Exe == "zen") + { + int ExitCode = SpawnZen(Pick); + if (ExitCode != 0) + { + throw ErrorWithReturnCode(fmt::format("zen exited with code {}", ExitCode), ExitCode); + } + return; + } + + throw zen::runtime_error("unknown executable '{}' in history record", Pick.Exe); +} + +} // namespace zen diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/history_cmd.h index 6eb0ba22b..d7dd2f078 100644 --- a/src/zen/cmds/trace_cmd.h +++ b/src/zen/cmds/history_cmd.h @@ -6,24 +6,23 @@ namespace zen { -class TraceCommand : public ZenCmdBase +class HistoryCommand : public ZenCmdBase { public: - static constexpr char Name[] = "trace"; - static constexpr char Description[] = "Control zen realtime tracing"; + HistoryCommand(); + ~HistoryCommand(); - TraceCommand(); - ~TraceCommand(); + static constexpr char Name[] = "history"; + static constexpr char Description[] = "List recent zen/zenserver invocations and optionally re-run one"; virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - bool m_Stop = false; - std::string m_TraceHost; - std::string m_TraceFile; + std::string m_Filter; + bool m_Print = false; + bool m_List = false; }; } // namespace zen diff --git a/src/zen/cmds/hub_cmd.cpp b/src/zen/cmds/hub_cmd.cpp index 5bdd3a922..75408a5e9 100644 --- a/src/zen/cmds/hub_cmd.cpp +++ b/src/zen/cmds/hub_cmd.cpp @@ -2,6 +2,8 @@ #include "hub_cmd.h" +#include "zenserviceclient.h" + #include <zencore/compactbinary.h> #include <zencore/compactbinaryutil.h> #include <zencore/filesystem.h> @@ -210,17 +212,13 @@ HubProvisionSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/provision", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -251,17 +249,13 @@ HubDeprovisionSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/deprovision", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -290,17 +284,13 @@ HubHibernateSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/hibernate", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -329,17 +319,13 @@ HubWakeSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/wake", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -369,13 +355,8 @@ HubStatusSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } - - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (!m_ModuleId.empty()) { diff --git a/src/zen/cmds/info_cmd.cpp b/src/zen/cmds/info_cmd.cpp index 9faad5691..b147b93c3 100644 --- a/src/zen/cmds/info_cmd.cpp +++ b/src/zen/cmds/info_cmd.cpp @@ -2,6 +2,8 @@ #include "info_cmd.h" +#include "zenserviceclient.h" + #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/string.h> @@ -31,14 +33,8 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/admin/info", HttpClient::Accept(ZenContentType::kJSON))) { @@ -46,7 +42,7 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - Result.ThrowError(fmt::format("Failed getting info from {}", m_HostName)); + Result.ThrowError(fmt::format("Failed getting info from {}", Service.HostSpec())); } } diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index d31c34fd0..c6a3434f8 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -2,16 +2,21 @@ #include "projectstore_cmd.h" +#include "zenserviceclient.h" + #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinaryutil.h> #include <zencore/compress.h> +#include <zencore/except_fmt.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/logging/broadcastsink.h> #include <zencore/parallelwork.h> #include <zencore/process.h> #include <zencore/scopeguard.h> +#include <zencore/session.h> #include <zencore/stream.h> #include <zencore/timer.h> #include <zencore/workthreadpool.h> @@ -21,16 +26,20 @@ #include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> #include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/buildstorageresolve.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/builds/jupiterbuildstorage.h> #include <zenremotestore/jupiter/jupiterhost.h> -#include <zenremotestore/operationlogoutput.h> #include <zenremotestore/projectstore/projectstoreoperations.h> #include <zenremotestore/projectstore/remoteprojectstore.h> #include <zenremotestore/transferthreadworkers.h> +#include <zenutil/authutils.h> +#include <zenutil/logging.h> +#include <zenutil/progress.h> +#include <zenutil/sessionsclient.h> #include <zenutil/workerpools.h> -#include "../progressbar.h" +#include "consoleprogress.h" ZEN_THIRD_PARTY_INCLUDES_START #include <json11.hpp> @@ -112,6 +121,26 @@ namespace projectstore_impl { } } + // `OplogMirrorCommand::Run` uses a latching boolean flag rather than the + // SignalCounter above, because it drives a worker pool that aborts on any + // interrupt. Kept separate from SignalCallbackHandler so neither interferes + // with the other when both are installed in the same process. + static std::atomic<bool> MirrorAbortFlag{false}; + + static void MirrorSignalCallbackHandler(int SigNum) + { + if (SigNum == SIGINT) + { + MirrorAbortFlag.store(true); + } +#if ZEN_PLATFORM_WINDOWS + if (SigNum == SIGBREAK) + { + MirrorAbortFlag.store(true); + } +#endif + } + void ExecuteAsyncOperation(HttpClient& Http, std::string_view Url, IoBuffer&& Payload, bool PlainProgress) { signal(SIGINT, SignalCallbackHandler); @@ -131,13 +160,16 @@ namespace projectstore_impl { throw std::runtime_error(fmt::format("invalid job id returned, received '{}'", JobIdText)); } - ProgressBar ProgressBar(PlainProgress ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty, ""sv); + std::unique_ptr<ProgressBase> ProgressOwner( + CreateConsoleProgress(PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty)); + std::unique_ptr<ProgressBase::ProgressBar> Bar = ProgressOwner->CreateProgressBar(""sv); + std::string ActiveTask; - auto OuputMessages = [&](CbObjectView StatusObject) { + auto OutputMessages = [&](CbObjectView StatusObject) { CbArrayView Messages = StatusObject["Messages"sv].AsArrayView(); if (Messages.Num() > 0) { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); for (auto M : Messages) { std::string_view Message = M.AsString(); @@ -169,33 +201,36 @@ namespace projectstore_impl { uint64_t RemainingCount = StatusObject["RemainingCount"sv].AsUInt64(); uint64_t ProgressElapsedTimeMs = StatusObject["ProgressElapsedTimeMs"sv].AsUInt64((uint64_t)-1); - if (!ProgressBar.IsSameTask(CurrentOp)) + if (ActiveTask != CurrentOp) { - ProgressBar.Finish(); + Bar->Finish(); + ActiveTask = ""; } - if (!ProgressBar.HasActiveTask()) + if (ActiveTask.empty()) { - OuputMessages(StatusObject); + OutputMessages(StatusObject); MessagesDone = true; + ActiveTask = std::string(CurrentOp); } - ProgressBar.UpdateState({.Task = std::string(CurrentOp), - .Details = std::string(CurrentOpDetails), - .TotalCount = TotalCount, - .RemainingCount = RemainingCount, - .OptionalElapsedTime = ProgressElapsedTimeMs}, - false); + Bar->UpdateState({.Task = std::string(CurrentOp), + .Details = std::string(CurrentOpDetails), + .TotalCount = TotalCount, + .RemainingCount = RemainingCount, + .OptionalElapsedTime = ProgressElapsedTimeMs}, + false); } if ((Status == "Complete") || (Status == "Aborted")) { - ProgressBar.Finish(); + Bar->Finish(); + ActiveTask = ""; } if (!MessagesDone) { - OuputMessages(StatusObject); + OutputMessages(StatusObject); } if (Status == "Complete") @@ -246,13 +281,13 @@ namespace projectstore_impl { #endif // ZEN_PLATFORM_WINDOWS if (HttpClient::Response DeleteResult = Http.Delete(fmt::format("/admin/jobs/{}", JobId))) { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); ZEN_CONSOLE("Requested cancel..."); Cancelled = true; } else { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); ZEN_CONSOLE("Failed cancelling job {}", DeleteResult); } continue; @@ -530,14 +565,8 @@ DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -569,7 +598,7 @@ DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_OplogName = ResolveOplog(Http, m_ProjectName, m_OplogName); if (m_OplogName.empty()) { - throw std::runtime_error(fmt::format("Can't find oplog in project '{}'", m_OplogName, m_ProjectName)); + throw zen::runtime_error("Can't find oplog '{}' in project '{}'", m_OplogName, m_ProjectName); } if (m_DryRun) { @@ -620,19 +649,13 @@ ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (!m_OplogName.empty() && m_ProjectName.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); std::string Url; if (m_ProjectName.empty()) @@ -709,19 +732,13 @@ CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectId.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); std::string Url = fmt::format("/prj/{}", m_ProjectId); @@ -779,20 +796,14 @@ CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectId.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ProjectId = ResolveProject(Http, m_ProjectId); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ProjectId = ResolveProject(Http, m_ProjectId); if (m_ProjectId.empty()) { throw std::runtime_error("Project can not be found"); @@ -1010,20 +1021,14 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_BoostWorkerMemory = true; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectName.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ProjectName = ResolveProject(Http, m_ProjectName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) { throw std::runtime_error("Project can not be found"); @@ -1114,7 +1119,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::string TargetUrlBase = m_ZenUrl; if (TargetUrlBase.find("://") == std::string::npos) { - // Assume https URL + // Assume http URL TargetUrlBase = fmt::format("http://{}", TargetUrlBase); } @@ -1303,7 +1308,14 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::filesystem::path MetadataPath(m_BuildsMetadataPath); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); - CbFieldIterator MetaData = LoadCompactBinaryFromJson(Json); + std::string JsonError; + CbFieldIterator MetaData = LoadCompactBinaryFromJson(Json, JsonError); + if (!JsonError.empty()) + { + throw zen::runtime_error("builds metadata file '{}' is malformed. Reason: '{}'", + MetadataPath.string(), + JsonError); + } Writer.AddBinary("metadata"sv, MetaData.GetBuffer()); } if (!m_BuildsMetadata.empty()) @@ -1313,7 +1325,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg size_t SplitPos = Pair.find('='); if (SplitPos == std::string::npos || SplitPos == 0) { - throw std::runtime_error(fmt::format("builds metadata key-value pair '{}' is malformed", Pair)); + throw zen::runtime_error("builds metadata key-value pair '{}' is malformed", Pair); } MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); return true; @@ -1323,7 +1335,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } } Writer.EndObject(); // "builds" - TargetDescription = fmt::format("[builds] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId); + TargetDescription = fmt::format("[builds] {}/{}/{}/{}", m_BuildsUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId); } if (!m_ZenUrl.empty()) { @@ -1517,13 +1529,6 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_BoostWorkerMemory = true; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectName.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); @@ -1541,8 +1546,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ProjectName = ResolveProject(Http, m_ProjectName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) { throw std::runtime_error("Project can not be found"); @@ -1738,7 +1744,11 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } } Writer.EndObject(); // "builds" - SourceDescription = fmt::format("[builds] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId); + SourceDescription = fmt::format("[builds] {}/{}/{}/{}", + m_BuildsHost.empty() ? m_BuildsOverrideHost : m_BuildsHost, + m_JupiterNamespace, + m_JupiterBucket, + m_BuildsId); } if (!m_ZenUrl.empty()) { @@ -1806,14 +1816,8 @@ SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (m_ProjectName.empty()) { @@ -1869,14 +1873,8 @@ ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/stats/prj", HttpClient::Accept(ZenContentType::kJSON))) { ZEN_CONSOLE("{}", Result.ToText()); @@ -1922,14 +1920,8 @@ ProjectOpDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char* return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -2038,14 +2030,8 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -2108,16 +2094,59 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::unordered_set<std::u8string> FileNames; std::atomic<uint64_t> WrittenByteCount = 0; - std::atomic<bool> AbortFlag(false); + // Install Ctrl-C handler so SIGINT aborts the worker pool rather than killing + // the process. Without this the local AbortFlag would shadow whatever global + // handler is installed elsewhere and interrupts would be dropped. RAII so + // the previous handler is restored when the function returns or throws. + MirrorAbortFlag.store(false); + ScopedSignalHandler SigIntGuard(SIGINT, MirrorSignalCallbackHandler); +#if ZEN_PLATFORM_WINDOWS + ScopedSignalHandler SigBreakGuard(SIGBREAK, MirrorSignalCallbackHandler); +#endif + std::atomic<bool>& AbortFlag = MirrorAbortFlag; Stopwatch WriteStopWatch; + // Filenames come from the remote oplog, which may be compromised or untrusted. + // Reject anything that could escape the mirror root via an absolute path, drive + // letter / UNC / device path prefix, or '..' component before it is joined to + // RootPath. Returns nullptr when the filename is safe. + auto UnsafeFileNameReason = [](const std::filesystem::path& FileName) -> const char* { + if (FileName.empty()) + { + return "filename is empty"; + } + if (FileName.has_root_name()) + { + return "filename has a root name (drive letter, UNC share, or device path)"; + } + if (FileName.has_root_directory()) + { + return "filename is absolute"; + } + for (const std::filesystem::path& Component : FileName) + { + const std::u8string C = Component.u8string(); + if (C.empty() || C == u8"..") + { + return "filename contains a '..' or empty component"; + } + } + return nullptr; + }; + auto EmitFilesForDataArray = [&](CbArrayView DataArray) { for (auto DataIter : DataArray) { if (CbObjectView Data = DataIter.AsObjectView()) { std::filesystem::path FileName(Data["filename"sv].AsU8String()); + if (const char* Reason = UnsafeFileNameReason(FileName)) + { + ZEN_CONSOLE_ERROR("Rejecting unsafe filename '{}' from remote oplog: {}", FileName.string(), Reason); + AbortFlag.store(true); + break; + } if (!m_FilenameFilter.empty()) { std::string FileNameLowerCase = ToLower(FileName.string()); @@ -2162,7 +2191,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg IoBuffer ChunkData = m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload; - if (!MoveToFile(TargetPath, ChunkData)) + if (std::error_code MoveEc = MoveToFile(TargetPath, ChunkData); MoveEc) { WriteFile(TargetPath, ChunkData); } @@ -2196,17 +2225,18 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ZEN_CONSOLE("Fetched oplog in {}", NiceTimeSpanMs(uint64_t(Response.ElapsedSeconds * 1000.0))); if (CbObject ResponseObject = Response.AsObject()) { - std::unique_ptr<ProgressBar> EmitProgressBar; + std::unique_ptr<ProgressBase> ProgressOwner2(CreateConsoleProgress(ConsoleProgressMode::Pretty)); + std::unique_ptr<ProgressBase::ProgressBar> EmitProgressBar; { - ProgressBar ParseProgressBar(ProgressBar::Mode::Pretty, ""); - CbArrayView Entries = ResponseObject["entries"sv].AsArrayView(); - uint64_t Remaining = Entries.Num(); + std::unique_ptr<ProgressBase::ProgressBar> ParseProgressBar = ProgressOwner2->CreateProgressBar(""); + CbArrayView Entries = ResponseObject["entries"sv].AsArrayView(); + uint64_t Remaining = Entries.Num(); for (auto EntryIter : Entries) { if (!AbortFlag) { CbObjectView Entry = EntryIter.AsObjectView(); - ParseProgressBar.UpdateState( + ParseProgressBar->UpdateState( {.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining}, false); Remaining--; @@ -2219,7 +2249,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } if (!EmitProgressBar) { - EmitProgressBar = std::make_unique<ProgressBar>(ProgressBar::Mode::Pretty, ""sv); + EmitProgressBar = ProgressOwner2->CreateProgressBar(""sv); WriteStopWatch.Reset(); } @@ -2229,7 +2259,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ++OplogEntryCount; } } - ParseProgressBar.Finish(); + ParseProgressBar->Finish(); } WorkRemaining.CountDown(); @@ -2262,7 +2292,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (AbortFlag) { - throw std::runtime_error("Failed top mirror oplog"); + throw std::runtime_error("Failed to mirror oplog"); } } else @@ -2306,14 +2336,8 @@ OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -2472,7 +2496,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a }; ParseSystemOptions(); - ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; auto ParseOutputOptions = [&]() { if (m_Verbose && m_Quiet) @@ -2494,23 +2518,19 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a if (m_LogProgress) { - ProgressMode = ProgressBar::Mode::Log; + ProgressMode = ConsoleProgressMode::Log; } else if (m_PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; - } - else if (m_Verbose) - { - ProgressMode = ProgressBar::Mode::Plain; + ProgressMode = ConsoleProgressMode::Plain; } else if (m_Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + ProgressMode = ConsoleProgressMode::Pretty; } }; ParseOutputOptions(); @@ -2565,7 +2585,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a m_BoostWorkerMemory = true; } - std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); TransferThreadWorkers Workers(m_BoostWorkerCount, /*SingleThreaded*/ false); if (!m_Quiet) @@ -2594,7 +2614,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a /*Hidden*/ false, m_Verbose); - BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*OperationLogOutput, + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(), ClientSettings, m_Host, m_OverrideHost, @@ -2629,6 +2649,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a .MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u}, [&AbortFlag]() { return AbortFlag.load(); }); Storage.CacheHost = ResolveRes.Cache; + + Storage.SetupCacheSession(ResolveRes.Cache.Address, Name, GetSessionId()); } if (!m_Quiet) @@ -2677,7 +2699,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } ProjectStoreOperationOplogState State( - *OperationLogOutput, + ConsoleLog(), Storage, BuildId, {.IsQuiet = m_Quiet, .IsVerbose = m_Verbose, .ForceDownload = m_ForceDownload, .TempFolderPath = StorageTempPath}); @@ -2704,7 +2726,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } std::atomic<bool> PauseFlag; - ProjectStoreOperationDownloadAttachments Op(*OperationLogOutput, + ProjectStoreOperationDownloadAttachments Op(ConsoleLog(), + *Progress, Storage, AbortFlag, PauseFlag, diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index 1ba98b39e..41db36139 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -217,7 +217,7 @@ class SnapshotOplogCommand : public ProjectStoreCommand { public: static constexpr char Name[] = "oplog-snapshot"; - static constexpr char Description[] = "Snapshot project store oplog"; + static constexpr char Description[] = "Copy oplog's loose files on disk into zenserver"; SnapshotOplogCommand(); ~SnapshotOplogCommand(); diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp deleted file mode 100644 index 3bf81a9df..000000000 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "rpcreplay_cmd.h" - -#include <zencore/compactbinarybuilder.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/process.h> -#include <zencore/scopeguard.h> -#include <zencore/session.h> -#include <zencore/stream.h> -#include <zencore/timer.h> -#include <zencore/workthreadpool.h> -#include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpcommon.h> -#include <zenhttp/packageformat.h> -#include <zenutil/rpcrecording.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <fmt/format.h> -#include <gsl/gsl-lite.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -#include <memory> - -namespace zen { - -using namespace std::literals; - -RpcStartRecordingCommand::RpcStartRecordingCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>"); - - m_Options.parse_positional("path"); -} - -RpcStartRecordingCommand::~RpcStartRecordingCommand() = default; - -void -RpcStartRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - if (m_RecordingPath.empty()) - { - throw OptionParseException("'--path' is required", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = - Http.Post("/z$/exec$/start-recording"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap({{"path", m_RecordingPath}}))) - { - ZEN_CONSOLE("{}", Response.ToText()); - } - else - { - Response.ThrowError("Failed to start recording"); - } -} - -//////////////////////////////////////////////////// - -RpcStopRecordingCommand::RpcStopRecordingCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); -} - -RpcStopRecordingCommand::~RpcStopRecordingCommand() = default; - -void -RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = Http.Post("/z$/exec$/stop-recording"sv)) - { - ZEN_CONSOLE("{}", Response.ToText()); - } - else - { - Response.ThrowError("Failed to stop recording"); - } -} - -//////////////////////////////////////////////////// - -RpcReplayCommand::RpcReplayCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>"); - m_Options.add_option("", "", "dry", "Do a dry run", cxxopts::value(m_DryRun), "<enable>"); - m_Options.add_option("", - "w", - "numthreads", - "Number of worker threads per process", - cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", GetHardwareConcurrency())), - "<count>"); - m_Options.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), "<onhost>"); - m_Options.add_option("", - "", - "showmethodstats", - "Show statistics of which RPC methods are used", - cxxopts::value(m_ShowMethodStats), - "<showmethodstats>"); - m_Options.add_option("", - "", - "offset", - "Offset into request recording to start replay", - cxxopts::value(m_Offset)->default_value("0"), - "<offset>"); - m_Options.add_option("", - "", - "stride", - "Stride for request recording when replaying requests", - cxxopts::value(m_Stride)->default_value("1"), - "<stride>"); - m_Options.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), "<count>"); - m_Options.add_option("", - "", - "forceallowlocalrefs", - "Force enable local refs in requests", - cxxopts::value(m_ForceAllowLocalRefs), - "<enable>"); - m_Options - .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), "<enable>"); - m_Options.add_option("", - "", - "forceallowlocalhandlerefs", - "Force enable local refs as handles in requests", - cxxopts::value(m_ForceAllowLocalHandleRef), - "<enable>"); - m_Options.add_option("", - "", - "disablelocalhandlerefs", - "Force disable local refs as handles in requests", - cxxopts::value(m_DisableLocalHandleRefs), - "<enable>"); - m_Options.add_option("", - "", - "forceallowpartiallocalrefs", - "Force enable local refs for all sizes", - cxxopts::value(m_ForceAllowPartialLocalRefs), - "<enable>"); - m_Options.add_option("", - "", - "disablepartiallocalrefs", - "Force disable local refs for all sizes", - cxxopts::value(m_DisablePartialLocalRefs), - "<enable>"); - - m_Options.parse_positional("path"); -} - -RpcReplayCommand::~RpcReplayCommand() = default; - -void -RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - if (m_RecordingPath.empty()) - { - throw OptionParseException("'--path' is required", m_Options.help()); - } - - if (!IsDir(m_RecordingPath)) - { - throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); - } - - m_ThreadCount = Max(m_ThreadCount, 1); - - ZEN_CONSOLE("Replay '{}' (start offset {}, stride {}) to '{}', {} threads", - m_RecordingPath, - m_Offset, - m_Stride, - m_HostName, - m_ThreadCount); - - Stopwatch TotalTimer; - - if (m_OnHost) - { - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = - Http.Post("/z$/exec$/replay-recording"sv, - HttpClient::KeyValueMap{}, - HttpClient::KeyValueMap({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}}))) - { - ZEN_CONSOLE("{}", Response.ToText()); - - return; - } - else - { - Response.ThrowError("Failed to start replay"); - } - } - - std::unique_ptr<cache::IRpcRequestReplayer> Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true); - uint64_t EntryCount = Replayer->GetRequestCount(); - - std::atomic_uint64_t EntryOffset = m_Offset; - std::atomic_uint64_t BytesSent = 0; - std::atomic_uint64_t BytesReceived = 0; - - Stopwatch Timer; - - if (m_ProcessCount > 1) - { - std::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses; - WorkerProcesses.resize(m_ProcessCount); - - ProcessMonitor Monitor; - for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex) - { - std::string CommandLine = - fmt::format("{} rpc-record-replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv, - argv[0], - m_HostName, - m_RecordingPath, - m_Stride == 1 ? 0 : m_Offset + ProcessIndex, - m_Stride, - m_ThreadCount, - 1); - CreateProcResult Result(CreateProc(std::filesystem::path(std::string(argv[0])), CommandLine)); - WorkerProcesses[ProcessIndex] = std::make_unique<ProcessHandle>(); - WorkerProcesses[ProcessIndex]->Initialize(Result); - Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid()); - } - while (Monitor.IsRunning()) - { - ZEN_CONSOLE("Waiting for worker processes..."); - Sleep(1000); - } - return; - } - else - { - std::map<std::string, size_t> MethodTypes; - RwLock MethodTypesLock; - - WorkerThreadPool WorkerPool(m_ThreadCount); - - Latch WorkLatch(m_ThreadCount); - for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex) - { - WorkerPool.ScheduleWork( - [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - - std::map<std::string, size_t> LocalMethodTypes; - - auto ReduceTypes = MakeGuard([&] { - RwLock::ExclusiveLockScope __(MethodTypesLock); - - for (auto& Entry : LocalMethodTypes) - { - MethodTypes[Entry.first] += Entry.second; - } - }); - - HttpClient Http = CreateHttpClient(m_HostName); - - uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride); - while (EntryIndex < EntryCount) - { - IoBuffer Payload; - const zen::cache::RecordedRequestInfo RequestInfo = Replayer->GetRequest(EntryIndex, /* out */ Payload); - - if (RequestInfo != zen::cache::RecordedRequestInfo::NullRequest) - { - CbPackage RequestPackage; - CbObject Request; - - switch (RequestInfo.ContentType) - { - case ZenContentType::kCbPackage: - { - if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage)) - { - Request = RequestPackage.GetObject(); - } - } - break; - case ZenContentType::kCbObject: - { - Request = LoadCompactBinaryObject(Payload); - } - break; - } - - RpcAcceptOptions OriginalAcceptOptions = static_cast<RpcAcceptOptions>(Request["AcceptFlags"sv].AsUInt16(0u)); - int OriginalProcessPid = Request["Pid"sv].AsInt32(0); - - int AdjustedPid = 0; - RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone; - - if (!m_DisableLocalRefs) - { - if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) || - m_ForceAllowLocalRefs) - { - AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences; - if (!m_DisablePartialLocalRefs) - { - if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) || - m_ForceAllowPartialLocalRefs) - { - AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences; - } - } - if (!m_DisableLocalHandleRefs) - { - if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef) - { - AdjustedPid = GetCurrentProcessId(); - } - } - } - } - - if (m_ShowMethodStats) - { - std::string MethodName = std::string(Request["Method"sv].AsString()); - if (auto It = LocalMethodTypes.find(MethodName); It != LocalMethodTypes.end()) - { - It->second++; - } - else - { - LocalMethodTypes[MethodName] = 1; - } - } - - if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid) - { - CbObjectWriter RequestCopyWriter; - for (const CbFieldView& Field : Request) - { - if (!Field.HasName()) - { - RequestCopyWriter.AddField(Field); - continue; - } - std::string_view FieldName = Field.GetName(); - if (FieldName == "Pid"sv) - { - continue; - } - if (FieldName == "AcceptFlags"sv) - { - continue; - } - RequestCopyWriter.AddField(FieldName, Field); - } - if (AdjustedPid != 0) - { - RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid); - } - if (AdjustedAcceptOptions != RpcAcceptOptions::kNone) - { - RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast<uint16_t>(AdjustedAcceptOptions)); - } - - if (RequestInfo.ContentType == ZenContentType::kCbPackage) - { - RequestPackage.SetObject(RequestCopyWriter.Save()); - std::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage); - std::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end()); - Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer(); - } - else - { - RequestCopyWriter.Finalize(); - Payload = IoBuffer(RequestCopyWriter.GetSaveSize()); - RequestCopyWriter.Save(Payload.GetMutableView()); - } - } - - if (!m_DryRun) - { - Http.SetSessionId(RequestInfo.SessionId); - Payload.SetContentType(RequestInfo.ContentType); - - HttpClient::Response Response = - Http.Post("/z$/$rpc", Payload, {HttpClient::Accept(RequestInfo.AcceptType)}); - - BytesSent.fetch_add(Payload.GetSize()); - if (!Response) - { - ZEN_CONSOLE_ERROR("{}", Response); - break; - } - BytesReceived.fetch_add(Response.DownloadedBytes); - } - } - - EntryIndex = EntryOffset.fetch_add(m_Stride); - } - }, - WorkerThreadPool::EMode::EnableBacklog); - } - - while (!WorkLatch.Wait(1000)) - { - const uint64_t RequestsTotal = (EntryCount - m_Offset) / m_Stride; - const uint64_t RequestsRemaining = (EntryCount - EntryOffset.load()) / m_Stride; - - ZEN_CONSOLE("[{:3}%] [{}] {} requests, {} remaining (sent {}, received {})", - (RequestsTotal - RequestsRemaining) * 100 / RequestsTotal, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - RequestsTotal, - RequestsRemaining, - NiceBytes(BytesSent.load()), - NiceBytes(BytesReceived.load())); - } - - if (m_ShowMethodStats) - { - for (const auto& It : MethodTypes) - { - ZEN_CONSOLE("{:18}: {:10}", It.first, It.second); - } - } - } - - const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride; - const uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); - const uint64_t Sent = BytesSent.load(); - const uint64_t Received = BytesReceived.load(); - - ZEN_CONSOLE("Processed requests: {} ({}), payloads sent {} ({}), payloads received {} ({}) in {}.\nTotal runtime: {}", - RequestsSent, - NiceRate(RequestsSent, ElapsedMS, "req"), - NiceBytes(Sent), - NiceByteRate(Sent, ElapsedMS), - NiceBytes(Received), - NiceByteRate(Received, ElapsedMS), - NiceTimeSpanMs(ElapsedMS), - NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs())); -} - -} // namespace zen diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h deleted file mode 100644 index 332a3126c..000000000 --- a/src/zen/cmds/rpcreplay_cmd.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class RpcStartRecordingCommand : public CacheStoreCommand -{ -public: - static constexpr char Name[] = "rpc-record-start"; - static constexpr char Description[] = "Starts recording of cache rpc requests on a host"; - - RpcStartRecordingCommand(); - ~RpcStartRecordingCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_RecordingPath; -}; - -class RpcStopRecordingCommand : public CacheStoreCommand -{ -public: - static constexpr char Name[] = "rpc-record-stop"; - static constexpr char Description[] = "Stops recording of cache rpc requests on a host"; - - RpcStopRecordingCommand(); - ~RpcStopRecordingCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; -}; - -class RpcReplayCommand : public CacheStoreCommand -{ -public: - static constexpr char Name[] = "rpc-record-replay"; - static constexpr char Description[] = "Replays a previously recorded session of rpc requests"; - - RpcReplayCommand(); - ~RpcReplayCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_RecordingPath; - bool m_OnHost = false; - bool m_ShowMethodStats = false; - int m_ProcessCount = 1; - int m_ThreadCount = 0; - uint64_t m_Offset = 0; - uint64_t m_Stride = 1; - bool m_ForceAllowLocalRefs = false; - bool m_DisableLocalRefs = false; - bool m_ForceAllowLocalHandleRef = false; - bool m_DisableLocalHandleRefs = false; - bool m_ForceAllowPartialLocalRefs = false; - bool m_DisablePartialLocalRefs = false; - bool m_DryRun = false; -}; - -} // namespace zen diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp deleted file mode 100644 index ee47eb9f3..000000000 --- a/src/zen/cmds/run_cmd.cpp +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "run_cmd.h" - -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/process.h> -#include <zencore/string.h> -#include <zencore/timer.h> - -using namespace std::literals; - -#define ZEN_COLOR_BLACK "\033[0;30m" -#define ZEN_COLOR_RED "\033[0;31m" -#define ZEN_COLOR_GREEN "\033[0;32m" -#define ZEN_COLOR_YELLOW "\033[0;33m" -#define ZEN_COLOR_BLUE "\033[0;34m" -#define ZEN_COLOR_MAGENTA "\033[0;35m" -#define ZEN_COLOR_CYAN "\033[0;36m" -#define ZEN_COLOR_WHITE "\033[0;37m" - -#define ZEN_BRIGHT_COLOR_BLACK "\033[1;30m" -#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" -#define ZEN_BRIGHT_COLOR_GREEN "\033[1;32m" -#define ZEN_BRIGHT_COLOR_YELLOW "\033[1;33m" -#define ZEN_BRIGHT_COLOR_BLUE "\033[1;34m" -#define ZEN_BRIGHT_COLOR_MAGENTA "\033[1;35m" -#define ZEN_BRIGHT_COLOR_CYAN "\033[1;36m" -#define ZEN_BRIGHT_COLOR_WHITE "\033[1;37m" - -#define ZEN_COLOR_RESET "\033[0m" - -namespace zen { - -RunCommand::RunCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "n", "count", "Number of times to run command", cxxopts::value(m_RunCount), "<count>"); - m_Options.add_option("", "t", "time", "How long to run command(s) for", cxxopts::value(m_RunTime), "<seconds>"); - m_Options.add_option("", - "", - "basepath", - "Where to run command. Each run will execute in its own numbered subdirectory below this directory. Additionally, " - "stdout will be redirected to a file in the provided directory", - cxxopts::value(m_BaseDirectory), - "<path>"); - m_Options.add_option("", - "", - "max-dirs", - "Number of base directories to retain on rotation", - cxxopts::value(m_MaxBaseDirectoryCount), - "<count>"); -} - -RunCommand::~RunCommand() -{ -} - -void -RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - if (!ParseOptions(argc, argv)) - { - return; - } - - // Validate arguments - - if (GlobalOptions.PassthroughArgV.empty() || GlobalOptions.PassthroughArgV[0].empty()) - throw OptionParseException("No command specified. The command to run is passed in after a double dash ('--') on the command line", - m_Options.help()); - - if (m_RunCount < 0) - throw OptionParseException(fmt::format("'--count' ('{}') is invalid", m_RunCount), m_Options.help()); - - if (m_RunTime < -1 || m_RunTime == 0) - throw OptionParseException(fmt::format("'--time' ('{}') is invalid", m_RunTime), m_Options.help()); - - if (m_MaxBaseDirectoryCount < 0) - throw OptionParseException(fmt::format("'--max-dirs' ('{}') is invalid", m_MaxBaseDirectoryCount), m_Options.help()); - - if (m_RunTime > 0 && m_RunCount > 0) - throw OptionParseException(fmt::format("'--time' ('{}') conflicts with '--count' ('{}') ", m_RunTime, m_RunCount), - m_Options.help()); - - if (m_RunCount == 0) - m_RunCount = 1; - - std::filesystem::path BaseDirectory; - - if (!m_BaseDirectory.empty()) - { - BaseDirectory = m_BaseDirectory; - - if (m_MaxBaseDirectoryCount) - { - RotateDirectories(BaseDirectory, m_MaxBaseDirectoryCount); - CreateDirectories(BaseDirectory); - } - else - { - CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false); - } - } - - bool TimedRun = false; - auto CommandDeadlineTime = std::chrono::system_clock::now(); - - if (m_RunTime > 0) - { - m_RunCount = 1'000'000; - TimedRun = true; - - CommandDeadlineTime += std::chrono::seconds(m_RunTime); - } - - struct RunResults - { - int ExitCode = 0; - std::chrono::duration<long, std::milli> Duration{}; - }; - - std::vector<RunResults> Results; - int ErrorCount = 0; - - std::filesystem::path ExecutablePath = SearchPathForExecutable(GlobalOptions.PassthroughArgV[0]); - std::string CommandArguments = GlobalOptions.PassthroughArgs; - - for (int i = 0; i < m_RunCount; ++i) - { - std::filesystem::path RunDir; - if (!BaseDirectory.empty()) - { - RunDir = BaseDirectory / IntNum(i + 1).c_str(); - CreateDirectories(RunDir); - } - - Stopwatch Timer; - - CreateProcOptions ProcOptions; - - if (!RunDir.empty()) - { - ProcOptions.WorkingDirectory = &RunDir; - ProcOptions.StdoutFile = RunDir / "stdout.txt"; - } - - fmt::print(ZEN_BRIGHT_COLOR_WHITE "run #{}" ZEN_COLOR_RESET ": {}\n", i + 1, GlobalOptions.PassthroughCommandLine); - - ProcessHandle Proc; - Proc.Initialize(CreateProc(ExecutablePath, GlobalOptions.PassthroughCommandLine, ProcOptions)); - if (!Proc.IsValid()) - { - throw std::runtime_error(fmt::format("failed to launch '{}'", ExecutablePath)); - } - - int ExitCode = Proc.WaitExitCode(); - - auto EndTime = std::chrono::system_clock::now(); - - if (ExitCode) - ++ErrorCount; - - Results.emplace_back(RunResults{.ExitCode = ExitCode, .Duration = std::chrono::milliseconds(Timer.GetElapsedTimeMs())}); - - if (TimedRun) - { - if (EndTime >= CommandDeadlineTime) - { - m_RunCount = i + 1; - break; - } - } - } - - fmt::print("{:>5} {:>3} {:>6}\n", "run", "rc", "time"); - int i = 0; - for (const auto& Entry : Results) - { - fmt::print("{:5} {:3} {:>6}\n", ++i, Entry.ExitCode, NiceTimeSpanMs(Entry.Duration.count())); - } - - if (ErrorCount) - { - fmt::print("run complete ({}/{} failed)\n", ErrorCount, m_RunCount); - } - else - { - fmt::print("run complete, no error exit code\n", m_RunCount); - } -} - -} // namespace zen diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h deleted file mode 100644 index 300c08c5b..000000000 --- a/src/zen/cmds/run_cmd.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class RunCommand : public ZenCmdBase -{ -public: - static constexpr char Name[] = "run"; - static constexpr char Description[] = "Run command with special options"; - - RunCommand(); - ~RunCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } - -private: - cxxopts::Options m_Options{Name, Description}; - int m_RunCount = 0; - int m_RunTime = -1; - std::string m_BaseDirectory; - int m_MaxBaseDirectoryCount = 10; -}; - -} // namespace zen diff --git a/src/zen/cmds/service_cmd.cpp b/src/zen/cmds/service_cmd.cpp index 37baf5483..c43c4e614 100644 --- a/src/zen/cmds/service_cmd.cpp +++ b/src/zen/cmds/service_cmd.cpp @@ -320,7 +320,7 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'verb' option is required", m_Options.help()); } - // Parse subcommand permissively — forward unrecognised options to the parent parser. + // Parse subcommand permissively - forward unrecognised options to the parent parser. std::vector<std::string> SubUnmatched; if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched)) { @@ -500,9 +500,9 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { std::filesystem::path Destination = m_InstallPath / File.filename(); - if (!CopyFile(File, Destination, {.EnableClone = false})) + if (std::error_code CopyEc = CopyFile(File, Destination, {.EnableClone = false}); CopyEc) { - throw std::runtime_error(fmt::format("Failed to copy '{}' to '{}'", File, Destination)); + throw std::system_error(CopyEc, fmt::format("Failed to copy '{}' to '{}'", File, Destination)); } ZEN_INFO("Copied '{}' to '{}'", File, Destination); diff --git a/src/zen/cmds/trace_cmd.cpp b/src/zen/cmds/trace_cmd.cpp deleted file mode 100644 index 54c0f080d..000000000 --- a/src/zen/cmds/trace_cmd.cpp +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "trace_cmd.h" -#include <zencore/logging.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpcommon.h> - -using namespace std::literals; - -namespace zen { - -TraceCommand::TraceCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "s", "stop", "Stop tracing", cxxopts::value(m_Stop)->default_value("false"), "<stop>"); - m_Options.add_option("", "", "host", "Start tracing to host", cxxopts::value(m_TraceHost), "<hostip>"); - m_Options.add_option("", "", "file", "Start tracing to file", cxxopts::value(m_TraceFile), "<filepath>"); -} - -TraceCommand::~TraceCommand() = default; - -void -TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - zen::HttpClient Http = CreateHttpClient(m_HostName); - - if (m_Stop) - { - if (zen::HttpClient::Response Response = Http.Post("/admin/trace/stop"sv)) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - } - else - { - Response.ThrowError("Trace stop failed"); - } - return; - } - - std::string StartArg; - if (!m_TraceHost.empty()) - { - StartArg = fmt::format("host={}", m_TraceHost); - } - else if (!m_TraceFile.empty()) - { - StartArg = fmt::format("file={}", m_TraceFile); - } - - if (!StartArg.empty()) - { - if (zen::HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg))) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - } - else - { - Response.ThrowError("Trace start failed"); - } - } - else - { - if (zen::HttpClient::Response Response = Http.Get("/admin/trace"sv)) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - } - else - { - Response.ThrowError("Trace status failed"); - } - } -} - -} // namespace zen diff --git a/src/zen/cmds/ui_cmd.cpp b/src/zen/cmds/ui_cmd.cpp index 4846b4d18..53dbb22da 100644 --- a/src/zen/cmds/ui_cmd.cpp +++ b/src/zen/cmds/ui_cmd.cpp @@ -2,6 +2,9 @@ #include "ui_cmd.h" +#include "browser_launcher.h" +#include "zenserviceclient.h" + #include <zencore/except_fmt.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -9,11 +12,6 @@ #include <zenutil/consoletui.h> #include <zenutil/zenserverprocess.h> -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -# include <shellapi.h> -#endif - namespace zen { namespace { @@ -83,40 +81,10 @@ UiCommand::OpenBrowser(std::string_view HostName) } } - bool Success = false; - ExtendableStringBuilder<256> FullUrl; FullUrl << HostName << m_DashboardPath; -#if ZEN_PLATFORM_WINDOWS - HINSTANCE Result = ShellExecuteA(nullptr, "open", FullUrl.c_str(), nullptr, nullptr, SW_SHOWNORMAL); - Success = reinterpret_cast<intptr_t>(Result) > 32; -#else - // Validate URL doesn't contain shell metacharacters that could lead to command injection - std::string_view FullUrlView = FullUrl; - constexpr std::string_view DangerousChars = ";|&$`\\\"'<>(){}[]!#*?~\n\r"; - if (FullUrlView.find_first_of(DangerousChars) != std::string_view::npos) - { - throw OptionParseException(fmt::format("URL contains invalid characters: '{}'", FullUrl), m_Options.help()); - } - -# if ZEN_PLATFORM_MAC - std::string Command = fmt::format("open \"{}\"", FullUrl); -# elif ZEN_PLATFORM_LINUX - std::string Command = fmt::format("xdg-open \"{}\"", FullUrl); -# else - ZEN_NOT_IMPLEMENTED("Browser launching not implemented on this platform"); -# endif - - Success = system(Command.c_str()) == 0; -#endif - - if (!Success) - { - throw zen::runtime_error("Failed to launch browser for '{}'", FullUrl); - } - - ZEN_CONSOLE("Web browser launched for '{}' successfully", FullUrl); + LaunchBrowser(std::string_view(FullUrl)); } void @@ -162,7 +130,7 @@ UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Labels.push_back(fmt::format("(all {} instances)", Servers.size())); const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns()); - constexpr int32_t kIndicator = 3; // " ▶ " or " " prefix + constexpr int32_t kIndicator = 3; // " > " or " " prefix constexpr int32_t kSeparator = 2; // " " before cmdline constexpr int32_t kEllipsis = 3; // "..." @@ -225,17 +193,14 @@ UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort); } - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); - if (IsUnixSocketSpec(m_HostName)) + if (IsUnixSocketSpec(Service.HostSpec())) { throw std::runtime_error("Cannot open browser for a Unix domain socket connection"); } - OpenBrowser(m_HostName); + OpenBrowser(Service.HostSpec()); } } // namespace zen diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index 809a41bb6..1f23e6819 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -2,6 +2,7 @@ #include "up_cmd.h" +#include <zencore/basicfile.h> #include <zencore/compactbinary.h> #include <zencore/compactbinaryutil.h> #include <zencore/filesystem.h> @@ -12,6 +13,57 @@ namespace zen { +namespace { + + bool TryShutdownByPid(ZenServerState& Instance, uint32_t Pid, const std::filesystem::path& ProgramBaseDir, bool ForceTerminate) + { + Instance.Sweep(); + + uint16_t DesiredPort = 0; + Instance.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (Entry.Pid.load() == Pid) + { + DesiredPort = Entry.DesiredListenPort.load(); + } + }); + + ZenServerState::ZenServerEntry* Entry = (DesiredPort != 0) ? Instance.Lookup(DesiredPort) : nullptr; + if (Entry && Entry->Pid.load() != Pid) + { + Entry = nullptr; + } + + if (Entry) + { + if (ShutdownZenServer(ConsoleLog(), Instance, Entry, ProgramBaseDir)) + { + return true; + } + } + + std::error_code Ec; + ProcessHandle Proc; + Proc.Initialize(int(Pid), Ec); + if (!Ec && Proc.IsValid() && !Proc.IsRunning()) + { + return true; + } + + if (ForceTerminate && !Ec && Proc.IsValid() && Proc.IsRunning()) + { + ZEN_CONSOLE_WARN("Hard terminating zen process with pid ({})", Pid); + if (Proc.Terminate(0)) + { + ZEN_CONSOLE("Terminate complete"); + return true; + } + } + + return false; + } + +} // namespace + UpCommand::UpCommand() { m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("0"), "<hostport>"); @@ -40,12 +92,19 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'--show-console' conflicts with '--show-log'", m_Options.help()); } + if (m_ProgramBaseDir.empty()) + { + m_ProgramBaseDir = GetRunningExecutablePath().parent_path(); + } + MakeSafeAbsolutePathInPlace(m_ProgramBaseDir); + std::optional<int> StartResult = StartupZenServer(ConsoleLog(), - {.ProgramBaseDir = m_ProgramBaseDir, - .Port = m_Port, - .OpenConsole = m_ShowConsole, - .ShowLog = m_ShowLog, - .ExtraArgs = GlobalOptions.PassthroughCommandLine}); + {.ProgramBaseDir = m_ProgramBaseDir, + .Port = m_Port, + .OpenConsole = m_ShowConsole, + .ShowLog = m_ShowLog, + .ExtraArgs = GlobalOptions.PassthroughCommandLine, + .EnableExecutionHistory = GlobalOptions.EnableExecutionHistory}); if (!StartResult.has_value()) { ZEN_CONSOLE("Zen server already running"); @@ -80,6 +139,11 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } + if (!m_DataDir.empty()) + { + MakeSafeAbsolutePathInPlace(m_DataDir); + } + ZenServerState Instance; Instance.Initialize(); Instance.Sweep(); @@ -87,9 +151,9 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { - if (!IsFile(m_DataDir / ".lock")) + if (!LockFile::IsHeldLive(m_DataDir / ".lock", /*AttemptCleanup*/ false)) { - throw std::runtime_error(fmt::format("Lock file does not exist in directory '{}'", m_DataDir)); + throw std::runtime_error(fmt::format("No live zen server holding lock file in directory '{}'", m_DataDir)); } CbValidateError ValidateResult = CbValidateError::None; if (CbObject LockFileObject = @@ -134,6 +198,13 @@ DownCommand::DownCommand() m_Options.add_option("", "f", "force", "Force terminate if graceful shutdown fails", cxxopts::value(m_ForceTerminate), "<force>"); m_Options.add_option("", "b", "base-dir", "Parent folder of server executable", cxxopts::value(m_ProgramBaseDir), "<directory>"); m_Options.add_option("", "", "data-dir", "Path to data directory to inspect for running server", cxxopts::value(m_DataDir), "<file>"); + m_Options.add_option("", "", "pid", "Shut down zen server process by PID", cxxopts::value(m_Pid)->default_value("0"), "<pid>"); + m_Options.add_option("", + "", + "executable", + "Shut down all zen server processes matching executable path", + cxxopts::value(m_ExecutablePath), + "<path>"); } DownCommand::~DownCommand() = default; @@ -148,16 +219,77 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } + const bool HasPid = m_Pid != 0; + const bool HasExecutable = !m_ExecutablePath.empty(); + const bool HasDataDir = !m_DataDir.empty(); + const int SelectorCount = int(m_All) + int(HasPid) + int(HasExecutable) + int(HasDataDir); + if (SelectorCount > 1) + { + throw OptionParseException("--all, --pid, --executable, and --data-dir are mutually exclusive", m_Options.help()); + } + if (m_ProgramBaseDir.empty()) { std::filesystem::path ExePath = GetRunningExecutablePath(); m_ProgramBaseDir = ExePath.parent_path(); } + MakeSafeAbsolutePathInPlace(m_ProgramBaseDir); + if (!m_DataDir.empty()) + { + MakeSafeAbsolutePathInPlace(m_DataDir); + } + if (!m_ExecutablePath.empty() && m_ExecutablePath.has_parent_path()) + { + MakeSafeAbsolutePathInPlace(m_ExecutablePath); + } // Discover executing instances ZenServerState Instance; Instance.Initialize(); + if (HasPid) + { + if (!TryShutdownByPid(Instance, m_Pid, m_ProgramBaseDir, m_ForceTerminate)) + { + throw std::runtime_error(fmt::format("Failed to shut down zen process with pid {}, use --force to hard terminate", m_Pid)); + } + ZEN_CONSOLE("Zen server with pid {} is down", m_Pid); + return; + } + + if (HasExecutable) + { + int ShutdownCount = 0; + while (true) + { + ProcessHandle Proc; + std::error_code Ec = FindProcess(m_ExecutablePath, Proc, /*IncludeSelf*/ false); + if (Ec) + { + throw std::system_error(Ec, fmt::format("FindProcess failed for '{}'", m_ExecutablePath)); + } + if (!Proc.IsValid()) + { + break; + } + const uint32_t Pid = uint32_t(Proc.Pid()); + if (!TryShutdownByPid(Instance, Pid, m_ProgramBaseDir, m_ForceTerminate)) + { + throw std::runtime_error(fmt::format("Failed to shut down zen process with pid {}, use --force to hard terminate", Pid)); + } + ++ShutdownCount; + } + if (ShutdownCount == 0) + { + ZEN_CONSOLE("No zen server processes matching executable '{}'", m_ExecutablePath); + } + else + { + ZEN_CONSOLE("Shut down {} zen server instance(s) matching executable '{}'", ShutdownCount, m_ExecutablePath); + } + return; + } + if (m_All) { struct EntryInfo @@ -184,15 +316,10 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) int FailCount = 0; for (const EntryInfo& Info : Entries) { - Instance.Sweep(); - ZenServerState::ZenServerEntry* Entry = Instance.Lookup(Info.Port); - if (Entry && Entry->Pid.load() == Info.Pid) + if (!TryShutdownByPid(Instance, Info.Pid, m_ProgramBaseDir, m_ForceTerminate)) { - if (!ShutdownZenServer(ConsoleLog(), Instance, Entry, m_ProgramBaseDir)) - { - ZEN_CONSOLE_WARN("Failed to shutdown server on port {} (pid {})", Info.Port, Info.Pid); - ++FailCount; - } + ZEN_CONSOLE_WARN("Failed to shutdown server on port {} (pid {})", Info.Port, Info.Pid); + ++FailCount; } } @@ -207,9 +334,10 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { - if (!IsFile(m_DataDir / ".lock")) + if (!LockFile::IsHeldLive(m_DataDir / ".lock", /*AttemptCleanup*/ true)) { - throw std::runtime_error(fmt::format("Lock file does not exist in directory '{}'", m_DataDir)); + ZEN_CONSOLE("No live zen server holding lock file in '{}', nothing to do", m_DataDir); + return; } CbValidateError ValidateResult = CbValidateError::None; if (CbObject LockFileObject = diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index f904fe0d9..bcc7d7da4 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -64,10 +64,12 @@ public: private: cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; + uint32_t m_Pid = 0; bool m_All = false; bool m_ForceTerminate = false; std::filesystem::path m_ProgramBaseDir; std::filesystem::path m_DataDir; + std::filesystem::path m_ExecutablePath; }; } // namespace zen diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp index 0948de1bb..19951f862 100644 --- a/src/zen/cmds/version_cmd.cpp +++ b/src/zen/cmds/version_cmd.cpp @@ -2,6 +2,8 @@ #include "version_cmd.h" +#include "zenserviceclient.h" + #include <zencore/basicfile.h> #include <zencore/config.h> #include <zencore/filesystem.h> @@ -57,7 +59,9 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Querying host {}", m_HostName); } - HttpClient Client = CreateHttpClient(m_HostName, {.Timeout = std::chrono::milliseconds(5000)}); + ZenServiceClient Service( + {.HostSpec = m_HostName, .CommandName = Name, .HttpSettings = {.Timeout = std::chrono::milliseconds(5000)}}); + HttpClient& Client = Service.Http(); HttpClient::KeyValueMap Parameters; if (m_DetailedVersion) diff --git a/src/zen/cmds/vfs_cmd.cpp b/src/zen/cmds/vfs_cmd.cpp index 29ad8dc7c..c07526789 100644 --- a/src/zen/cmds/vfs_cmd.cpp +++ b/src/zen/cmds/vfs_cmd.cpp @@ -2,6 +2,8 @@ #include "vfs_cmd.h" +#include "zenserviceclient.h" + #include <zencore/compactbinarybuilder.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -40,12 +42,8 @@ VfsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // Validate arguments - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (m_Verb == "mount"sv) { diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index 10f5ad8e1..d5344fb01 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -4,6 +4,7 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/iohash.h> #include <zencore/logging.h> #include <zencore/parallelwork.h> #include <zencore/string.h> @@ -11,7 +12,7 @@ #include <zencore/trace.h> #include <zenutil/workerpools.h> -#include "../progressbar.h" +#include "consoleprogress.h" #include <signal.h> @@ -34,13 +35,13 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace wipe_impl { - static std::atomic<bool> AbortFlag = false; - static std::atomic<bool> PauseFlag = false; - static bool IsVerbose = false; - static bool Quiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - const bool SingleThreaded = false; - bool BoostWorkerThreads = true; + static std::atomic<bool> AbortFlag = false; + static std::atomic<bool> PauseFlag = false; + static bool IsVerbose = false; + static bool Quiet = false; + static ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + const bool SingleThreaded = false; + bool BoostWorkerThreads = true; WorkerThreadPool& GetIOWorkerPool() { @@ -167,7 +168,8 @@ namespace wipe_impl { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; - ProgressBar Progress(ProgressMode, "Clean Folder"); + std::unique_ptr<ProgressBase> ProgressOwner(CreateConsoleProgress(ProgressMode)); + std::unique_ptr<ProgressBase::ProgressBar> Progress = ProgressOwner->CreateProgressBar("Clean Folder"); std::atomic<bool> CleanWipe = true; std::atomic<uint64_t> DiscoveredItemCount = 0; @@ -413,7 +415,7 @@ namespace wipe_impl { GetIOWorkerPool(), Work.PendingWork()); - Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { + Work.Wait(ProgressOwner->GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); if (Quiet) { @@ -424,12 +426,12 @@ namespace wipe_impl { uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); - Progress.UpdateState({.Task = "Removing files ", - .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), - .TotalCount = Discovered, - .RemainingCount = Discovered - Deleted, - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); + Progress->UpdateState({.Task = "Removing files ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted, + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); }); std::vector<std::filesystem::path> DirectoriesToDelete; @@ -473,22 +475,22 @@ namespace wipe_impl { } uint64_t NowMs = Timer.GetElapsedTimeMs(); - if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode)) + if ((NowMs - LastUpdateTimeMs) >= ProgressOwner->GetProgressUpdateDelayMS()) { LastUpdateTimeMs = NowMs; uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); - Progress.UpdateState({.Task = "Removing folders", - .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), - .TotalCount = DirectoriesToDelete.size(), - .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, - false); + Progress->UpdateState({.Task = "Removing folders", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = DirectoriesToDelete.size(), + .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, + false); } } - Progress.Finish(); + Progress->Finish(); uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); if (!Quiet) @@ -536,10 +538,10 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) using namespace wipe_impl; ZEN_UNUSED(GlobalOptions); - signal(SIGINT, SignalCallbackHandler); + ScopedSignalHandler SigIntGuard(SIGINT, SignalCallbackHandler); #if ZEN_PLATFORM_WINDOWS - signal(SIGBREAK, SignalCallbackHandler); -#endif // ZEN_PLATFORM_WINDOWS + ScopedSignalHandler SigBreakGuard(SIGBREAK, SignalCallbackHandler); +#endif if (!ParseOptions(argc, argv)) { @@ -548,7 +550,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Quiet = m_Quiet; IsVerbose = m_Verbose; - ProgressMode = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty; + ProgressMode = m_PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty; BoostWorkerThreads = m_BoostWorkerThreads; MakeSafeAbsolutePathInPlace(m_Directory); diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 9e49b464e..3ab3b9e04 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -2,6 +2,8 @@ #include "workspaces_cmd.h" +#include "zenserviceclient.h" + #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -137,7 +139,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'verb' option is required", m_Options.help()); } - // Parse subcommand permissively — forward unrecognised options to the parent parser. + // Parse subcommand permissively - forward unrecognised options to the parent parser. std::vector<std::string> SubUnmatched; if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched)) { @@ -159,8 +161,6 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - if (m_SystemRootDir.empty()) { m_SystemRootDir = PickDefaultSystemRootDirectory(); @@ -199,7 +199,8 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -271,7 +272,8 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -403,7 +405,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw OptionParseException("'verb' option is required", m_Options.help()); } - // Parse subcommand permissively — forward unrecognised options to the parent parser. + // Parse subcommand permissively - forward unrecognised options to the parent parser. std::vector<std::string> SubUnmatched; if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched)) { @@ -425,8 +427,6 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - if (m_SystemRootDir.empty()) { m_SystemRootDir = PickDefaultSystemRootDirectory(); @@ -509,7 +509,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -626,7 +627,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -674,12 +676,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** Params.Entries.insert_or_assign("refresh", ToString(m_Refresh)); } - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/files", GetShareIdentityUrl(m_FilesOptions)), {}, Params)) { ZEN_CONSOLE("{}: {}", Result, Result.ToText()); @@ -707,12 +705,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** Params.Entries.insert_or_assign("refresh", ToString(m_Refresh)); } - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/entries", GetShareIdentityUrl(m_EntriesOptions)), {}, Params)) { ZEN_CONSOLE("{}: {}", Result, Result.ToText()); @@ -777,18 +771,14 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_GetChunkOptions) { - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - if (m_ChunkId.empty()) { throw OptionParseException("'--chunk' is required", SubOption->help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, std::vector<std::string>{m_ChunkId})[0]; + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, std::vector<std::string>{m_ChunkId})[0]; HttpClient::KeyValueMap Params; if (m_Offset != 0) @@ -813,11 +803,6 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_GetChunkBatchOptions) { - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - if (m_ShareId.empty()) { throw OptionParseException("'--share' is required", SubOption->help()); @@ -828,8 +813,9 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw OptionParseException("'--chunks' is required", SubOption->help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ChunkIds = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, m_ChunkIds); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ChunkIds = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, m_ChunkIds); std::vector<RequestChunkEntry> ChunkRequests; ChunkRequests.resize(m_ChunkIds.size()); diff --git a/src/zen/consoleprogress.cpp b/src/zen/consoleprogress.cpp new file mode 100644 index 000000000..1726a08aa --- /dev/null +++ b/src/zen/consoleprogress.cpp @@ -0,0 +1,582 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +// Zen command line client utility +// + +#include "consoleprogress.h" + +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/timer.h> +#include <zencore/windows.h> +#include <zenutil/consoletui.h> +#include <zenutil/progress.h> + +#if !ZEN_PLATFORM_WINDOWS +# include <csignal> +#endif + +ZEN_THIRD_PARTY_INCLUDES_START +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +// Global tracking for scroll region cleanup on abnormal termination (Ctrl+C etc.) +// Only one ProgressBar can own a scroll region at a time. +static std::atomic<class ConsoleProgressBar*> g_ActiveScrollRegionOwner{nullptr}; +static std::atomic<uint32_t> g_ActiveScrollRegionRows{0}; + +static void +ResetScrollRegionRaw() +{ + // Signal-safe: emit raw escape sequences to restore terminal state. + // These are async-signal-safe on POSIX (write()) and safe in console + // ctrl handlers on Windows (WriteConsole is allowed). + uint32_t Rows = g_ActiveScrollRegionRows.load(std::memory_order_acquire); + if (Rows >= 3) + { + // Move to status line, erase it, reset scroll region, move cursor to end of content + TuiMoveCursor(Rows, 1); + TuiEraseLine(); + TuiResetScrollRegion(); + TuiMoveCursor(Rows - 1, 1); + } + else + { + TuiResetScrollRegion(); + } + TuiShowCursor(true); + TuiFlush(); +} + +#if ZEN_PLATFORM_WINDOWS +static BOOL WINAPI +ScrollRegionCtrlHandler(DWORD CtrlType) +{ + if (CtrlType == CTRL_C_EVENT || CtrlType == CTRL_BREAK_EVENT) + { + ResetScrollRegionRaw(); + } + // Return FALSE so the default handler (process termination) still runs + return FALSE; +} +#else +static struct sigaction s_PrevSigIntAction; +static struct sigaction s_PrevSigTermAction; + +static void +ScrollRegionSignalHandler(int Signal) +{ + ResetScrollRegionRaw(); + + // Re-raise with the previous handler + struct sigaction* PrevAction = (Signal == SIGINT) ? &s_PrevSigIntAction : &s_PrevSigTermAction; + sigaction(Signal, PrevAction, nullptr); + raise(Signal); +} +#endif + +static void +InstallScrollRegionCleanupHandler() +{ +#if ZEN_PLATFORM_WINDOWS + SetConsoleCtrlHandler(ScrollRegionCtrlHandler, TRUE); +#else + struct sigaction Action = {}; + Action.sa_handler = ScrollRegionSignalHandler; + Action.sa_flags = SA_RESETHAND; // one-shot + sigemptyset(&Action.sa_mask); + sigaction(SIGINT, &Action, &s_PrevSigIntAction); + sigaction(SIGTERM, &Action, &s_PrevSigTermAction); +#endif +} + +static void +RemoveScrollRegionCleanupHandler() +{ +#if ZEN_PLATFORM_WINDOWS + SetConsoleCtrlHandler(ScrollRegionCtrlHandler, FALSE); +#else + sigaction(SIGINT, &s_PrevSigIntAction, nullptr); + sigaction(SIGTERM, &s_PrevSigTermAction, nullptr); +#endif +} + +#if ZEN_PLATFORM_WINDOWS +static HANDLE +GetConsoleHandle() +{ + static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + return hStdOut; +} +#endif + +static void +OutputToConsoleRaw(const char* String, size_t Length) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdOut = GetConsoleHandle(); + if (TuiIsStdoutTty()) + { + WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); + } + else + { + ::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0); + } +#else + fwrite(String, 1, Length, stdout); +#endif +} + +static void +OutputToConsoleRaw(const std::string& String) +{ + OutputToConsoleRaw(String.c_str(), String.length()); +} + +static void +OutputToConsoleRaw(const StringBuilderBase& SB) +{ + OutputToConsoleRaw(SB.c_str(), SB.Size()); +} + +static uint32_t +GetUpdateDelayMS(ConsoleProgressMode InMode) +{ + switch (InMode) + { + case ConsoleProgressMode::Plain: + return 5000; + case ConsoleProgressMode::Pretty: + return 200; + case ConsoleProgressMode::Log: + return 2000; + case ConsoleProgressMode::Quiet: + return 5000; + default: + ZEN_ASSERT(false); + return 0; + } +} + +class ConsoleProgressBar : public ProgressBase::ProgressBar +{ +public: + explicit ConsoleProgressBar(ConsoleProgressMode InMode, std::string_view InSubTask); + ~ConsoleProgressBar(); + + void UpdateState(const State& NewState, bool DoLinebreak) override; + void ForceLinebreak() override; + void Finish() override; + +private: + void SetupScrollRegion(); + void TeardownScrollRegion(); + void RenderStatusLine(std::string_view Line); + + const ConsoleProgressMode m_Mode; + Stopwatch m_SW; + uint64_t m_LastUpdateMS; + uint64_t m_PausedMS; + State m_State; + const std::string m_SubTask; + size_t m_LastOutputLength = 0; + bool m_ScrollRegionActive = false; + uint32_t m_ScrollRegionRows = 0; +}; + +ConsoleProgressBar::ConsoleProgressBar(ConsoleProgressMode InMode, std::string_view InSubTask) +: m_Mode((!TuiIsStdoutTty() && InMode == ConsoleProgressMode::Pretty) ? ConsoleProgressMode::Plain : InMode) +, m_LastUpdateMS((uint64_t)-1) +, m_PausedMS(0) +, m_SubTask(InSubTask) +{ + ZEN_ASSERT(InSubTask.find('\"') == std::string_view::npos); + if (!m_SubTask.empty()) + { + if (m_Mode == ConsoleProgressMode::Log) + { + std::string String = fmt::format("@progress push \"{}\"\n", m_SubTask); + OutputToConsoleRaw(String); + } + } + + if (m_Mode == ConsoleProgressMode::Pretty) + { + SetupScrollRegion(); + } +} + +ConsoleProgressBar::~ConsoleProgressBar() +{ + try + { + TeardownScrollRegion(); + ForceLinebreak(); + if (!m_SubTask.empty()) + { + if (m_Mode == ConsoleProgressMode::Log) + { + const std::string String("@progress pop\n"); + OutputToConsoleRaw(String); + } + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("ConsoleProgressBar::~ConsoleProgressBar() failed with {}", Ex.what()); + } +} + +void +ConsoleProgressBar::SetupScrollRegion() +{ + // Only one scroll region owner at a time; nested bars fall back to the inline \r path. + if (g_ActiveScrollRegionOwner.load(std::memory_order_acquire) != nullptr) + { + return; + } + + uint32_t Rows = TuiConsoleRows(0); + if (Rows < 3) + { + return; + } + + TuiEnableOutput(); + + // Ensure cursor is not on the last row before we install the region. + // Print a newline to push content up if needed, then set the region. + OutputToConsoleRaw("\n", 1); + TuiSetScrollRegion(1, Rows - 1); + + // Move cursor into the scroll region so normal output stays there + TuiMoveCursor(Rows - 1, 1); + + m_ScrollRegionActive = true; + m_ScrollRegionRows = Rows; + + g_ActiveScrollRegionRows.store(Rows, std::memory_order_release); + g_ActiveScrollRegionOwner.store(this, std::memory_order_release); + InstallScrollRegionCleanupHandler(); +} + +void +ConsoleProgressBar::TeardownScrollRegion() +{ + if (!m_ScrollRegionActive) + { + return; + } + m_ScrollRegionActive = false; + + RemoveScrollRegionCleanupHandler(); + g_ActiveScrollRegionOwner.store(nullptr, std::memory_order_release); + g_ActiveScrollRegionRows.store(0, std::memory_order_release); + + // Emit all teardown escape sequences as a single atomic write + ExtendableStringBuilder<128> Buf; + Buf << fmt::format("\x1b[{};1H", m_ScrollRegionRows) // move to status line + << "\x1b[2K" // erase it + << "\x1b[r" // reset scroll region + << fmt::format("\x1b[{};1H", m_ScrollRegionRows - 1); // move to end of content + OutputToConsoleRaw(Buf); + TuiFlush(); +} + +void +ConsoleProgressBar::RenderStatusLine(std::string_view Line) +{ + // Handle terminal resizes by re-querying row count + uint32_t CurrentRows = TuiConsoleRows(0); + if (CurrentRows >= 3 && CurrentRows != m_ScrollRegionRows) + { + // Terminal was resized - reinstall scroll region + TuiSetScrollRegion(1, CurrentRows - 1); + m_ScrollRegionRows = CurrentRows; + } + + // Build the entire escape sequence as a single string so the console write + // is atomic and log output from other threads cannot interleave. + ExtendableStringBuilder<512> Buf; + Buf << "\x1b" + "7" // ESC 7 - save cursor + << fmt::format("\x1b[{};1H", m_ScrollRegionRows) // move to bottom row + << "\x1b[2K" // erase entire line + << Line // progress bar content + << "\x1b" + "8"; // ESC 8 - restore cursor + OutputToConsoleRaw(Buf); +} + +void +ConsoleProgressBar::UpdateState(const State& NewState, bool DoLinebreak) +{ + ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount); + ZEN_ASSERT(NewState.Task.find('\"') == std::string::npos); + if (DoLinebreak == false && m_State == NewState) + { + return; + } + + uint64_t ElapsedTimeMS = NewState.OptionalElapsedTime == (uint64_t)-1 ? m_SW.GetElapsedTimeMs() : NewState.OptionalElapsedTime; + if (m_LastUpdateMS != (uint64_t)-1) + { + if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) && + ((m_LastUpdateMS + 200) > ElapsedTimeMS)) + { + return; + } + if (m_State.Status == State::EStatus::Paused) + { + uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS; + m_PausedMS += ElapsedSinceLast; + } + } + + m_LastUpdateMS = ElapsedTimeMS; + + std::string Task = NewState.Task; + switch (NewState.Status) + { + case State::EStatus::Aborted: + Task = "Aborting"; + break; + case State::EStatus::Paused: + Task = "Paused"; + break; + default: + break; + } + if (NewState.Task.length() > Task.length()) + { + Task += std::string(NewState.Task.length() - Task.length(), ' '); + } + + const size_t PercentDone = + NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; + + uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; + uint64_t ETAElapsedMS = ElapsedTimeMS - m_PausedMS; + uint64_t ETAMS = ((m_State.TotalCount == NewState.TotalCount) && (NewState.Status == State::EStatus::Running)) && (PercentDone > 5) + ? (ETAElapsedMS * NewState.RemainingCount) / Completed + : 0; + const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; + + if (m_Mode == ConsoleProgressMode::Plain) + { + const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + const std::string Output = fmt::format("{} {}% {}{}{}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); + OutputToConsoleRaw(Output); + m_State = NewState; + } + else if (m_Mode == ConsoleProgressMode::Pretty) + { + size_t ProgressBarSize = 20; + + size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; + + uint32_t ConsoleColumns = TuiConsoleColumns(1024); + + const std::string PercentString = fmt::format("{:#3}%", PercentDone); + + const std::string ProgressBarString = + fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' ')); + + const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS)); + + const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : ""; + + ExtendableStringBuilder<256> OutputBuilder; + + OutputBuilder << Task << " " << PercentString; + if (OutputBuilder.Size() + 1 < ConsoleColumns) + { + size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); + bool ElapsedFits = RemainingSpace >= ElapsedString.length(); + RemainingSpace -= ElapsedString.length(); + bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length(); + RemainingSpace -= ETAString.length(); + bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length(); + RemainingSpace -= DetailsString.length(); + bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length(); + RemainingSpace -= ProgressBarString.length(); + + if (ProgressBarFits) + { + OutputBuilder << ProgressBarString; + } + if (ElapsedFits) + { + OutputBuilder << ElapsedString; + } + if (ETAFits) + { + OutputBuilder << ETAString; + } + if (DetailsFits) + { + OutputBuilder << DetailsString; + } + } + + if (m_ScrollRegionActive) + { + // Render on the pinned bottom status line + RenderStatusLine(OutputBuilder.ToView()); + } + else + { + // Fallback: inline \r-based overwrite (terminal too small for scroll region) + std::string_view Output = OutputBuilder.ToView(); + std::string::size_type EraseLength = + m_LastOutputLength > (Output.length() + 1) ? (m_LastOutputLength - Output.length() - 1) : 0; + ExtendableStringBuilder<256> LineToPrint; + + if (Output.length() + 1 + EraseLength >= ConsoleColumns) + { + if (m_LastOutputLength > 0) + { + LineToPrint << "\n"; + } + LineToPrint << Output; + DoLinebreak = true; + } + else + { + LineToPrint << "\r" << Output << std::string(EraseLength, ' '); + } + + if (DoLinebreak) + { + LineToPrint << "\n"; + } + + OutputToConsoleRaw(LineToPrint); + m_LastOutputLength = DoLinebreak ? 0 : (Output.length() + 1); // +1 for \r prefix + } + + m_State = NewState; + } + else if (m_Mode == ConsoleProgressMode::Log) + { + if (m_State.Task != NewState.Task || + m_State.Details != NewState.Details) // TODO: Should we output just because details change? Will this spam the log collector? + { + std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + for (std::string::value_type& Char : Details) + { + if (Char == '"') + { + Char = '\''; + } + } + const std::string Message = + fmt::format("@progress \"{} {}{}{}\"\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); + OutputToConsoleRaw(Message); + } + + const size_t OldPercentDone = + m_State.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u; + + if (OldPercentDone != PercentDone) + { + const std::string Progress = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(Progress); + } + m_State = NewState; + } +} + +void +ConsoleProgressBar::ForceLinebreak() +{ + if (m_LastOutputLength > 0) + { + State NewState = m_State; + UpdateState(NewState, /*DoLinebreak*/ true); + } +} + +void +ConsoleProgressBar::Finish() +{ + TeardownScrollRegion(); + + if (m_LastOutputLength > 0 || m_State.RemainingCount > 0) + { + State NewState = m_State; + NewState.RemainingCount = 0; + NewState.Details = ""; + UpdateState(NewState, /*DoLinebreak*/ true); + } + m_State = State{}; + m_LastOutputLength = 0; + m_SW.Reset(); +} + +class ConsoleProgress : public ProgressBase +{ +public: + ConsoleProgress(ConsoleProgressMode InMode) : m_Mode(InMode) {} + + virtual void SetLogOperationName(std::string_view Name) override + { + ZEN_ASSERT(Name.find('\"') == std::string_view::npos); + if (m_Mode == ConsoleProgressMode::Log) + { + std::string String = fmt::format("@progress \"{}\"\n", Name); + OutputToConsoleRaw(String); + } + } + + virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override + { + if (m_Mode == ConsoleProgressMode::Log) + { + const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u; + std::string String = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(String); + } + } + virtual void PushLogOperation(std::string_view Name) override + { + ZEN_ASSERT(Name.find('\"') == std::string_view::npos); + if (m_Mode == ConsoleProgressMode::Log) + { + std::string String = fmt::format("@progress push \"{}\"\n", Name); + OutputToConsoleRaw(String); + } + } + + virtual void PopLogOperation() override + { + if (m_Mode == ConsoleProgressMode::Log) + { + const std::string String("@progress pop\n"); + OutputToConsoleRaw(String); + } + } + + virtual uint32_t GetProgressUpdateDelayMS() const override { return GetUpdateDelayMS(m_Mode); } + + virtual std::unique_ptr<ProgressBase::ProgressBar> CreateProgressBar(std::string_view InSubTask) override + { + return std::make_unique<ConsoleProgressBar>(m_Mode, InSubTask); + } + +private: + ConsoleProgressMode m_Mode; +}; + +ProgressBase* +CreateConsoleProgress(ConsoleProgressMode InMode) +{ + return new ConsoleProgress(InMode); +} + +} // namespace zen diff --git a/src/zen/consoleprogress.h b/src/zen/consoleprogress.h new file mode 100644 index 000000000..3b68f4ca5 --- /dev/null +++ b/src/zen/consoleprogress.h @@ -0,0 +1,19 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenutil/progress.h> + +namespace zen { + +enum class ConsoleProgressMode +{ + Plain, + Pretty, + Log, + Quiet +}; + +ProgressBase* CreateConsoleProgress(ConsoleProgressMode InMode); + +} // namespace zen diff --git a/src/zen/frontend/html/api.js b/src/zen/frontend/html/api.js new file mode 100644 index 000000000..fbe5304ca --- /dev/null +++ b/src/zen/frontend/html/api.js @@ -0,0 +1,137 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// Thin wrappers around the /api/* endpoints exposed by TraceViewerService. + +const API = "api/"; + +const JSON_HEADERS = { Accept: "application/json" }; + +async function getJson(path) { + const response = await fetch(API + path, { headers: JSON_HEADERS }); + if (!response.ok) { + throw new Error(`${path}: HTTP ${response.status}`); + } + return response.json(); +} + +export function getSession() { + return getJson("session"); +} + +export function getThreads() { + return getJson("threads"); +} + +export function getChannels() { + return getJson("channels"); +} + +export function getScopeStats() { + return getJson("scope-stats"); +} + +export function getScopeNames() { + return getJson("scope-names"); +} + +export async function getTimeline(threadId, startUs, endUs, minDurUs = 0, resolution = 0, { signal } = {}) { + const params = new URLSearchParams({ + thread: String(threadId), + start: String(startUs), + end: String(endUs), + }); + if (minDurUs > 0) params.set("mindur", String(minDurUs)); + if (resolution > 0) params.set("resolution", String(resolution)); + const response = await fetch(API + "timeline?" + params.toString(), { signal, headers: JSON_HEADERS }); + if (!response.ok) { + throw new Error(`timeline: HTTP ${response.status}`); + } + return response.json(); +} + +export async function getTimelineBatch(threadIds, startUs, endUs, minDurUs = 0, resolution = 0, { signal } = {}) { + let url = `${API}timeline-batch?threads=${threadIds.join(",")}&start=${startUs}&end=${endUs}`; + if (minDurUs > 0) url += `&mindur=${minDurUs}`; + if (resolution > 0) url += `&resolution=${resolution}`; + const response = await fetch(url, { signal, headers: JSON_HEADERS }); + if (!response.ok) { + throw new Error(`timeline-batch: HTTP ${response.status}`); + } + return response.json(); +} + +export function getLogCategories() { + return getJson("log-categories"); +} + +export function getLogs({ startUs = 0, endUs = 0xffffffff, minVerbosity = 0, category = null, limit = 5000 } = {}) { + const params = new URLSearchParams({ + start: String(startUs), + end: String(endUs), + min_verbosity: String(minVerbosity), + limit: String(limit), + }); + if (category !== null && category !== undefined) { + params.set("category", String(category)); + } + return getJson("logs?" + params.toString()); +} + +export function getBookmarks() { + return getJson("bookmarks"); +} + +export function getRegions() { + return getJson("regions"); +} + +export function getCsvCategories() { + return getJson("csv-categories"); +} + +export function getCsvStats() { + return getJson("csv-stats"); +} + +export function getCsvSeries(statId, threadId) { + let url = "csv-series?"; + if (statId != null) url += `stat=${statId}&`; + if (threadId != null) url += `thread=${threadId}&`; + return getJson(url); +} + +export function getCsvEvents() { + return getJson("csv-events"); +} + +export function getCsvMetadata() { + return getJson("csv-metadata"); +} + +export function getAllocSummary() { + return getJson("alloc-summary"); +} + +export function getMemoryTimeline({ startUs = 0, endUs = 0xffffffff, maxSamples = 2000 } = {}) { + const params = new URLSearchParams({ + start: String(startUs), + end: String(endUs), + max_samples: String(maxSamples), + }); + return getJson("memory-timeline?" + params.toString()); +} + +export function getCallstackStats(limit = 100) { + return getJson("callstack-stats?limit=" + encodeURIComponent(limit)); +} + +export function getChurnStats(limit = 100) { + return getJson("churn-stats?limit=" + encodeURIComponent(limit)); +} + +export function getCallstack(callstackId) { + return getJson("callstacks?id=" + encodeURIComponent(callstackId)); +} + +export function getAllocSizeHistogram() { + return getJson("alloc-size-histogram"); +} diff --git a/src/zen/frontend/html/csvstats.js b/src/zen/frontend/html/csvstats.js new file mode 100644 index 000000000..a50b2f068 --- /dev/null +++ b/src/zen/frontend/html/csvstats.js @@ -0,0 +1,383 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// CSV Profiler stats viewer — category/stat tree with line-chart visualization. + +import { getCsvSeries } from "./api.js"; + +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, (c) => ({"&":"&","<":"<",">":">","\"":""","'":"'"}[c])); +} + +function formatTime(us) { + if (us < 1000) return `${us} \u00b5s`; + if (us < 1_000_000) return `${(us / 1000).toFixed(2)} ms`; + return `${(us / 1_000_000).toFixed(2)} s`; +} + +// Palette for chart lines — distinct hues. +const LINE_COLORS = [ + "#4fc3f7", "#81c784", "#ffb74d", "#e57373", "#ba68c8", + "#4db6ac", "#fff176", "#f06292", "#7986cb", "#a1887f", +]; + +export class CsvStatsView { + constructor(model, containerEl) { + this.model = model; + this.container = containerEl; + this.loaded = false; + + this.categories = model.csvCategories || []; + this.statDefs = model.csvStats || []; + + // Group stats by category index. + this.catMap = new Map(); + for (const cat of this.categories) { + this.catMap.set(cat.index, cat.name); + } + this.statsByCategory = new Map(); + for (const s of this.statDefs) { + const catIdx = s.category_index; + if (!this.statsByCategory.has(catIdx)) this.statsByCategory.set(catIdx, []); + this.statsByCategory.get(catIdx).push(s); + } + + // Selected series: Set of stat_id values to chart. + this.selectedStats = new Set(); + this.seriesData = new Map(); // stat_id → [{time_us, value}, ...] + this.colorIndex = 0; + this.statColors = new Map(); // stat_id → color + + this.buildLayout(); + } + + buildLayout() { + this.container.innerHTML = + `<div class="csv-layout">` + + `<div class="csv-tree-panel">` + + `<div class="sidebar-label">Stats</div>` + + `<div class="csv-tree"></div>` + + `</div>` + + `<div class="csv-chart-panel">` + + `<canvas class="csv-chart-canvas"></canvas>` + + `<div class="csv-chart-tooltip" hidden></div>` + + `</div>` + + `</div>`; + + this.treeEl = this.container.querySelector(".csv-tree"); + this.canvas = this.container.querySelector(".csv-chart-canvas"); + this.tooltipEl = this.container.querySelector(".csv-chart-tooltip"); + this.ctx = this.canvas.getContext("2d"); + this.dpr = Math.max(1, window.devicePixelRatio || 1); + + this.renderTree(); + + // Chart interaction + this.resizeObserver = new ResizeObserver(() => this.drawChart()); + this.resizeObserver.observe(this.canvas); + this.canvas.addEventListener("mousemove", (e) => this.onChartHover(e)); + this.canvas.addEventListener("mouseleave", () => { this.tooltipEl.hidden = true; }); + + // Pan + zoom state + this.viewStartUs = 0; + this.viewEndUs = this.model.session.trace_end_us || 1; + this.panning = false; + this.canvas.addEventListener("mousedown", (e) => this.onPanStart(e)); + window.addEventListener("mousemove", (e) => this.onPanMove(e)); + window.addEventListener("mouseup", () => this.onPanEnd()); + this.canvas.addEventListener("wheel", (e) => this.onWheel(e), { passive: false }); + } + + renderTree() { + const parts = []; + // Sort categories by index. + const catIndices = Array.from(this.statsByCategory.keys()).sort((a, b) => a - b); + for (const catIdx of catIndices) { + const catName = this.catMap.get(catIdx) || `Category ${catIdx}`; + const stats = this.statsByCategory.get(catIdx); + parts.push(`<div class="csv-cat-header">${escapeHtml(catName)}</div>`); + for (const s of stats) { + const id = s.stat_id; + parts.push( + `<label class="csv-stat-row">` + + `<input type="checkbox" data-stat-id="${id}">` + + `<span class="csv-stat-name"></span>` + + `</label>` + ); + } + } + if (parts.length === 0) { + parts.push(`<div class="csv-empty">No CSV profiler data in this trace.</div>`); + } + this.treeEl.innerHTML = parts.join(""); + + // Set names via DOM (XSS safe). + let statIdx = 0; + for (const catIdx of catIndices) { + const stats = this.statsByCategory.get(catIdx); + for (const s of stats) { + const rows = this.treeEl.querySelectorAll(".csv-stat-row"); + if (statIdx < rows.length) { + rows[statIdx].querySelector(".csv-stat-name").textContent = s.name; + } + statIdx++; + } + } + + // Wire checkboxes. + for (const cb of this.treeEl.querySelectorAll("input[type=checkbox]")) { + cb.addEventListener("change", () => { + const statId = Number(cb.dataset.statId); + if (cb.checked) { + this.selectedStats.add(statId); + if (!this.statColors.has(statId)) { + this.statColors.set(statId, LINE_COLORS[this.colorIndex++ % LINE_COLORS.length]); + } + this.fetchSeries(statId); + } else { + this.selectedStats.delete(statId); + this.drawChart(); + } + }); + } + } + + async ensureLoaded() { + // Tree is built in constructor; nothing extra to lazy-load. + this.loaded = true; + this.drawChart(); + } + + async fetchSeries(statId) { + if (this.seriesData.has(statId)) { + this.drawChart(); + return; + } + try { + const result = await getCsvSeries(statId); + // Merge all threads' samples for this stat into one combined array for now. + const allSamples = []; + for (const series of result) { + for (const [timeUs, value] of series.samples) { + allSamples.push({ timeUs, value }); + } + } + allSamples.sort((a, b) => a.timeUs - b.timeUs); + this.seriesData.set(statId, allSamples); + this.drawChart(); + } catch (e) { + console.error(`Failed to fetch CSV series for stat ${statId}: ${e.message}`); + } + } + + resizeCanvas() { + const rect = this.canvas.getBoundingClientRect(); + this.width = Math.floor(rect.width); + this.height = Math.floor(rect.height); + const bw = Math.floor(rect.width * this.dpr); + const bh = Math.floor(rect.height * this.dpr); + if (this.canvas.width !== bw || this.canvas.height !== bh) { + this.canvas.width = bw; + this.canvas.height = bh; + } + this.ctx.setTransform(this.dpr, 0, 0, this.dpr, 0, 0); + } + + drawChart() { + this.resizeCanvas(); + const ctx = this.ctx; + const W = this.width; + const H = this.height; + + const bg = getComputedStyle(document.body).getPropertyValue("--bg0") || "#0d1117"; + const fg2 = getComputedStyle(document.body).getPropertyValue("--fg2") || "#8b949e"; + const border = getComputedStyle(document.body).getPropertyValue("--border") || "#30363d"; + ctx.fillStyle = bg; + ctx.fillRect(0, 0, W, H); + + if (this.selectedStats.size === 0) { + ctx.fillStyle = fg2; + ctx.font = "12px -apple-system, Segoe UI, sans-serif"; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.fillText("Select stats from the tree to chart them", W / 2, H / 2); + return; + } + + const PAD_L = 60, PAD_R = 12, PAD_T = 12, PAD_B = 28; + const chartW = W - PAD_L - PAD_R; + const chartH = H - PAD_T - PAD_B; + if (chartW <= 0 || chartH <= 0) return; + + const startUs = this.viewStartUs; + const endUs = this.viewEndUs; + const rangeUs = Math.max(1, endUs - startUs); + + // Compute value range across all visible selected series. + let minVal = Infinity, maxVal = -Infinity; + for (const statId of this.selectedStats) { + const samples = this.seriesData.get(statId); + if (!samples) continue; + for (const s of samples) { + if (s.timeUs < startUs || s.timeUs > endUs) continue; + if (s.value < minVal) minVal = s.value; + if (s.value > maxVal) maxVal = s.value; + } + } + if (!isFinite(minVal)) { minVal = 0; maxVal = 1; } + if (minVal === maxVal) { minVal -= 0.5; maxVal += 0.5; } + const valRange = maxVal - minVal; + const valPad = valRange * 0.05; + minVal -= valPad; + maxVal += valPad; + + const xAt = (us) => PAD_L + (us - startUs) / rangeUs * chartW; + const yAt = (v) => PAD_T + (1 - (v - minVal) / (maxVal - minVal)) * chartH; + + // Grid lines. + ctx.strokeStyle = border; + ctx.lineWidth = 0.5; + for (let i = 0; i <= 4; i++) { + const y = PAD_T + chartH * i / 4; + ctx.beginPath(); ctx.moveTo(PAD_L, y); ctx.lineTo(PAD_L + chartW, y); ctx.stroke(); + } + + // Y axis labels. + ctx.fillStyle = fg2; + ctx.font = "10px -apple-system, Segoe UI, sans-serif"; + ctx.textAlign = "right"; + ctx.textBaseline = "middle"; + for (let i = 0; i <= 4; i++) { + const v = minVal + (maxVal - minVal) * (1 - i / 4); + const y = PAD_T + chartH * i / 4; + ctx.fillText(v.toFixed(2), PAD_L - 4, y); + } + + // X axis labels. + ctx.textAlign = "center"; + ctx.textBaseline = "top"; + const tickCount = Math.max(2, Math.min(8, Math.floor(chartW / 80))); + for (let i = 0; i <= tickCount; i++) { + const us = startUs + rangeUs * i / tickCount; + const x = xAt(us); + ctx.fillText(formatTime(us), x, PAD_T + chartH + 4); + } + + // Draw lines. + for (const statId of this.selectedStats) { + const samples = this.seriesData.get(statId); + if (!samples || samples.length === 0) continue; + const color = this.statColors.get(statId) || "#fff"; + + ctx.strokeStyle = color; + ctx.lineWidth = 1.5; + ctx.beginPath(); + let started = false; + for (const s of samples) { + if (s.timeUs < startUs || s.timeUs > endUs) continue; + const x = xAt(s.timeUs); + const y = yAt(s.value); + if (!started) { ctx.moveTo(x, y); started = true; } + else { ctx.lineTo(x, y); } + } + ctx.stroke(); + } + + // Chart border. + ctx.strokeStyle = border; + ctx.lineWidth = 1; + ctx.strokeRect(PAD_L, PAD_T, chartW, chartH); + + // Legend. + ctx.font = "10px -apple-system, Segoe UI, sans-serif"; + ctx.textAlign = "left"; + ctx.textBaseline = "top"; + let legendX = PAD_L + 6; + for (const statId of this.selectedStats) { + const def = this.statDefs.find(d => d.stat_id === statId); + const name = def ? def.name : `stat ${statId}`; + const color = this.statColors.get(statId) || "#fff"; + ctx.fillStyle = color; + ctx.fillRect(legendX, PAD_T + 4, 10, 10); + ctx.fillStyle = "#ccc"; + ctx.fillText(name, legendX + 14, PAD_T + 4); + legendX += ctx.measureText(name).width + 24; + } + + // Store layout for hover. + this._chartLayout = { PAD_L, PAD_R, PAD_T, PAD_B, chartW, chartH, startUs, endUs, rangeUs, minVal, maxVal, xAt, yAt }; + } + + onChartHover(e) { + if (!this._chartLayout || this.selectedStats.size === 0) { + this.tooltipEl.hidden = true; + return; + } + const rect = this.canvas.getBoundingClientRect(); + const mx = e.clientX - rect.left; + const my = e.clientY - rect.top; + const { PAD_L, PAD_T, chartW, chartH, startUs, rangeUs } = this._chartLayout; + + if (mx < PAD_L || mx > PAD_L + chartW || my < PAD_T || my > PAD_T + chartH) { + this.tooltipEl.hidden = true; + return; + } + + const cursorUs = startUs + (mx - PAD_L) / chartW * rangeUs; + const lines = []; + for (const statId of this.selectedStats) { + const samples = this.seriesData.get(statId); + if (!samples || samples.length === 0) continue; + // Find nearest sample. + let best = null, bestDist = Infinity; + for (const s of samples) { + const d = Math.abs(s.timeUs - cursorUs); + if (d < bestDist) { bestDist = d; best = s; } + } + if (best) { + const def = this.statDefs.find(d => d.stat_id === statId); + const name = def ? def.name : `stat ${statId}`; + const color = this.statColors.get(statId) || "#fff"; + lines.push(`<span style="color:${color}">${escapeHtml(name)}</span>: ${best.value.toFixed(3)}`); + } + } + if (lines.length === 0) { this.tooltipEl.hidden = true; return; } + this.tooltipEl.innerHTML = `<div style="margin-bottom:2px">${formatTime(cursorUs)}</div>` + lines.join("<br>"); + this.tooltipEl.style.left = `${mx + 12}px`; + this.tooltipEl.style.top = `${my + 12}px`; + this.tooltipEl.hidden = false; + } + + onPanStart(e) { + if (e.button !== 0) return; + this.panning = true; + this.panStartX = e.clientX; + this.panStartViewStart = this.viewStartUs; + this.panStartViewEnd = this.viewEndUs; + } + + onPanMove(e) { + if (!this.panning || !this._chartLayout) return; + const dx = e.clientX - this.panStartX; + const usPerPx = (this.panStartViewEnd - this.panStartViewStart) / this._chartLayout.chartW; + const shift = -dx * usPerPx; + this.viewStartUs = this.panStartViewStart + shift; + this.viewEndUs = this.panStartViewEnd + shift; + this.drawChart(); + } + + onPanEnd() { this.panning = false; } + + onWheel(e) { + e.preventDefault(); + if (!this._chartLayout) return; + const rect = this.canvas.getBoundingClientRect(); + const mx = e.clientX - rect.left; + const { PAD_L, chartW, startUs, rangeUs } = this._chartLayout; + const cursorUs = startUs + (mx - PAD_L) / chartW * rangeUs; + const factor = e.deltaY > 0 ? 1.25 : 0.8; + const newRange = Math.max(10, (this.viewEndUs - this.viewStartUs) * factor); + const ratio = (mx - PAD_L) / chartW; + this.viewStartUs = cursorUs - ratio * newRange; + this.viewEndUs = this.viewStartUs + newRange; + this.drawChart(); + } +} diff --git a/src/zen/frontend/html/index.html b/src/zen/frontend/html/index.html new file mode 100644 index 000000000..5853a80dc --- /dev/null +++ b/src/zen/frontend/html/index.html @@ -0,0 +1,95 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>zen trace viewer</title> + <link rel="stylesheet" href="trace.css"> +</head> +<body> + <noscript>This viewer requires JavaScript.</noscript> + <div class="header"> + <div class="header-title">zen trace viewer</div> + <div class="header-file" id="hdr-file"></div> + <div class="header-stats" id="hdr-stats"></div> + <button id="theme-toggle" class="header-btn" type="button" title="Toggle dark/light mode">Theme</button> + </div> + <div class="layout"> + <aside class="sidebar"> + <nav class="tabs"> + <button class="tab active" data-tab="timeline">Timeline</button> + <button class="tab" data-tab="stats">Stats</button> + <button class="tab" data-tab="memory">Memory</button> + <button class="tab" data-tab="logs">Logs</button> + <button class="tab" data-tab="csv">CSV</button> + <button class="tab" data-tab="session">Session</button> + </nav> + <div class="sidebar-section"> + <div class="sidebar-label">Search scopes</div> + <input id="search-input" type="text" placeholder="filter scopes..." autocomplete="off" spellcheck="false"> + <div id="search-results" class="search-results"></div> + </div> + <div class="sidebar-section" id="regions-panel" hidden> + <div class="sidebar-label">Regions <button id="regions-toggle-all" class="sidebar-action">deselect all</button></div> + <div id="regions-list" class="regions-list"></div> + </div> + <div class="sidebar-section" id="threads-panel"> + <div class="sidebar-label">Threads <button id="threads-toggle-all" class="sidebar-action">deselect all</button></div> + <div id="threads-list" class="threads-list"></div> + </div> + </aside> + <main class="content"> + <section class="view view-timeline" data-view="timeline"> + <div class="timeline-toolbar"> + <div id="viewport-info" class="viewport-info"></div> + <label class="toolbar-toggle" title="Show or hide bookmark markers"> + <input type="checkbox" id="bookmarks-toggle" checked> + <span>Bookmarks</span> + </label> + <label class="toolbar-toggle" title="Disable LOD to always fetch full-resolution scopes (slower but useful for validating LOD correctness)"> + <input type="checkbox" id="lod-toggle" checked> + <span>LOD</span> + </label> + <button id="zoom-reset" class="btn">Reset view</button> + </div> + <div class="timeline-frame"> + <canvas id="timeline-canvas"></canvas> + <div id="tooltip" class="tooltip" hidden></div> + </div> + <div id="selection-panel" class="selection-panel"> + <div class="selection-hint">Click a scope to see details. Drag to pan, wheel to zoom.</div> + </div> + </section> + <section class="view view-stats" data-view="stats" hidden> + <table class="stats-table"> + <thead> + <tr> + <th data-sort="name">Scope</th> + <th data-sort="count" class="num">Count</th> + <th data-sort="min_us" class="num">Min (ms)</th> + <th data-sort="mean_us" class="num">Mean (ms)</th> + <th data-sort="max_us" class="num">Max (ms)</th> + <th data-sort="stdev_us" class="num">σ (ms)</th> + </tr> + </thead> + <tbody id="stats-tbody"></tbody> + </table> + </section> + <section class="view view-memory" data-view="memory" hidden> + <div id="memory-content"></div> + </section> + <section class="view view-logs" data-view="logs" hidden> + <div id="logs-content"></div> + </section> + <section class="view view-csv" data-view="csv" hidden> + <div id="csv-content"></div> + </section> + <section class="view view-session" data-view="session" hidden> + <div id="session-content" class="session-content"></div> + </section> + </main> + </div> + <div id="loading" class="loading">Loading trace…</div> + <script type="module" src="trace.js"></script> +</body> +</html> diff --git a/src/zen/frontend/html/logs.js b/src/zen/frontend/html/logs.js new file mode 100644 index 000000000..d9646ba39 --- /dev/null +++ b/src/zen/frontend/html/logs.js @@ -0,0 +1,237 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// Log viewer: filterable list of captured Logging.LogMessage events. + +import { getLogs } from "./api.js"; + +// UE ELogVerbosity::Type values — lower number = more severe. +const VERBOSITY_LABELS = [ + "NoLogging", + "Fatal", + "Error", + "Warning", + "Display", + "Log", + "Verbose", + "VeryVerbose", + "All", +]; + +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, (c) => ({ + "&": "&", + "<": "<", + ">": ">", + "\"": """, + "'": "'", + }[c])); +} + +function verbosityLabel(v) { + return VERBOSITY_LABELS[v] || `V${v}`; +} + +function verbosityClass(v) { + switch (v) { + case 1: return "vb-fatal"; + case 2: return "vb-error"; + case 3: return "vb-warn"; + case 4: return "vb-display"; + case 5: return "vb-log"; + case 6: case 7: return "vb-verbose"; + default: return "vb-other"; + } +} + +function formatTime(us) { + const totalMs = Math.floor(us / 1000); + const ms = totalMs % 1000; + const totalS = Math.floor(totalMs / 1000); + const s = totalS % 60; + const totalM = Math.floor(totalS / 60); + const m = totalM % 60; + const h = Math.floor(totalM / 60); + if (h > 0) { + return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(ms).padStart(3, "0")}`; + } + return `${m}:${String(s).padStart(2, "0")}.${String(ms).padStart(3, "0")}`; +} + +export class LogsView { + constructor(model, containerEl) { + this.model = model; + this.container = containerEl; + this.minVerbosity = 0; + this.category = ""; + this.textFilter = ""; + this.loaded = false; + this.rendering = null; + + this.container.innerHTML = + `<div class="logs-toolbar"> + <div class="logs-filter"> + <span class="logs-filter-label">Verbosity</span> + <select id="log-verbosity"> + <option value="0">All</option> + <option value="6">Verbose</option> + <option value="5">Log</option> + <option value="4">Display</option> + <option value="3">Warning</option> + <option value="2">Error</option> + <option value="1">Fatal</option> + </select> + </div> + <div class="logs-filter"> + <span class="logs-filter-label">Category</span> + <select id="log-category"> + <option value="">All</option> + </select> + </div> + <div class="logs-filter logs-filter-grow"> + <span class="logs-filter-label">Search</span> + <input id="log-search" type="text" placeholder="filter messages..." autocomplete="off" spellcheck="false"> + </div> + <div id="log-count" class="logs-count"></div> + </div> + <div class="logs-list-wrap"> + <table class="logs-table"> + <thead><tr> + <th class="col-time">Time</th> + <th class="col-verb">Verbosity</th> + <th class="col-cat">Category</th> + <th class="col-msg">Message</th> + <th class="col-loc">Source</th> + </tr></thead> + <tbody id="logs-tbody"></tbody> + </table> + </div>`; + + const catSelect = this.container.querySelector("#log-category"); + if ((this.model.bookmarks || []).length > 0) { + const bmOpt = document.createElement("option"); + bmOpt.value = "bookmarks"; + bmOpt.textContent = "(bookmarks only)"; + catSelect.appendChild(bmOpt); + } + for (let i = 0; i < this.model.logCategories.length; i++) { + const c = this.model.logCategories[i]; + const opt = document.createElement("option"); + opt.value = String(i); + opt.textContent = c.name || `category ${i}`; + catSelect.appendChild(opt); + } + + this.container.querySelector("#log-verbosity").addEventListener("change", (e) => { + this.minVerbosity = Number(e.target.value) || 0; + this.refresh(); + }); + this.container.querySelector("#log-category").addEventListener("change", (e) => { + this.category = e.target.value; + this.refresh(); + }); + this.container.querySelector("#log-search").addEventListener("input", (e) => { + this.textFilter = e.target.value.toLowerCase(); + this.renderFiltered(); + }); + } + + async ensureLoaded() { + if (this.loaded) return; + this.loaded = true; + await this.refresh(); + } + + async refresh() { + // "(bookmarks only)" is a synthetic category that displays just the + // bookmark rows without hitting the /api/logs endpoint. + if (this.category === "bookmarks") { + this.result = { entries: [], total: 0, returned: 0 }; + this.renderFiltered(); + return; + } + + const opts = { + minVerbosity: this.minVerbosity, + limit: 5000, + }; + if (this.category !== "") { + opts.category = Number(this.category); + } + try { + this.result = await getLogs(opts); + } catch (e) { + this.container.querySelector("#logs-tbody").innerHTML = + `<tr><td colspan="5" class="logs-error">Failed to load logs: ${escapeHtml(e.message)}</td></tr>`; + return; + } + this.renderFiltered(); + } + + renderFiltered() { + if (!this.result) return; + const entries = this.result.entries || []; + const filter = this.textFilter; + const tbody = this.container.querySelector("#logs-tbody"); + const count = this.container.querySelector("#log-count"); + + // Bookmarks are interleaved into the display list when the category + // filter is "All" or "(bookmarks only)". Any other category is log- + // specific so bookmarks are hidden to avoid confusion. + const showBookmarks = (this.category === "" || this.category === "bookmarks"); + const bookmarks = showBookmarks ? (this.model.bookmarks || []) : []; + + // Build a combined, time-sorted row list. Each item keeps its kind + // so we can render log and bookmark rows differently. + const items = []; + for (const e of entries) { + items.push({ kind: "log", time: e.time_us, entry: e }); + } + for (const b of bookmarks) { + items.push({ kind: "bookmark", time: b.time_us, entry: b }); + } + items.sort((a, b) => a.time - b.time); + + const rows = []; + let shown = 0; + for (const it of items) { + if (it.kind === "log") { + const e = it.entry; + if (filter && !e.message.toLowerCase().includes(filter)) continue; + const cat = this.model.logCategories[e.category_index] || { name: "(unknown)" }; + const file = e.file ? String(e.file).split(/[\\/]/).pop() : ""; + rows.push( + `<tr class="${verbosityClass(e.verbosity)}">` + + `<td class="col-time mono">${formatTime(e.time_us)}</td>` + + `<td class="col-verb">${escapeHtml(verbosityLabel(e.verbosity))}</td>` + + `<td class="col-cat">${escapeHtml(cat.name)}</td>` + + `<td class="col-msg">${escapeHtml(e.message)}</td>` + + `<td class="col-loc mono">${escapeHtml(file)}${e.line ? ":" + e.line : ""}</td>` + + `</tr>`, + ); + } else { + const b = it.entry; + if (filter && !b.text.toLowerCase().includes(filter)) continue; + const file = b.file ? String(b.file).split(/[\\/]/).pop() : ""; + rows.push( + `<tr class="bm-row">` + + `<td class="col-time mono">${formatTime(b.time_us)}</td>` + + `<td class="col-verb">BOOKMARK</td>` + + `<td class="col-cat">—</td>` + + `<td class="col-msg">${escapeHtml(b.text)}</td>` + + `<td class="col-loc mono">${escapeHtml(file)}${b.line ? ":" + b.line : ""}</td>` + + `</tr>`, + ); + } + shown++; + } + tbody.innerHTML = rows.join("") || + `<tr><td colspan="5" class="logs-empty">No entries match the current filter.</td></tr>`; + + const total = (this.result.total || 0) + bookmarks.length; + const returned = (this.result.returned || 0) + bookmarks.length; + if (total > returned) { + count.textContent = `${shown.toLocaleString()} shown · ${returned.toLocaleString()} of ${total.toLocaleString()} loaded`; + } else { + count.textContent = `${shown.toLocaleString()} of ${total.toLocaleString()}`; + } + } +} diff --git a/src/zen/frontend/html/memory.js b/src/zen/frontend/html/memory.js new file mode 100644 index 000000000..6b9760439 --- /dev/null +++ b/src/zen/frontend/html/memory.js @@ -0,0 +1,790 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// Interactive memory analysis view: summary cards, memory timeline, leak/churn/hot callsite tables. + +import { getAllocSummary, getMemoryTimeline, getCallstackStats, getChurnStats, getCallstack, getAllocSizeHistogram } from "./api.js"; + +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, (c) => ({ + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", + }[c])); +} + +function formatNum(n) { + return Number(n || 0).toLocaleString(); +} + +function formatBytes(bytes) { + const sign = bytes < 0 ? "-" : ""; + let value = Math.abs(Number(bytes || 0)); + const units = ["B", "KB", "MB", "GB", "TB"]; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + const decimals = unit === 0 ? 0 : value >= 100 ? 0 : value >= 10 ? 1 : 2; + return `${sign}${value.toFixed(decimals)} ${units[unit]}`; +} + +function formatDistance(events) { + return `${Math.round(Number(events || 0)).toLocaleString()} ev`; +} + +function formatTimeAxis(us) { + const value = Number(us || 0); + if (value < 1000) return `${Math.round(value)} µs`; + if (value < 1_000_000) return `${Math.round(value / 1000)} ms`; + return `${Math.round(value / 1_000_000)} s`; +} + +function chooseNiceTimeStep(spanUs, targetTickCount) { + const rawStep = Math.max(1, spanUs / Math.max(1, targetTickCount)); + const bases = [1, 2, 5]; + const magnitude = Math.pow(10, Math.floor(Math.log10(rawStep))); + for (const scale of [1, 10]) { + for (const base of bases) { + const step = base * magnitude * scale; + if (step >= rawStep) { + return step; + } + } + } + return 10 * magnitude; +} + +function buildNiceTimeTicks(startUs, endUs, targetTickCount) { + const spanUs = Math.max(1, endUs - startUs); + const stepUs = chooseNiceTimeStep(spanUs, targetTickCount); + const firstTickUs = Math.ceil(startUs / stepUs) * stepUs; + const ticks = []; + for (let tickUs = firstTickUs; tickUs <= endUs; tickUs += stepUs) { + ticks.push(tickUs); + } + if (ticks.length === 0) { + const roundedStart = Math.floor(startUs / stepUs) * stepUs; + const roundedEnd = Math.ceil(endUs / stepUs) * stepUs; + if (roundedStart >= startUs && roundedStart <= endUs) { + ticks.push(roundedStart); + } + if (roundedEnd >= startUs && roundedEnd <= endUs && roundedEnd !== roundedStart) { + ticks.push(roundedEnd); + } + } + return ticks; +} + +function trimPath(path) { + if (!path) return ""; + const parts = String(path).split(/[\\/]/); + return parts[parts.length - 1] || path; +} + +function compareValues(a, b, desc) { + if (typeof a === "string" || typeof b === "string") { + const result = String(a || "").localeCompare(String(b || ""), undefined, { numeric: true, sensitivity: "base" }); + return desc ? -result : result; + } + const result = Number(a || 0) - Number(b || 0); + return desc ? -result : result; +} + +function escapeRegExp(s) { + return String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function highlightMatch(text, filterText) { + const source = String(text || ""); + if (!filterText) { + return escapeHtml(source); + } + const regex = new RegExp(`(${escapeRegExp(filterText)})`, "ig"); + return escapeHtml(source).replace(regex, '<mark class="memory-mark">$1</mark>'); +} + +function buildSummaryHtml(row, filterText = "") { + const top = highlightMatch(row.top_frame || row.summary || `Callstack ${row.callstack_id}`, filterText); + const second = row.secondary_frame ? `<div class="memory-summary-secondary">${highlightMatch(row.secondary_frame, filterText)}</div>` : ""; + const badges = []; + if (row.hidden_prefix_count > 0) { + badges.push(`<span class="memory-badge">skip ${escapeHtml(formatNum(row.hidden_prefix_count))}</span>`); + } + if (row.included_third_party_boundary) { + badges.push(`<span class="memory-badge">3p boundary</span>`); + } + const badgeHtml = badges.length ? `<div class="memory-summary-badges">${badges.join("")}</div>` : ""; + return `<div class="memory-summary"><div class="memory-summary-top-row"><div class="memory-summary-top">${top}</div>${badgeHtml}</div>${second}</div>`; +} + +function describeBucket(bucket) { + const min = Number(bucket.min_size || 0); + const max = Number(bucket.max_size || 0); + if (min === 0 && max === 0) { + return "0 bytes"; + } + if (min === max) { + return `${formatBytes(min)}`; + } + return `${formatBytes(min)} – ${formatBytes(max)}`; +} + +function formatBucketEdge(bucket) { + const max = Number(bucket.max_size || 0); + if (max === 0) { + return "0"; + } + return formatBytes(max); +} + +function sparklinePath(samples, width, height, valueIndex) { + if (!samples || samples.length === 0) { + return ""; + } + let minValue = Infinity; + let maxValue = -Infinity; + for (const sample of samples) { + const value = Number(sample[valueIndex] || 0); + minValue = Math.min(minValue, value); + maxValue = Math.max(maxValue, value); + } + if (!isFinite(minValue) || !isFinite(maxValue)) { + return ""; + } + if (minValue === maxValue) { + minValue -= 1; + maxValue += 1; + } + const count = samples.length; + const points = []; + for (let i = 0; i < count; ++i) { + const x = count > 1 ? (i / (count - 1)) * width : width * 0.5; + const norm = (Number(samples[i][valueIndex] || 0) - minValue) / (maxValue - minValue); + const y = height - norm * height; + points.push(`${i === 0 ? "M" : "L"}${x.toFixed(1)} ${y.toFixed(1)}`); + } + return points.join(" "); +} + +export class MemoryView { + constructor(model, containerEl) { + this.model = model; + this.container = containerEl; + this.loaded = false; + this.summary = null; + this.memoryTimeline = null; + this.leaks = []; + this.churn = []; + this.hot = []; + this.sizeHistogram = null; + this.histogramMetric = "count"; + this.callstackCache = new Map(); + this.selectedCallstackId = 0; + this.tableState = { + leaks: { sortKey: "live_bytes", desc: true, groupMode: "none", filterText: "" }, + churn: { sortKey: "churn_allocs", desc: true, groupMode: "none", filterText: "" }, + hot: { sortKey: "total_allocs", desc: true, groupMode: "none", filterText: "" }, + }; + this.loadStateFromUrl(); + this.buildLayout(); + } + + buildLayout() { + this.container.innerHTML = + `<div class="memory-view">` + + `<div class="memory-cards" id="memory-cards"></div>` + + `<div class="memory-panel">` + + `<div class="memory-panel-header">` + + `<div class="memory-panel-title">Memory timeline</div>` + + `<div class="memory-panel-subtitle" id="memory-timeline-meta"></div>` + + `</div>` + + `<div class="memory-chart-wrap" id="memory-chart-wrap">` + + `<svg class="memory-chart" id="memory-chart"></svg>` + + `</div>` + + `</div>` + + `<div class="memory-panel">` + + `<div class="memory-panel-header memory-panel-header-wrap">` + + `<div>` + + `<div class="memory-panel-title">Allocation size distribution</div>` + + `<div class="memory-panel-subtitle" id="memory-histogram-meta"></div>` + + `</div>` + + `<div class="memory-controls">` + + `<label>Metric <select id="memory-histogram-metric">` + + `<option value="count">Alloc count</option>` + + `<option value="bytes">Total bytes</option>` + + `</select></label>` + + `</div>` + + `</div>` + + `<div class="memory-chart-wrap" id="memory-histogram-wrap">` + + `<svg class="memory-chart memory-histogram" id="memory-histogram"></svg>` + + `</div>` + + `</div>` + + `<div class="memory-grid">` + + this.buildPanelMarkup("leaks", "Leaky callsites", "Top live allocation stacks", [ + ["live_bytes", "Live bytes"], + ["live_count", "Live allocs"], + ["summary", "Summary"], + ]) + + this.buildPanelMarkup("churn", "Churn", "Short-lived allocation sites", [ + ["churn_allocs", "Short-lived allocs"], + ["churn_bytes", "Churn bytes"], + ["mean_distance", "Avg distance"], + ["summary", "Summary"], + ]) + + this.buildPanelMarkup("hot", "Hot callsites", "Highest total allocation activity", [ + ["total_allocs", "Total allocs"], + ["total_bytes", "Total bytes"], + ["churn_allocs", "Churn allocs"], + ["summary", "Summary"], + ]) + + `<div class="memory-panel memory-callstack-panel">` + + `<div class="memory-panel-header"><div class="memory-panel-title">Callstack details</div><div class="memory-panel-subtitle" id="memory-callstack-meta">Select a row to inspect its frames</div></div>` + + `<div class="memory-callstack-body" id="memory-callstack-body"><div class="memory-empty">No callstack selected.</div></div>` + + `</div>` + + `</div>` + + `</div>`; + + this.cardsEl = this.container.querySelector("#memory-cards"); + this.chartWrapEl = this.container.querySelector("#memory-chart-wrap"); + this.chartEl = this.container.querySelector("#memory-chart"); + this.chartMetaEl = this.container.querySelector("#memory-timeline-meta"); + this.histogramWrapEl = this.container.querySelector("#memory-histogram-wrap"); + this.histogramEl = this.container.querySelector("#memory-histogram"); + this.histogramMetaEl = this.container.querySelector("#memory-histogram-meta"); + this.histogramMetricEl = this.container.querySelector("#memory-histogram-metric"); + this.callstackMetaEl = this.container.querySelector("#memory-callstack-meta"); + this.callstackBodyEl = this.container.querySelector("#memory-callstack-body"); + this.panelRefs = { + leaks: { + tbody: this.container.querySelector("#memory-leaks-body"), + sort: this.container.querySelector("#memory-leaks-sort"), + filter: this.container.querySelector("#memory-leaks-filter"), + clear: this.container.querySelector("#memory-leaks-clear"), + direction: this.container.querySelector("#memory-leaks-direction"), + group: this.container.querySelector("#memory-leaks-group"), + }, + churn: { + tbody: this.container.querySelector("#memory-churn-body"), + sort: this.container.querySelector("#memory-churn-sort"), + filter: this.container.querySelector("#memory-churn-filter"), + clear: this.container.querySelector("#memory-churn-clear"), + direction: this.container.querySelector("#memory-churn-direction"), + group: this.container.querySelector("#memory-churn-group"), + }, + hot: { + tbody: this.container.querySelector("#memory-hot-body"), + sort: this.container.querySelector("#memory-hot-sort"), + filter: this.container.querySelector("#memory-hot-filter"), + clear: this.container.querySelector("#memory-hot-clear"), + direction: this.container.querySelector("#memory-hot-direction"), + group: this.container.querySelector("#memory-hot-group"), + }, + }; + + this.resizeObserver = new ResizeObserver(() => { + if (this.loaded) { + this.renderTimeline(); + this.renderSizeHistogram(); + } + }); + this.resizeObserver.observe(this.chartWrapEl); + this.resizeObserver.observe(this.histogramWrapEl); + + this.histogramMetricEl.value = this.histogramMetric; + this.histogramMetricEl.addEventListener("change", () => { + this.histogramMetric = this.histogramMetricEl.value; + this.saveStateToUrl(); + this.renderSizeHistogram(); + }); + + for (const [name, refs] of Object.entries(this.panelRefs)) { + refs.sort.value = this.tableState[name].sortKey; + refs.group.value = this.tableState[name].groupMode; + refs.filter.value = this.tableState[name].filterText; + refs.sort.addEventListener("change", () => { + this.tableState[name].sortKey = refs.sort.value; + this.tableState[name].desc = refs.sort.value !== "mean_distance"; + this.updateDirectionButton(name); + this.saveStateToUrl(); + this.renderTableByName(name); + }); + refs.direction.addEventListener("click", () => { + this.tableState[name].desc = !this.tableState[name].desc; + this.updateDirectionButton(name); + this.saveStateToUrl(); + this.renderTableByName(name); + }); + refs.filter.addEventListener("input", () => { + this.tableState[name].filterText = refs.filter.value; + this.updateFilterButton(name); + this.saveStateToUrl(); + this.renderTableByName(name); + }); + refs.clear.addEventListener("click", () => { + refs.filter.value = ""; + this.tableState[name].filterText = ""; + this.updateFilterButton(name); + this.saveStateToUrl(); + this.renderTableByName(name); + refs.filter.focus(); + }); + refs.group.addEventListener("change", () => { + this.tableState[name].groupMode = refs.group.value; + this.saveStateToUrl(); + this.renderTableByName(name); + }); + this.updateDirectionButton(name); + this.updateFilterButton(name); + } + + this.container.addEventListener("keydown", (e) => { + if (e.key !== "/" || e.defaultPrevented) { + return; + } + const target = e.target; + if (target && (target.tagName === "INPUT" || target.tagName === "TEXTAREA" || target.tagName === "SELECT" || target.isContentEditable)) { + return; + } + e.preventDefault(); + const activeView = this.container.closest(".view"); + if (activeView && activeView.hidden) { + return; + } + const firstFilter = this.panelRefs.leaks.filter; + if (firstFilter) { + firstFilter.focus(); + firstFilter.select(); + } + }); + this.container.tabIndex = -1; + this.container.dataset.memoryView = "true"; + } + + buildPanelMarkup(name, title, subtitle, sortOptions) { + const sortHtml = sortOptions.map(([value, label]) => `<option value="${value}">${escapeHtml(label)}</option>`).join(""); + return ` + <div class="memory-panel"> + <div class="memory-panel-header memory-panel-header-wrap"> + <div> + <div class="memory-panel-title">${escapeHtml(title)}</div> + <div class="memory-panel-subtitle">${escapeHtml(subtitle)}</div> + </div> + <div class="memory-controls"> + <label>Filter <input type="text" id="memory-${name}-filter" class="memory-filter-input" placeholder="filter entries..."></label> + <button type="button" class="memory-clear-btn" id="memory-${name}-clear">Clear</button> + <label>Sort <select id="memory-${name}-sort">${sortHtml}</select></label> + <button type="button" class="memory-direction-btn" id="memory-${name}-direction"></button> + <label>Group <select id="memory-${name}-group"> + <option value="none">None</option> + <option value="top_frame">Top frame</option> + <option value="prefix">Trimmed prefix</option> + </select></label> + </div> + </div> + <div class="memory-table-wrap"><table class="memory-table"><tbody id="memory-${name}-body"></tbody></table></div> + </div>`; + } + + loadStateFromUrl() { + const params = new URLSearchParams(window.location.search); + const histogramMetric = params.get("mem_hist_metric"); + if (histogramMetric === "count" || histogramMetric === "bytes") { + this.histogramMetric = histogramMetric; + } + for (const [name, state] of Object.entries(this.tableState)) { + const sortKey = params.get(`mem_${name}_sort`); + const groupMode = params.get(`mem_${name}_group`); + const dir = params.get(`mem_${name}_dir`); + const filterText = params.get(`mem_${name}_filter`); + if (sortKey) { + state.sortKey = sortKey; + } + if (groupMode) { + state.groupMode = groupMode; + } + if (filterText) { + state.filterText = filterText; + } + if (dir === "asc") { + state.desc = false; + } + else if (dir === "desc") { + state.desc = true; + } + } + } + + saveStateToUrl() { + const url = new URL(window.location.href); + url.searchParams.set("mem_hist_metric", this.histogramMetric); + for (const [name, state] of Object.entries(this.tableState)) { + url.searchParams.set(`mem_${name}_sort`, state.sortKey); + url.searchParams.set(`mem_${name}_group`, state.groupMode); + url.searchParams.set(`mem_${name}_dir`, state.desc ? "desc" : "asc"); + if (state.filterText) { + url.searchParams.set(`mem_${name}_filter`, state.filterText); + } + else { + url.searchParams.delete(`mem_${name}_filter`); + } + } + window.history.replaceState({}, "", url); + } + + updateFilterButton(name) { + const refs = this.panelRefs[name]; + const hasText = !!this.tableState[name].filterText; + refs.clear.disabled = !hasText; + refs.clear.title = hasText ? "Clear filter" : "Filter is empty"; + } + + updateDirectionButton(name) { + const refs = this.panelRefs[name]; + const desc = this.tableState[name].desc; + refs.direction.textContent = desc ? "↓ Desc" : "↑ Asc"; + refs.direction.setAttribute("aria-label", desc ? "Sort descending" : "Sort ascending"); + refs.direction.title = desc ? "Sorting descending" : "Sorting ascending"; + } + + async ensureLoaded() { + if (this.loaded) return; + this.loaded = true; + await this.refresh(); + } + + async refresh() { + this.renderLoading(); + try { + const [summary, memoryTimeline, leakResponse, churnResponse, sizeHistogram] = await Promise.all([ + getAllocSummary(), + getMemoryTimeline({ maxSamples: 1200 }), + getCallstackStats(100), + getChurnStats(200), + getAllocSizeHistogram(), + ]); + this.summary = summary; + this.memoryTimeline = memoryTimeline; + this.leaks = (leakResponse && leakResponse.stats) || []; + this.churn = (churnResponse && churnResponse.stats) || []; + this.sizeHistogram = sizeHistogram; + this.hot = this.churn.slice().sort((a, b) => { + if (b.total_allocs !== a.total_allocs) return b.total_allocs - a.total_allocs; + return b.total_bytes - a.total_bytes; + }).slice(0, 100); + this.render(); + } catch (e) { + this.cardsEl.innerHTML = ""; + this.chartEl.innerHTML = ""; + this.chartMetaEl.textContent = ""; + this.histogramEl.innerHTML = ""; + this.histogramMetaEl.textContent = ""; + for (const refs of Object.values(this.panelRefs)) { + refs.tbody.innerHTML = `<tr><td class="memory-empty">Failed to load memory data: ${escapeHtml(e.message)}</td></tr>`; + } + this.callstackBodyEl.innerHTML = `<div class="memory-empty">Failed to load memory data.</div>`; + } + } + + renderLoading() { + this.cardsEl.innerHTML = `<div class="memory-card"><div class="memory-card-label">Loading</div><div class="memory-card-value">Memory analysis…</div></div>`; + this.chartEl.innerHTML = ""; + this.chartMetaEl.textContent = ""; + this.histogramEl.innerHTML = ""; + this.histogramMetaEl.textContent = "Loading…"; + for (const refs of Object.values(this.panelRefs)) { + refs.tbody.innerHTML = `<tr><td class="memory-empty">Loading…</td></tr>`; + } + } + + render() { + this.renderCards(); + this.renderTimeline(); + this.renderSizeHistogram(); + this.renderTableByName("leaks"); + this.renderTableByName("churn"); + this.renderTableByName("hot"); + } + + renderCards() { + const s = this.summary || {}; + this.cardsEl.innerHTML = [ + this.formatCard("Peak memory", formatBytes(s.peak_bytes)), + this.formatCard("End memory", formatBytes(s.end_bytes)), + this.formatCard("Live allocations", formatNum(s.live_allocations)), + this.formatCard("Total allocs", formatNum(s.total_allocs)), + this.formatCard("Total frees", formatNum(s.total_frees)), + this.formatCard("Reallocs", formatNum((s.total_realloc_allocs || 0) + (s.total_realloc_frees || 0))), + ].join(""); + } + + formatCard(label, value) { + return `<div class="memory-card"><div class="memory-card-label">${escapeHtml(label)}</div><div class="memory-card-value">${escapeHtml(value)}</div></div>`; + } + + renderTimeline() { + const samples = (this.memoryTimeline && this.memoryTimeline.samples) || []; + if (samples.length === 0) { + this.chartMetaEl.textContent = "No memory timeline samples"; + this.chartEl.innerHTML = `<text x="500" y="110" text-anchor="middle" class="memory-chart-text">No memory timeline samples</text>`; + return; + } + + const width = Math.max(320, Math.floor(this.chartWrapEl.getBoundingClientRect().width) - 24); + const height = 220; + const padLeft = 56; + const padRight = 12; + const padTop = 12; + const padBottom = 22; + const chartWidth = width - padLeft - padRight; + const chartHeight = height - padTop - padBottom; + let minValue = Infinity; + let maxValue = -Infinity; + for (const sample of samples) { + minValue = Math.min(minValue, Number(sample[1] || 0)); + maxValue = Math.max(maxValue, Number(sample[1] || 0)); + } + if (minValue === maxValue) { + minValue -= 1; + maxValue += 1; + } + + const xAt = (index) => samples.length > 1 ? (padLeft + (index / (samples.length - 1)) * chartWidth) : (padLeft + chartWidth * 0.5); + const yAt = (value) => { + const norm = (Number(value || 0) - minValue) / Math.max(1, maxValue - minValue); + return padTop + chartHeight - norm * chartHeight; + }; + const path = samples.map((sample, index) => `${index === 0 ? "M" : "L"}${xAt(index).toFixed(1)} ${yAt(sample[1]).toFixed(1)}`).join(" "); + + const grid = []; + for (let i = 0; i <= 4; ++i) { + const y = padTop + chartHeight * i / 4; + const value = maxValue + (minValue - maxValue) * i / 4; + grid.push(`<line x1="${padLeft}" y1="${y}" x2="${padLeft + chartWidth}" y2="${y}" class="memory-chart-grid"/>`); + grid.push(`<text x="${padLeft - 6}" y="${y + 4}" text-anchor="end" class="memory-chart-axis">${escapeHtml(formatBytes(value))}</text>`); + } + + const startUs = Number(samples[0][0] || 0); + const endUs = Number(samples[samples.length - 1][0] || 0); + const spanUs = Math.max(1, endUs - startUs); + const targetTickCount = Math.max(3, Math.min(8, Math.floor(chartWidth / 110))); + const ticks = buildNiceTimeTicks(startUs, endUs, targetTickCount); + for (const timeUs of ticks) { + const t = spanUs > 0 ? ((timeUs - startUs) / spanUs) : 0; + const x = padLeft + chartWidth * t; + grid.push(`<line x1="${x}" y1="${padTop}" x2="${x}" y2="${padTop + chartHeight}" class="memory-chart-grid memory-chart-grid-vert"/>`); + grid.push(`<text x="${x}" y="${height - 4}" text-anchor="middle" class="memory-chart-axis">${escapeHtml(formatTimeAxis(timeUs))}</text>`); + } + + const durationUs = (this.model.session.trace_end_us || 0) - (this.model.session.trace_start_us || 0); + this.chartMetaEl.textContent = `${formatNum(samples.length)} samples across ${(durationUs / 1_000_000).toFixed(2)} s`; + this.chartEl.setAttribute("viewBox", `0 0 ${width} ${height}`); + this.chartEl.setAttribute("preserveAspectRatio", "xMinYMin meet"); + this.chartEl.innerHTML = + `<rect x="0" y="0" width="${width}" height="${height}" class="memory-chart-bg"/>` + + grid.join("") + + `<path d="${path}" class="memory-chart-line"/>`; + } + + renderSizeHistogram() { + const buckets = (this.sizeHistogram && this.sizeHistogram.buckets) || []; + if (buckets.length === 0) { + this.histogramMetaEl.textContent = "No allocations recorded"; + this.histogramEl.innerHTML = `<text x="500" y="110" text-anchor="middle" class="memory-chart-text">No allocations recorded</text>`; + return; + } + + const metric = this.histogramMetric === "bytes" ? "bytes" : "count"; + const metricLabel = metric === "bytes" ? "Total bytes" : "Alloc count"; + const valueFor = (b) => Number((metric === "bytes" ? b.bytes : b.count) || 0); + const formatValue = metric === "bytes" ? formatBytes : formatNum; + + let maxValue = 0; + let totalValue = 0; + for (const bucket of buckets) { + const v = valueFor(bucket); + if (v > maxValue) maxValue = v; + totalValue += v; + } + if (maxValue === 0) { + maxValue = 1; + } + + const width = Math.max(320, Math.floor(this.histogramWrapEl.getBoundingClientRect().width) - 24); + const height = 240; + const padLeft = 64; + const padRight = 12; + const padTop = 12; + const padBottom = 42; + const chartWidth = width - padLeft - padRight; + const chartHeight = height - padTop - padBottom; + + const bucketCount = buckets.length; + const slotWidth = chartWidth / bucketCount; + const barGap = Math.max(1, Math.min(4, slotWidth * 0.15)); + const barWidth = Math.max(1, slotWidth - barGap); + + const parts = []; + parts.push(`<rect x="0" y="0" width="${width}" height="${height}" class="memory-chart-bg"/>`); + + // Horizontal grid + y-axis labels at 0, 25, 50, 75, 100% of max. + for (let i = 0; i <= 4; ++i) { + const y = padTop + chartHeight * i / 4; + const value = maxValue * (1 - i / 4); + parts.push(`<line x1="${padLeft}" y1="${y}" x2="${padLeft + chartWidth}" y2="${y}" class="memory-chart-grid"/>`); + parts.push(`<text x="${padLeft - 6}" y="${y + 4}" text-anchor="end" class="memory-chart-axis">${escapeHtml(formatValue(value))}</text>`); + } + + // Bars. X-axis labels are drawn for a subset of buckets to avoid overlap. + const labelStride = Math.max(1, Math.ceil(bucketCount / Math.max(3, Math.floor(chartWidth / 64)))); + for (let i = 0; i < bucketCount; ++i) { + const bucket = buckets[i]; + const value = valueFor(bucket); + const barHeight = (value / maxValue) * chartHeight; + const x = padLeft + i * slotWidth + barGap / 2; + const y = padTop + chartHeight - barHeight; + const label = describeBucket(bucket); + const tooltip = `${label}\n${metricLabel}: ${formatValue(value)}\nAlloc count: ${formatNum(bucket.count)}\nTotal bytes: ${formatBytes(bucket.bytes)}`; + parts.push( + `<rect x="${x.toFixed(1)}" y="${y.toFixed(1)}" width="${barWidth.toFixed(1)}" height="${Math.max(0, barHeight).toFixed(1)}" class="memory-histogram-bar"><title>${escapeHtml(tooltip)}</title></rect>` + ); + if (i % labelStride === 0 || i === bucketCount - 1) { + const tickX = padLeft + i * slotWidth + slotWidth / 2; + parts.push( + `<text x="${tickX.toFixed(1)}" y="${(padTop + chartHeight + 14).toFixed(1)}" text-anchor="middle" class="memory-chart-axis">${escapeHtml(formatBucketEdge(bucket))}</text>` + ); + } + } + + // Axis title for x. + parts.push( + `<text x="${(padLeft + chartWidth / 2).toFixed(1)}" y="${(height - 4).toFixed(1)}" text-anchor="middle" class="memory-chart-axis">Allocation size (power-of-two buckets)</text>` + ); + + const summaryTotalCount = Number((this.sizeHistogram && this.sizeHistogram.total_count) || 0); + const summaryTotalBytes = Number((this.sizeHistogram && this.sizeHistogram.total_bytes) || 0); + this.histogramMetaEl.textContent = `${formatNum(summaryTotalCount)} allocations, ${formatBytes(summaryTotalBytes)} total across ${bucketCount} bucket${bucketCount === 1 ? "" : "s"}`; + + this.histogramEl.setAttribute("viewBox", `0 0 ${width} ${height}`); + this.histogramEl.setAttribute("preserveAspectRatio", "xMinYMin meet"); + this.histogramEl.innerHTML = parts.join(""); + } + + getRowsForTable(name) { + if (name === "leaks") return this.leaks.slice(0, 100); + if (name === "churn") return this.churn.slice(0, 100); + return this.hot.slice(0, 100); + } + + renderTableByName(name) { + const refs = this.panelRefs[name]; + const state = this.tableState[name]; + let rows = this.getRowsForTable(name).slice(); + const filterText = state.filterText.trim().toLowerCase(); + if (filterText) { + rows = rows.filter((row) => { + const haystack = [ + row.summary, + row.top_frame, + row.secondary_frame, + row.group_key, + `callstack ${row.callstack_id}`, + ].join("\n").toLowerCase(); + return haystack.includes(filterText); + }); + } + rows.sort((a, b) => compareValues(a[state.sortKey], b[state.sortKey], state.desc)); + + if (!rows.length) { + refs.tbody.innerHTML = `<tr><td class="memory-empty">No data available.</td></tr>`; + return; + } + + const parts = []; + let currentGroup = null; + for (let index = 0; index < rows.length; ++index) { + const row = rows[index]; + const groupKey = state.groupMode === "top_frame" ? (row.top_frame || "(unknown)") : + (state.groupMode === "prefix" ? (row.group_key || row.top_frame || "(unknown)") : null); + if (groupKey !== null && groupKey !== currentGroup) { + currentGroup = groupKey; + parts.push(`<tr class="memory-group-row"><td colspan="5">${escapeHtml(groupKey)}</td></tr>`); + } + parts.push(this.renderDataRow(name, row, index)); + } + refs.tbody.innerHTML = parts.join(""); + for (const tr of refs.tbody.querySelectorAll("tr[data-callstack-id]")) { + tr.addEventListener("click", () => { + const callstackId = Number(tr.dataset.callstackId); + this.selectCallstack(callstackId); + for (const rowEl of this.container.querySelectorAll("tr[data-callstack-id]")) { + rowEl.classList.toggle("selected", Number(rowEl.dataset.callstackId) === callstackId); + } + }); + } + } + + renderDataRow(name, row, index) { + if (name === "leaks") { + return `<tr data-callstack-id="${row.callstack_id}">` + + `<td class="num">${index + 1}</td>` + + `<td class="num">${escapeHtml(formatBytes(row.live_bytes))}</td>` + + `<td class="num">${escapeHtml(formatNum(row.live_count))}</td>` + + `<td>${buildSummaryHtml(row)}</td>` + + `</tr>`; + } + if (name === "churn") { + return `<tr data-callstack-id="${row.callstack_id}">` + + `<td class="num">${index + 1}</td>` + + `<td class="num">${escapeHtml(formatNum(row.churn_allocs))}</td>` + + `<td class="num">${escapeHtml(formatBytes(row.churn_bytes))}</td>` + + `<td class="num">${escapeHtml(formatDistance(row.mean_distance))}</td>` + + `<td>${buildSummaryHtml(row)}</td>` + + `</tr>`; + } + return `<tr data-callstack-id="${row.callstack_id}">` + + `<td class="num">${index + 1}</td>` + + `<td class="num">${escapeHtml(formatNum(row.total_allocs))}</td>` + + `<td class="num">${escapeHtml(formatBytes(row.total_bytes))}</td>` + + `<td class="num">${escapeHtml(formatNum(row.churn_allocs))}</td>` + + `<td>${buildSummaryHtml(row)}</td>` + + `</tr>`; + } + + async selectCallstack(callstackId) { + this.selectedCallstackId = callstackId; + this.callstackMetaEl.textContent = `Callstack ${callstackId}`; + this.callstackBodyEl.innerHTML = `<div class="memory-empty">Loading callstack ${callstackId}…</div>`; + try { + let callstack = this.callstackCache.get(callstackId); + if (!callstack) { + callstack = await getCallstack(callstackId); + this.callstackCache.set(callstackId, callstack); + } + const frames = callstack.frames || []; + if (!frames.length) { + this.callstackBodyEl.innerHTML = `<div class="memory-empty">No frames recorded for this callstack.</div>`; + return; + } + const notes = []; + if (callstack.hidden_prefix_count > 0) { + let note = `Skipped ${formatNum(callstack.hidden_prefix_count)} leading frame(s)`; + if (callstack.included_third_party_boundary) { + note += "; kept boundary third-party callsite"; + } + notes.push(`<div class="memory-empty">${escapeHtml(note)}.</div>`); + } + const items = []; + for (let i = 0; i < frames.length; ++i) { + const frame = frames[i]; + const display = frame.display || frame.address || "(unknown frame)"; + const extra = frame.module_path ? ` <span class="memory-frame-path">${escapeHtml(trimPath(frame.module_path))}</span>` : ""; + items.push(`<li><span class="memory-frame-index">#${frame.index ?? i}</span> <span class="memory-frame-display">${escapeHtml(display)}</span>${extra}</li>`); + } + this.callstackBodyEl.innerHTML = `${notes.join("")}<ol class="memory-callstack-list">${items.join("")}</ol>`; + } catch (e) { + this.callstackBodyEl.innerHTML = `<div class="memory-empty">Failed to load callstack ${callstackId}: ${escapeHtml(e.message)}</div>`; + } + } +} diff --git a/src/zen/frontend/html/stats.js b/src/zen/frontend/html/stats.js new file mode 100644 index 000000000..741ad7ef9 --- /dev/null +++ b/src/zen/frontend/html/stats.js @@ -0,0 +1,95 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// Sortable stats table view. + +const US_PER_MS = 1000; + +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, (c) => ({ + "&": "&", + "<": "<", + ">": ">", + "\"": """, + "'": "'", + }[c])); +} + +export class StatsView { + constructor(tbody, headerRow, model, onSelect) { + this.tbody = tbody; + this.headerRow = headerRow; + this.stats = model.scopeStats.slice(); + this.onSelect = onSelect; + this.sortKey = "count"; + this.sortAsc = false; + this.selectedName = null; + + for (const th of headerRow.querySelectorAll("th[data-sort]")) { + th.addEventListener("click", () => this.handleSort(th.dataset.sort)); + } + this.render(); + } + + handleSort(key) { + if (this.sortKey === key) { + this.sortAsc = !this.sortAsc; + } else { + this.sortKey = key; + this.sortAsc = key === "name"; + } + this.render(); + } + + selectByName(name) { + this.selectedName = name; + for (const tr of this.tbody.querySelectorAll("tr")) { + tr.classList.toggle("selected", tr.dataset.name === name); + if (tr.dataset.name === name) { + tr.scrollIntoView({ block: "nearest" }); + } + } + } + + render() { + const key = this.sortKey; + const asc = this.sortAsc; + this.stats.sort((a, b) => { + const av = a[key]; + const bv = b[key]; + if (typeof av === "string") { + return asc ? av.localeCompare(bv) : bv.localeCompare(av); + } + return asc ? av - bv : bv - av; + }); + + for (const th of this.headerRow.querySelectorAll("th[data-sort]")) { + th.classList.toggle("sorted", th.dataset.sort === key); + th.classList.toggle("asc", th.dataset.sort === key && asc); + } + + const rows = []; + for (const stat of this.stats) { + const selected = stat.name === this.selectedName ? " class=\"selected\"" : ""; + rows.push( + `<tr data-name="${escapeHtml(stat.name)}"${selected}>` + + `<td>${escapeHtml(stat.name)}</td>` + + `<td class="num">${stat.count.toLocaleString()}</td>` + + `<td class="num">${(stat.min_us / US_PER_MS).toFixed(3)}</td>` + + `<td class="num">${(stat.mean_us / US_PER_MS).toFixed(3)}</td>` + + `<td class="num">${(stat.max_us / US_PER_MS).toFixed(3)}</td>` + + `<td class="num">${(stat.stdev_us / US_PER_MS).toFixed(3)}</td>` + + `</tr>`, + ); + } + this.tbody.innerHTML = rows.join(""); + + for (const tr of this.tbody.querySelectorAll("tr")) { + tr.addEventListener("click", () => { + const name = tr.dataset.name; + this.selectByName(name); + if (this.onSelect) { + this.onSelect(name); + } + }); + } + } +} diff --git a/src/zen/frontend/html/timeline.js b/src/zen/frontend/html/timeline.js new file mode 100644 index 000000000..f463a8418 --- /dev/null +++ b/src/zen/frontend/html/timeline.js @@ -0,0 +1,973 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// Canvas-drawn flame graph with per-thread swimlanes, pan+zoom, hover +// tooltip, click-to-select and scope-name highlighting. + +const HEADER_H = 18; // thread name row height +const DEPTH_H = 16; // scope lane row height +const MAX_DRAWN_DEPTH = 32; +const MIN_RECT_W = 1.5; // don't draw narrower than this (px) +const RULER_H = 20; +const THREAD_GAP = 6; +const PADDING_X = 0; +const REGION_LANE_H = 18; // region band row height +const REGION_HEADER_H = 16; // category header row height +const REGIONS_GAP = 6; // gap between the region rack and the first thread + +// Scope colors: golden-angle hue rotation keyed on NameId so the same scope +// always renders in the same color across zoom levels. +function scopeFillColor(nameId) { + const hue = ((nameId * 137.508) % 360 + 360) % 360; + return `hsl(${hue.toFixed(0)}, 55%, 42%)`; +} + +function scopeHighlightColor(nameId) { + const hue = ((nameId * 137.508) % 360 + 360) % 360; + return `hsl(${hue.toFixed(0)}, 80%, 60%)`; +} + +function stringHash(s) { + let h = 0; + for (let i = 0; i < s.length; i++) { + h = ((h << 5) - h + s.charCodeAt(i)) | 0; + } + return h >>> 0; +} + +// Desaturated palette for regions so they don't compete visually with the +// colourful CPU scopes below them. +function regionFillColor(name) { + const hue = (stringHash(name || "region") * 2.3) % 360; + return `hsla(${hue.toFixed(0)}, 35%, 55%, 0.55)`; +} + +function formatTime(us) { + if (us < 1000) { + return `${us} µs`; + } + if (us < 1_000_000) { + return `${(us / 1000).toFixed(3)} ms`; + } + return `${(us / 1_000_000).toFixed(3)} s`; +} + +function formatRange(startUs, endUs) { + return `${formatTime(startUs)} → ${formatTime(endUs)} (${formatTime(endUs - startUs)})`; +} + +export class Timeline { + constructor(opts) { + this.canvas = opts.canvas; + this.tooltip = opts.tooltip; + this.selectionEl = opts.selectionEl; + this.viewportInfoEl = opts.viewportInfoEl; + this.zoomResetBtn = opts.zoomResetBtn; + this.model = opts.model; + this.onScopeSelect = opts.onScopeSelect || (() => {}); + + this.ctx = this.canvas.getContext("2d"); + this.dpr = Math.max(1, window.devicePixelRatio || 1); + + this.bookmarks = (this.model.bookmarks || []).slice().sort((a, b) => a.time_us - b.time_us); + this.bookmarksVisible = true; + this.regionCategories = (this.model.regionCategories || []).filter(c => c.lane_count > 0); + // All categories enabled by default; renderRegionCategories() calls + // setEnabledRegionCategories() shortly after construction. + this.enabledRegionCategories = new Set(this.regionCategories.map((_, i) => i)); + this.recomputeRegionsBlockH(); + + // Per-thread timelines keyed by threadId; each entry is an object + // { scopes, perDepth } where scopes is an array of tuples + // [beginUs, durationUs, nameId, depth, mergeCount?]. + this.timelines = new Map(); + // Set of threadIds the user wants visible. + this.enabledThreads = new Set(); + + // Viewport-driven fetch state. + this.lodEnabled = true; // when false, always request LOD 0 (raw) + this.fetchThrottled = false; + this.fetchPending = false; + this.fetchThrottleTimer = null; + this.abortControllers = new Map(); // threadId → AbortController + this.fetchSeq = new Map(); // threadId → monotonic fetch sequence id + this.cachedRanges = new Map(); // threadId → { startUs, endUs, resolution } + // Lookup helpers. + this.threadMeta = new Map(); // threadId → { name, sortHint, scopeCount } + for (const t of this.model.threads) { + this.threadMeta.set(t.thread_id, t); + } + + // Viewport state — time units throughout are microseconds from trace start. + this.traceStart = 0; + this.traceEnd = Math.max(1, this.model.session.trace_end_us || 0); + if (this.traceEnd <= this.traceStart) { + this.traceEnd = this.traceStart + 1000; + } + this.startUs = this.traceStart; + const maxInitialUs = 60_000_000; // cap initial view to 60 seconds + this.endUs = (this.traceEnd - this.traceStart > maxInitialUs) + ? this.traceStart + maxInitialUs + : this.traceEnd; + + // Vertical scroll offset in canvas pixels (0 = first thread flush + // against the ruler). Updated by both drag-pan and shift-wheel. + this.scrollY = 0; + this.maxScrollY = 0; + + // Hit-test rects computed during the last draw. + this.hits = []; + this.selectedId = null; + this.highlightName = null; + + // Pan state + this.panStartX = 0; + this.panStartY = 0; + this.panStartUs = 0; + this.panStartScrollY = 0; + this.panning = false; + this.panMoved = false; + + this.resizeObserver = new ResizeObserver(() => this.requestDraw()); + this.resizeObserver.observe(this.canvas); + + this.canvas.addEventListener("mousedown", (e) => this.onMouseDown(e)); + window.addEventListener("mousemove", (e) => this.onMouseMove(e)); + window.addEventListener("mouseup", (e) => this.onMouseUp(e)); + this.canvas.addEventListener("wheel", (e) => this.onWheel(e), { passive: false }); + this.canvas.addEventListener("mouseleave", () => this.hideTooltip()); + this.zoomResetBtn.addEventListener("click", () => this.resetView()); + + this.drawPending = false; + } + + setBookmarksVisible(visible) { + this.bookmarksVisible = visible; + this.requestDraw(); + } + + setEnabledRegionCategories(indices) { + this.enabledRegionCategories = indices instanceof Set ? indices : new Set(indices); + this.recomputeRegionsBlockH(); + this.requestDraw(); + } + + recomputeRegionsBlockH() { + this.regionsBlockH = 0; + for (let i = 0; i < this.regionCategories.length; i++) { + if (!this.enabledRegionCategories || !this.enabledRegionCategories.has(i)) continue; + const cat = this.regionCategories[i]; + this.regionsBlockH += REGION_HEADER_H + cat.lane_count * REGION_LANE_H; + } + if (this.regionsBlockH > 0) { + this.regionsBlockH += REGIONS_GAP; + } + } + + setLodEnabled(enabled) { + this.lodEnabled = enabled; + // Invalidate all caches so the next fetch uses the new setting. + this.cachedRanges.clear(); + this.scheduleFetch(); + } + + setEnabledThreads(ids) { + this.enabledThreads = new Set(ids); + this.scheduleFetch(); + this.requestDraw(); + } + + setHighlightName(name) { + this.highlightName = name || null; + this.requestDraw(); + } + + jumpToScopeName(name) { + // Find the first scope with the given name and frame it. + this.setHighlightName(name); + for (const threadId of this.enabledThreads) { + const timeline = this.timelines.get(threadId); + if (!timeline) continue; + for (const s of timeline.scopes) { + const nameId = s[2]; + if (this.model.scopeNames[nameId] !== name) continue; + const beginUs = s[0]; + const durationUs = s[1]; + const pad = Math.max(durationUs * 3, 500); + this.startUs = Math.max(0, beginUs - pad); + this.endUs = beginUs + durationUs + pad; + this.selectScope({ threadId, tuple: s }); + this.scheduleFetch(); + this.requestDraw(); + return; + } + } + } + + resetView() { + this.startUs = this.traceStart; + this.endUs = this.traceEnd; + this.scrollY = 0; + this.scheduleFetch(); + this.requestDraw(); + } + + // ── Viewport-driven fetch engine ────────────────────────────────── + + computeResolution() { + const w = this.width || this.canvas.getBoundingClientRect().width || 1; + // The resolution tells the server the minimum renderable scope duration. + // A scope must be at least MIN_RECT_W pixels wide to be drawn, so the + // threshold is usPerPixel * MIN_RECT_W, not just usPerPixel. This + // selects a coarser LOD that merges across gaps smaller than one + // renderable unit, preventing empty holes in the timeline. + return Math.ceil((this.endUs - this.startUs) / w * MIN_RECT_W); + } + + computeFetchWindow() { + const range = this.endUs - this.startUs; + const margin = range * 0.5; + return { + startUs: Math.max(0, Math.floor(this.startUs - margin)), + endUs: Math.ceil(this.endUs + margin), + }; + } + + // Map a resolution to the LOD index the server would select. + // Mirrors the server's selection: finest LOD where ResolutionUs >= res. + // Returns -1 for LOD 0 (raw), 0–4 for LOD 1–5. + lodForResolution(res) { + if (!this.lodEnabled || res <= 0) return -1; + const levels = [100, 1000, 8000, 40000, 200000]; + for (let i = 0; i < levels.length; i++) { + if (levels[i] >= res) return i; + } + return levels.length - 1; // coarsest + } + + needsRefetch(threadId) { + const cached = this.cachedRanges.get(threadId); + if (!cached) return true; + const currentRes = this.computeResolution(); + // Re-fetch when the LOD level would change — this catches the exact + // boundary crossing and prevents jarring LOD transitions during pan. + if (this.lodForResolution(cached.resolution) !== this.lodForResolution(currentRes)) return true; + // Re-fetch when the viewport nears the edge of the cached range. + const margin = (cached.endUs - cached.startUs) * 0.25; + if (this.startUs < cached.startUs + margin) return true; + if (this.endUs > cached.endUs - margin) return true; + return false; + } + + checkViewportFetch() { + for (const id of this.enabledThreads) { + if (this.needsRefetch(id)) { + this.scheduleFetch(); + return; + } + } + } + + scheduleFetch() { + // Leading+trailing throttle: fires immediately on the first call, + // then suppresses further calls for 150ms. If any calls arrived + // during the suppression window, one trailing fetch fires at the end. + // This keeps data flowing during continuous pan/zoom without flooding. + this.fetchPending = true; + if (this.fetchThrottled) return; + this.fetchThrottled = true; + this.fetchPending = false; + this.fetchViewport(); + this.fetchThrottleTimer = setTimeout(() => { + this.fetchThrottled = false; + if (this.fetchPending) { + this.fetchPending = false; + this.scheduleFetch(); + } + }, 150); + } + + async fetchViewport() { + const { startUs, endUs } = this.computeFetchWindow(); + const currentRes = this.lodEnabled ? this.computeResolution() : 0; + + const threadIds = []; + let resolution = currentRes; + for (const threadId of this.enabledThreads) { + if (!this.needsRefetch(threadId)) continue; + + // If the LOD level hasn't changed, reuse the cached resolution so + // the server selects the same LOD. This prevents a pan-triggered + // refetch from accidentally switching LOD levels due to minor + // resolution drift within the same LOD band. + const cached = this.cachedRanges.get(threadId); + if (cached && currentRes > 0 && + this.lodForResolution(cached.resolution) === this.lodForResolution(currentRes)) { + resolution = cached.resolution; + } + + threadIds.push(threadId); + } + if (threadIds.length === 0) return; + + // Cancel any in-flight batch request. + if (this.batchAbort) this.batchAbort.abort(); + const controller = new AbortController(); + this.batchAbort = controller; + + const seq = (this.batchSeq || 0) + 1; + this.batchSeq = seq; + + try { + const { getTimelineBatch } = await import("./api.js"); + const result = await getTimelineBatch(threadIds, startUs, endUs, 0, resolution, { signal: controller.signal }); + // Discard stale responses. + if (this.batchSeq !== seq) return; + + for (const threadId of threadIds) { + const entry = result[String(threadId)]; + const scopes = entry ? (entry.scopes || []) : []; + + const perDepth = []; + for (let i = 0; i < scopes.length; i++) { + const d = scopes[i][3]; + while (perDepth.length <= d) perDepth.push([]); + perDepth[d].push(i); + } + + this.timelines.set(threadId, { scopes, perDepth }); + this.cachedRanges.set(threadId, { startUs, endUs, resolution }); + } + + this.batchAbort = null; + this.requestDraw(); + } catch (e) { + if (e.name === "AbortError") return; + console.error(`failed to load timeline batch: ${e.message}`); + this.batchAbort = null; + } + } + + requestDraw() { + if (this.drawPending) return; + this.drawPending = true; + requestAnimationFrame(() => { + this.drawPending = false; + this.draw(); + }); + } + + resizeBackingStore() { + const rect = this.canvas.getBoundingClientRect(); + this.width = Math.floor(rect.width); + this.height = Math.floor(rect.height); + const bw = Math.floor(rect.width * this.dpr); + const bh = Math.floor(rect.height * this.dpr); + if (this.canvas.width !== bw || this.canvas.height !== bh) { + this.canvas.width = bw; + this.canvas.height = bh; + } + this.ctx.setTransform(this.dpr, 0, 0, this.dpr, 0, 0); + } + + pxPerUs() { + return (this.width - PADDING_X * 2) / Math.max(1, this.endUs - this.startUs); + } + + usAtX(x) { + return this.startUs + (x - PADDING_X) / this.pxPerUs(); + } + + xAtUs(us) { + return PADDING_X + (us - this.startUs) * this.pxPerUs(); + } + + layoutThreads() { + // Returns { rows: [{threadId, y, maxDepth}], totalH }. The y values + // are in canvas coordinates with the current scrollY already applied. + const rows = []; + let y = RULER_H + this.regionsBlockH - this.scrollY; + const sorted = Array.from(this.enabledThreads) + .filter((id) => this.threadMeta.has(id)) + .sort((a, b) => { + const ma = this.threadMeta.get(a); + const mb = this.threadMeta.get(b); + // OS threads first, then lanes + if (ma.is_lane !== mb.is_lane) return ma.is_lane ? 1 : -1; + // Group by group name (ungrouped first) + const ga = ma.group || ""; + const gb = mb.group || ""; + if (ga !== gb) { + if (!ga) return -1; + if (!gb) return 1; + return ga.localeCompare(gb, undefined, { numeric: true }); + } + // Match sidebar sort: sort_hint → scopes-first → thread_id → name + if (ma.sort_hint !== mb.sort_hint) return ma.sort_hint - mb.sort_hint; + if ((ma.scope_count > 0) !== (mb.scope_count > 0)) return mb.scope_count - ma.scope_count; + if (ma.thread_id !== mb.thread_id) return ma.thread_id - mb.thread_id; + return (ma.name || "").localeCompare(mb.name || "", undefined, { numeric: true }); + }); + for (const threadId of sorted) { + const timeline = this.timelines.get(threadId); + const scopes = timeline ? timeline.scopes : []; + let maxDepth = 0; + for (const s of scopes) { + if (s[3] > maxDepth) maxDepth = s[3]; + } + if (maxDepth > MAX_DRAWN_DEPTH) maxDepth = MAX_DRAWN_DEPTH; + const rowH = HEADER_H + (maxDepth + 1) * DEPTH_H; + rows.push({ threadId, y, headerH: HEADER_H, maxDepth, height: rowH }); + y += rowH + THREAD_GAP; + } + // y now points at the bottom of the last row in scrolled coords. + // Recover the unscrolled total content height for scroll clamping. + const totalContentH = y + this.scrollY; + return { rows, totalH: totalContentH }; + } + + draw() { + this.resizeBackingStore(); + const ctx = this.ctx; + const W = this.width; + const H = this.height; + + ctx.fillStyle = getComputedStyle(document.body).getPropertyValue("--bg0") || "#0d1117"; + ctx.fillRect(0, 0, W, H); + + this.hits = []; + + // First pass: lay out threads to discover total content height and + // clamp scrollY. We may re-layout after clamping so coordinates + // are accurate for the real draw. + let layout = this.layoutThreads(); + const visibleH = Math.max(0, H - RULER_H); + this.maxScrollY = Math.max(0, layout.totalH - RULER_H - visibleH); + if (this.scrollY > this.maxScrollY) + { + this.scrollY = this.maxScrollY; + layout = this.layoutThreads(); + } + if (this.scrollY < 0) + { + this.scrollY = 0; + layout = this.layoutThreads(); + } + const { rows } = layout; + + // Clip thread rendering to below the ruler strip so scrolled-up + // content never bleeds over it. The ruler is drawn after restoring. + ctx.save(); + ctx.beginPath(); + ctx.rect(0, RULER_H, W, H - RULER_H); + ctx.clip(); + + this.drawRegions(ctx, W); + + const pxPerUs = this.pxPerUs(); + const bg1 = getComputedStyle(document.body).getPropertyValue("--bg1") || "#161b22"; + const fg1 = getComputedStyle(document.body).getPropertyValue("--fg1") || "#c9d1d9"; + const fg2 = getComputedStyle(document.body).getPropertyValue("--fg2") || "#8b949e"; + + for (const row of rows) { + if (row.y > H) break; + if (row.y + row.height < RULER_H) continue; + + // Thread header strip + const meta = this.threadMeta.get(row.threadId); + const isLane = meta && meta.is_lane; + ctx.fillStyle = isLane ? "rgba(130, 80, 220, 0.12)" : bg1; + ctx.fillRect(0, row.y, W, row.headerH); + const prefix = isLane ? "⬦ " : ""; + const label = `${prefix}${(meta && meta.name) || `tid ${row.threadId}`} · ${row.threadId}`; + ctx.fillStyle = isLane ? "rgba(180, 140, 255, 0.8)" : fg2; + ctx.font = "11px -apple-system, Segoe UI, sans-serif"; + ctx.textBaseline = "middle"; + ctx.fillText(label, 6, row.y + row.headerH / 2); + + // Swimlane backgrounds + for (let d = 0; d <= row.maxDepth; d++) { + ctx.fillStyle = d % 2 === 0 ? "rgba(255,255,255,0.015)" : "rgba(255,255,255,0.00)"; + ctx.fillRect(0, row.y + row.headerH + d * DEPTH_H, W, DEPTH_H); + } + + const timeline = this.timelines.get(row.threadId); + if (timeline) { + this.drawScopes(ctx, timeline, row, pxPerUs, fg1); + } + } + + this.drawSelectionOutline(ctx); + + ctx.restore(); + + // Ruler is drawn last so it always overlays the thread region + // regardless of how far the content has scrolled. + this.drawRuler(ctx, W); + + // Bookmark lines span the whole content area, drawn after the ruler + // so the little diamond markers sit inside the ruler strip. + this.drawBookmarks(ctx, W, H); + + this.drawViewportInfo(); + } + + drawRuler(ctx, W) { + const bg1 = getComputedStyle(document.body).getPropertyValue("--bg1") || "#161b22"; + const fg2 = getComputedStyle(document.body).getPropertyValue("--fg2") || "#8b949e"; + const border = getComputedStyle(document.body).getPropertyValue("--border") || "#30363d"; + + ctx.fillStyle = bg1; + ctx.fillRect(0, 0, W, RULER_H); + ctx.strokeStyle = border; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(0, RULER_H - 0.5); + ctx.lineTo(W, RULER_H - 0.5); + ctx.stroke(); + + // Pick a tick interval that yields 6–12 ticks across the visible range. + const rangeUs = this.endUs - this.startUs; + const targetTicks = Math.max(4, Math.min(12, Math.floor(W / 100))); + const roughInterval = rangeUs / targetTicks; + const pow10 = Math.pow(10, Math.floor(Math.log10(roughInterval))); + let interval = pow10; + if (roughInterval / pow10 > 5) interval = 10 * pow10; + else if (roughInterval / pow10 > 2) interval = 5 * pow10; + else if (roughInterval / pow10 > 1) interval = 2 * pow10; + + ctx.fillStyle = fg2; + ctx.font = "10px -apple-system, Segoe UI, sans-serif"; + ctx.textBaseline = "middle"; + ctx.textAlign = "left"; + + const firstTick = Math.ceil(this.startUs / interval) * interval; + for (let t = firstTick; t <= this.endUs; t += interval) { + const x = this.xAtUs(t); + ctx.strokeStyle = border; + ctx.beginPath(); + ctx.moveTo(x + 0.5, 0); + ctx.lineTo(x + 0.5, RULER_H); + ctx.stroke(); + ctx.fillText(formatTime(t), x + 4, RULER_H / 2); + } + } + + drawRegions(ctx, W) { + if (this.regionCategories.length === 0) return; + + const startUs = this.startUs; + const endUs = this.endUs; + const pxPerUs = this.pxPerUs(); + const fg2 = getComputedStyle(document.body).getPropertyValue("--fg2") || "#8b949e"; + const bg1 = getComputedStyle(document.body).getPropertyValue("--bg1") || "#161b22"; + let catY = RULER_H - this.scrollY; + + for (let ci = 0; ci < this.regionCategories.length; ci++) { + if (!this.enabledRegionCategories.has(ci)) continue; + const cat = this.regionCategories[ci]; + // Category header + ctx.fillStyle = bg1; + ctx.fillRect(0, catY, W, REGION_HEADER_H); + ctx.fillStyle = fg2; + ctx.font = "10px -apple-system, Segoe UI, sans-serif"; + ctx.textBaseline = "middle"; + ctx.textAlign = "left"; + ctx.fillText(`Timing Regions \u2013 ${cat.name}`, 6, catY + REGION_HEADER_H / 2); + catY += REGION_HEADER_H; + + // Region bands for this category + for (const r of cat.regions) { + const beginUs = r.begin_us; + const endRegUs = r.end_us; + if (endRegUs < startUs) continue; + if (beginUs > endUs) continue; + + const x = this.xAtUs(beginUs); + const w = Math.max(MIN_RECT_W, (endRegUs - beginUs) * pxPerUs); + if (w < MIN_RECT_W) continue; + + const y = catY + r.depth * REGION_LANE_H; + + ctx.fillStyle = regionFillColor(r.name); + ctx.fillRect(x, y + 1, w, REGION_LANE_H - 2); + + ctx.strokeStyle = "rgba(255,255,255,0.2)"; + ctx.lineWidth = 1; + ctx.strokeRect(x + 0.5, y + 1.5, w - 1, REGION_LANE_H - 3); + + const visX = Math.max(x, 0); + const visRight = Math.min(x + w, this.width); + const visW = visRight - visX; + if (visW > 24 && r.name) { + ctx.fillStyle = "rgba(255,255,255,0.95)"; + ctx.font = "11px -apple-system, Segoe UI, sans-serif"; + ctx.textBaseline = "middle"; + ctx.textAlign = "left"; + ctx.save(); + ctx.beginPath(); + ctx.rect(visX + 3, y, visW - 6, REGION_LANE_H); + ctx.clip(); + ctx.fillText(r.name, visX + 5, y + REGION_LANE_H / 2); + ctx.restore(); + } + + this.hits.push({ x, y, w, h: REGION_LANE_H - 2, region: r, regionCategory: cat.name }); + } + + catY += cat.lane_count * REGION_LANE_H; + } + } + + drawBookmarks(ctx, W, H) { + if (!this.bookmarksVisible || !this.bookmarks || this.bookmarks.length === 0) return; + + const startUs = this.startUs; + const endUs = this.endUs; + + ctx.save(); + ctx.strokeStyle = "rgba(227, 179, 65, 0.85)"; + ctx.fillStyle = "rgba(227, 179, 65, 0.95)"; + ctx.lineWidth = 1; + + for (const b of this.bookmarks) { + if (b.time_us < startUs) continue; + if (b.time_us > endUs) break; + + const x = this.xAtUs(b.time_us); + if (x < -2 || x > W + 2) continue; + + // Dashed vertical line spanning the whole content area. + ctx.setLineDash([3, 3]); + ctx.beginPath(); + ctx.moveTo(x + 0.5, RULER_H); + ctx.lineTo(x + 0.5, H); + ctx.stroke(); + ctx.setLineDash([]); + + // Diamond marker inside the ruler strip. + const cy = RULER_H - 6; + ctx.beginPath(); + ctx.moveTo(x, cy - 4); + ctx.lineTo(x + 4, cy); + ctx.lineTo(x, cy + 4); + ctx.lineTo(x - 4, cy); + ctx.closePath(); + ctx.fill(); + + this.hits.push({ x: x - 4, y: 0, w: 9, h: H, bookmark: b }); + } + ctx.restore(); + } + + drawScopes(ctx, timeline, row, pxPerUs, textColor) { + const { scopes, perDepth } = timeline; + const startUs = this.startUs; + const endUs = this.endUs; + + const highlightNameId = this.highlightName + ? this.model.scopeNameIds.get(this.highlightName) + : undefined; + + ctx.textBaseline = "middle"; + ctx.textAlign = "left"; + ctx.font = "11px -apple-system, Segoe UI, sans-serif"; + + const rowTop = row.y + row.headerH; + const maxDepth = Math.min(row.maxDepth, perDepth.length - 1); + + for (let depth = 0; depth <= maxDepth; depth++) { + const indices = perDepth[depth]; + if (!indices || indices.length === 0) continue; + + // Sibling scopes at the same depth never overlap, so their end + // times are monotonic in begin order — a standard lower_bound + // on (end >= startUs) correctly finds the first visible scope, + // including outer-depth scopes whose begin is far before + // the viewport start. + let lo = 0; + let hi = indices.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + const s = scopes[indices[mid]]; + if (s[0] + s[1] < startUs) { + lo = mid + 1; + } else { + hi = mid; + } + } + + const y = rowTop + depth * DEPTH_H; + let rendered = 0; + for (let j = lo; j < indices.length; j++) { + const s = scopes[indices[j]]; + if (s[0] > endUs) break; + + const beginUs = s[0]; + const durationUs = s[1]; + const nameId = s[2]; + const mergeCount = s[4] || 0; + + ++rendered; + const x = this.xAtUs(beginUs); + const w = Math.max(MIN_RECT_W, durationUs * pxPerUs); + + const hue = ((nameId * 137.508) % 360 + 360) % 360; + const isHighlighted = highlightNameId !== undefined && nameId === highlightNameId; + + if (mergeCount > 1) { + // Merged scope — desaturated fill with dashed top indicator. + ctx.fillStyle = isHighlighted + ? `hsl(${hue.toFixed(0)}, 50%, 50%)` + : `hsl(${hue.toFixed(0)}, 30%, 35%)`; + ctx.fillRect(x, y + 1, w, DEPTH_H - 2); + ctx.strokeStyle = "rgba(255,255,255,0.25)"; + ctx.lineWidth = 1; + ctx.setLineDash([2, 2]); + ctx.beginPath(); + ctx.moveTo(x, y + 1.5); + ctx.lineTo(x + w, y + 1.5); + ctx.stroke(); + ctx.setLineDash([]); + } else { + ctx.fillStyle = isHighlighted ? scopeHighlightColor(nameId) : scopeFillColor(nameId); + ctx.fillRect(x, y + 1, w, DEPTH_H - 2); + } + + // Draw the label pinned to the visible portion of the rect + // so zooming into a long scope still shows its name. + const visX = Math.max(x, 0); + const visRight = Math.min(x + w, this.width); + const visW = visRight - visX; + if (visW > 30) { + const name = this.model.scopeNames[nameId] || "?"; + const maxChars = Math.floor((visW - 6) / 6); + const shown = name.length > maxChars ? name.slice(0, Math.max(0, maxChars - 1)) + "…" : name; + ctx.fillStyle = "rgba(255,255,255,0.95)"; + ctx.save(); + ctx.beginPath(); + ctx.rect(visX + 3, y, visW - 6, DEPTH_H); + ctx.clip(); + ctx.fillText(shown, visX + 4, y + DEPTH_H / 2); + ctx.restore(); + } + + this.hits.push({ x, y, w, h: DEPTH_H - 2, threadId: row.threadId, tuple: s }); + } + } + } + + drawSelectionOutline(ctx) { + if (!this.selected || !this.selected.tuple) return; + const s = this.selected.tuple; + const beginUs = s[0]; + const durationUs = s[1]; + const depth = s[3]; + const { rows } = this.layoutThreads(); + const row = rows.find((r) => r.threadId === this.selected.threadId); + if (!row) return; + const x = this.xAtUs(beginUs); + const w = Math.max(MIN_RECT_W, durationUs * this.pxPerUs()); + const y = row.y + row.headerH + depth * DEPTH_H; + ctx.strokeStyle = "#ffffff"; + ctx.lineWidth = 1.5; + ctx.strokeRect(x - 0.5, y + 0.5, w + 1, DEPTH_H - 1); + } + + drawViewportInfo() { + const text = `${formatRange(this.startUs, this.endUs)} · ${(this.pxPerUs() * 1000).toFixed(2)} px/ms`; + this.viewportInfoEl.textContent = text; + } + + hitTest(clientX, clientY) { + const rect = this.canvas.getBoundingClientRect(); + const x = clientX - rect.left; + const y = clientY - rect.top; + for (let i = this.hits.length - 1; i >= 0; i--) { + const h = this.hits[i]; + if (x >= h.x && x < h.x + h.w && y >= h.y && y < h.y + h.h) { + return h; + } + } + return null; + } + + onMouseDown(e) { + if (e.button !== 0) return; + this.panning = true; + this.panMoved = false; + this.panStartX = e.clientX; + this.panStartY = e.clientY; + this.panStartUs = this.startUs; + this.panStartScrollY = this.scrollY; + this.panRangeUs = this.endUs - this.startUs; + } + + onMouseMove(e) { + if (this.panning) { + const dx = e.clientX - this.panStartX; + const dy = e.clientY - this.panStartY; + if (Math.abs(dx) > 2 || Math.abs(dy) > 2) this.panMoved = true; + const deltaUs = -dx / this.pxPerUs(); + this.startUs = this.panStartUs + deltaUs; + this.endUs = this.startUs + this.panRangeUs; + this.scrollY = this.panStartScrollY - dy; + this.checkViewportFetch(); + this.requestDraw(); + this.hideTooltip(); + return; + } + const hit = this.hitTest(e.clientX, e.clientY); + if (hit) { + this.showTooltip(hit, e.clientX, e.clientY); + } else { + this.hideTooltip(); + } + } + + onMouseUp(e) { + if (!this.panning) return; + this.panning = false; + if (!this.panMoved) { + const hit = this.hitTest(e.clientX, e.clientY); + if (hit) { + this.selectScope(hit); + } + } + } + + onWheel(e) { + e.preventDefault(); + + // Shift+wheel (or horizontal wheel delta from a trackpad) scrolls + // vertically without changing the zoom. + if (e.shiftKey || Math.abs(e.deltaX) > Math.abs(e.deltaY)) { + const step = e.deltaX !== 0 ? e.deltaX : e.deltaY; + this.scrollY += step; + this.requestDraw(); + return; + } + + const rect = this.canvas.getBoundingClientRect(); + const cursorX = e.clientX - rect.left; + const cursorUs = this.usAtX(cursorX); + const factor = e.deltaY > 0 ? 1.25 : 0.8; + const newRange = Math.max(10, (this.endUs - this.startUs) * factor); + this.startUs = cursorUs - (cursorX - PADDING_X) * (newRange / (this.width - PADDING_X * 2)); + this.endUs = this.startUs + newRange; + this.checkViewportFetch(); + this.requestDraw(); + } + + showTooltip(hit, clientX, clientY) { + let title = ""; + let meta = ""; + + if (hit.bookmark) { + const b = hit.bookmark; + title = b.text || "(unnamed bookmark)"; + const loc = b.file ? `${b.file.split(/[\\/]/).pop()}:${b.line}` : ""; + meta = `bookmark · ${formatTime(b.time_us)}${loc ? " · " + loc : ""}`; + } + else if (hit.region) { + const r = hit.region; + const dur = r.end_us - r.begin_us; + title = r.name || "(unnamed region)"; + meta = `region · ${formatTime(dur)} · start ${formatTime(r.begin_us)}${hit.regionCategory ? " · " + hit.regionCategory : ""}`; + } + else { + const s = hit.tuple; + title = this.model.scopeNames[s[2]] || "?"; + const tm = this.threadMeta.get(hit.threadId); + const threadName = (tm && tm.name) || `tid ${hit.threadId}`; + meta = `${formatTime(s[1])} · depth ${s[3]} · ${threadName} · start ${formatTime(s[0])}`; + if (s[4] > 1) { + meta += ` · ${s[4]} merged`; + } + } + + this.tooltip.innerHTML = + `<div class="tt-name"></div>` + + `<div class="tt-meta"></div>`; + this.tooltip.querySelector(".tt-name").textContent = title; + this.tooltip.querySelector(".tt-meta").textContent = meta; + + const rect = this.canvas.getBoundingClientRect(); + const tx = clientX - rect.left + 12; + const ty = clientY - rect.top + 12; + this.tooltip.style.left = `${tx}px`; + this.tooltip.style.top = `${ty}px`; + this.tooltip.hidden = false; + } + + hideTooltip() { + this.tooltip.hidden = true; + } + + selectScope(hit) { + this.selected = hit; + + if (hit.bookmark) { + const b = hit.bookmark; + this.selectionEl.innerHTML = + `<div class="selection-title"></div>` + + `<div class="selection-meta">` + + `<div><span class="k">Kind:</span> <span class="v">bookmark</span></div>` + + `<div><span class="k">Time:</span> <span class="v" data-k="time"></span></div>` + + `<div><span class="k">Source:</span> <span class="v" data-k="src"></span></div>` + + `</div>`; + this.selectionEl.querySelector(".selection-title").textContent = b.text || "(unnamed bookmark)"; + this.selectionEl.querySelector("[data-k=time]").textContent = formatTime(b.time_us); + this.selectionEl.querySelector("[data-k=src]").textContent = b.file ? `${b.file}:${b.line}` : ""; + this.requestDraw(); + return; + } + + if (hit.region) { + const r = hit.region; + this.selectionEl.innerHTML = + `<div class="selection-title"></div>` + + `<div class="selection-meta">` + + `<div><span class="k">Kind:</span> <span class="v">region</span></div>` + + `<div><span class="k">Duration:</span> <span class="v" data-k="dur"></span></div>` + + `<div><span class="k">Begin:</span> <span class="v" data-k="begin"></span></div>` + + `<div><span class="k">End:</span> <span class="v" data-k="end"></span></div>` + + `<div><span class="k">Category:</span> <span class="v" data-k="cat"></span></div>` + + `</div>`; + this.selectionEl.querySelector(".selection-title").textContent = r.name || "(unnamed region)"; + this.selectionEl.querySelector("[data-k=dur]").textContent = formatTime(r.end_us - r.begin_us); + this.selectionEl.querySelector("[data-k=begin]").textContent = formatTime(r.begin_us); + this.selectionEl.querySelector("[data-k=end]").textContent = formatTime(r.end_us); + this.selectionEl.querySelector("[data-k=cat]").textContent = hit.regionCategory || "\u2014"; + this.requestDraw(); + return; + } + + const s = hit.tuple; + const name = this.model.scopeNames[s[2]] || "?"; + const meta = this.threadMeta.get(hit.threadId); + const threadName = (meta && meta.name) || `tid ${hit.threadId}`; + const mergedRow = s[4] > 1 + ? `<div><span class="k">Merged:</span> <span class="v" data-k="merged"></span></div>` + : ""; + this.selectionEl.innerHTML = + `<div class="selection-title"></div>` + + `<div class="selection-meta">` + + `<div><span class="k">Duration:</span> <span class="v" data-k="dur"></span></div>` + + `<div><span class="k">Begin:</span> <span class="v" data-k="begin"></span></div>` + + `<div><span class="k">End:</span> <span class="v" data-k="end"></span></div>` + + `<div><span class="k">Depth:</span> <span class="v" data-k="depth"></span></div>` + + `<div><span class="k">Thread:</span> <span class="v" data-k="thread"></span></div>` + + mergedRow + + `</div>`; + this.selectionEl.querySelector(".selection-title").textContent = name; + this.selectionEl.querySelector("[data-k=dur]").textContent = formatTime(s[1]); + this.selectionEl.querySelector("[data-k=begin]").textContent = formatTime(s[0]); + this.selectionEl.querySelector("[data-k=end]").textContent = formatTime(s[0] + s[1]); + this.selectionEl.querySelector("[data-k=depth]").textContent = String(s[3]); + this.selectionEl.querySelector("[data-k=thread]").textContent = `${threadName} (${hit.threadId})`; + if (s[4] > 1) { + this.selectionEl.querySelector("[data-k=merged]").textContent = `${s[4]} scopes`; + } + this.requestDraw(); + this.onScopeSelect(name); + } +} diff --git a/src/zen/frontend/html/trace.css b/src/zen/frontend/html/trace.css new file mode 100644 index 000000000..2ff324019 --- /dev/null +++ b/src/zen/frontend/html/trace.css @@ -0,0 +1,1312 @@ +/* Copyright Epic Games, Inc. All Rights Reserved. */ + +:root, +:root[data-theme="dark"] { + --bg0: #0d1117; + --bg1: #161b22; + --bg2: #1c2128; + --bg3: #21262d; + --border: #30363d; + --border-soft: #21262d; + --fg0: #f0f6fc; + --fg1: #c9d1d9; + --fg2: #8b949e; + --accent: #58a6ff; + --accent-soft: #1c2128; + --warn: #d29922; + --ok: #3fb950; + --fail: #f85149; + --highlight: #e3b34166; +} + +@media (prefers-color-scheme: light) { + :root:not([data-theme]), + :root[data-theme="system"] { + --bg0: #ffffff; + --bg1: #f6f8fa; + --bg2: #ffffff; + --bg3: #eaeef2; + --border: #d0d7de; + --border-soft: #d8dee4; + --fg0: #1f2328; + --fg1: #24292f; + --fg2: #656d76; + --accent: #0969da; + --accent-soft: #ddf4ff; + --warn: #9a6700; + --ok: #1a7f37; + --fail: #cf222e; + --highlight: #b8860b44; + } +} + +:root[data-theme="light"] { + --bg0: #ffffff; + --bg1: #f6f8fa; + --bg2: #ffffff; + --bg3: #eaeef2; + --border: #d0d7de; + --border-soft: #d8dee4; + --fg0: #1f2328; + --fg1: #24292f; + --fg2: #656d76; + --accent: #0969da; + --accent-soft: #ddf4ff; + --warn: #9a6700; + --ok: #1a7f37; + --fail: #cf222e; + --highlight: #b8860b44; +} + +* { + box-sizing: border-box; +} + +html, body { + margin: 0; + padding: 0; + height: 100%; + background: var(--bg0); + color: var(--fg1); + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + font-size: 13px; + overflow: hidden; +} + +body { + display: flex; + flex-direction: column; +} + +pre, code, .mono { + font-family: 'SF Mono', 'Cascadia Mono', Consolas, 'DejaVu Sans Mono', monospace; + font-size: 12px; +} + +/* -- header ---------------------------------------------------------------- */ + +.header { + display: flex; + align-items: center; + gap: 16px; + padding: 10px 16px; + background: var(--bg1); + border-bottom: 1px solid var(--border); + flex-shrink: 0; +} + +.header-title { + font-weight: 600; + color: var(--fg0); + font-size: 14px; +} + +.header-file { + color: var(--fg2); + font-family: 'SF Mono', 'Cascadia Mono', Consolas, monospace; + font-size: 12px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + flex: 1; + min-width: 0; +} + +.header-stats { + color: var(--fg2); + font-size: 12px; + display: flex; + gap: 16px; +} + +.header-stats .k { + color: var(--fg2); + margin-right: 4px; +} + +.header-stats .v { + color: var(--fg0); + font-weight: 500; +} + +.header-btn { + background: var(--bg2); + color: var(--fg1); + border: 1px solid var(--border); + border-radius: 6px; + padding: 6px 10px; + font-size: 12px; + cursor: pointer; + flex-shrink: 0; +} + +.header-btn:hover { + background: var(--bg3); + color: var(--fg0); +} + +/* -- layout ---------------------------------------------------------------- */ + +.layout { + display: flex; + flex: 1; + min-height: 0; +} + +.sidebar { + width: 260px; + flex-shrink: 0; + background: var(--bg1); + border-right: 1px solid var(--border); + display: flex; + flex-direction: column; + min-height: 0; +} + +.content { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + background: var(--bg0); +} + +/* -- tabs ------------------------------------------------------------------ */ + +.tabs { + display: flex; + border-bottom: 1px solid var(--border); + flex-shrink: 0; +} + +.tab { + flex: 1; + padding: 10px 8px; + background: transparent; + border: none; + border-bottom: 2px solid transparent; + color: var(--fg2); + font-size: 12px; + font-weight: 500; + cursor: pointer; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.tab:hover { + color: var(--fg0); + background: var(--bg2); +} + +.tab.active { + color: var(--accent); + border-bottom-color: var(--accent); +} + +/* -- sidebar sections ------------------------------------------------------ */ + +.sidebar-section { + padding: 12px 12px; + border-bottom: 1px solid var(--border-soft); + flex-shrink: 0; + min-height: 0; + display: flex; + flex-direction: column; +} + +.sidebar-section:last-child { + flex: 1; + overflow-y: auto; +} + +.sidebar-label { + display: flex; + align-items: baseline; + gap: 6px; + font-size: 10px; + color: var(--fg2); + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 6px; + font-weight: 600; +} + +.sidebar-action { + font-size: 9px; + color: var(--fg2); + background: none; + border: none; + cursor: pointer; + padding: 0; + text-transform: lowercase; + letter-spacing: 0; + font-weight: 400; + opacity: 0.7; +} + +.sidebar-action:hover { + color: var(--fg0); + opacity: 1; +} + +#search-input { + width: 100%; + background: var(--bg2); + border: 1px solid var(--border); + color: var(--fg0); + padding: 5px 8px; + border-radius: 4px; + font-size: 12px; +} + +#search-input:focus { + outline: none; + border-color: var(--accent); +} + +.search-results { + margin-top: 6px; + max-height: 180px; + overflow-y: auto; + font-size: 12px; +} + +.search-results .hit { + padding: 3px 6px; + border-radius: 3px; + cursor: pointer; + color: var(--fg1); + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 8px; +} + +.search-results .hit:hover { + background: var(--accent-soft); + color: var(--fg0); +} + +.search-results .hit-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.search-results .hit-count { + color: var(--fg2); + font-size: 11px; + flex-shrink: 0; +} + +/* -- threads list ---------------------------------------------------------- */ + +.threads-list, .regions-list { + display: flex; + flex-direction: column; + gap: 2px; + overflow-y: auto; +} + +.thread-group-header { + font-size: 10px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--fg2); + padding: 6px 4px 2px; + user-select: none; +} + +.thread-group-header[data-group] { + cursor: pointer; + border-radius: 3px; +} + +.thread-group-header[data-group]:hover { + color: var(--fg1); + background: var(--bg2); +} + +.thread-group-header:first-child { + padding-top: 0; +} + +.group-checkbox { + margin: 0 2px 0 0; + accent-color: var(--accent); + cursor: pointer; +} + +.group-chevron { + display: inline-block; + margin-right: 2px; + transition: transform 0.15s; +} + +.thread-group-header.collapsed .group-chevron { + transform: rotate(-90deg); +} + +.thread-row { + display: flex; + align-items: center; + gap: 6px; + padding: 3px 4px; + border-radius: 3px; + cursor: pointer; + font-size: 12px; + color: var(--fg1); +} + +.thread-row.lane .thread-name { + font-style: italic; +} + +.thread-row:hover { + background: var(--bg2); +} + +.thread-row.empty { + color: var(--fg2); + opacity: 0.6; +} + +.thread-row input[type=checkbox] { + margin: 0; + accent-color: var(--accent); +} + +.thread-row .thread-name { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.thread-row .thread-count { + color: var(--fg2); + font-size: 11px; + flex-shrink: 0; +} + +/* -- views ----------------------------------------------------------------- */ + +.view { + flex: 1; + display: flex; + flex-direction: column; + min-height: 0; + min-width: 0; + overflow: hidden; +} + +.view[hidden] { + display: none; +} + +/* -- timeline -------------------------------------------------------------- */ + +.view-timeline { + position: relative; +} + +.timeline-toolbar { + display: flex; + align-items: center; + gap: 12px; + padding: 6px 12px; + border-bottom: 1px solid var(--border-soft); + background: var(--bg1); + flex-shrink: 0; +} + +.viewport-info { + color: var(--fg2); + font-size: 11px; + flex: 1; + font-family: 'SF Mono', 'Cascadia Mono', Consolas, monospace; +} + +.toolbar-toggle { + display: flex; + align-items: center; + gap: 4px; + font-size: 11px; + color: var(--fg2); + cursor: pointer; + user-select: none; +} + +.toolbar-toggle input[type="checkbox"] { + margin: 0; +} + +.btn { + background: var(--bg2); + border: 1px solid var(--border); + color: var(--fg1); + padding: 3px 10px; + border-radius: 4px; + font-size: 11px; + cursor: pointer; +} + +.btn:hover { + background: var(--bg3); + color: var(--fg0); +} + +.timeline-frame { + flex: 1; + position: relative; + min-height: 0; + overflow: hidden; +} + +#timeline-canvas { + display: block; + width: 100%; + height: 100%; + cursor: grab; +} + +#timeline-canvas:active { + cursor: grabbing; +} + +.tooltip { + position: absolute; + background: var(--bg1); + border: 1px solid var(--border); + border-radius: 4px; + padding: 6px 10px; + font-size: 11px; + color: var(--fg0); + pointer-events: none; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4); + max-width: 360px; + z-index: 10; +} + +.tooltip .tt-name { + font-weight: 600; + margin-bottom: 2px; +} + +.tooltip .tt-meta { + color: var(--fg2); + font-family: 'SF Mono', 'Cascadia Mono', Consolas, monospace; + font-size: 10px; +} + +.selection-panel { + background: var(--bg1); + border-top: 1px solid var(--border-soft); + padding: 10px 14px; + flex-shrink: 0; + min-height: 56px; + max-height: 140px; + overflow-y: auto; +} + +.selection-hint { + color: var(--fg2); + font-size: 11px; + font-style: italic; +} + +.selection-title { + color: var(--fg0); + font-weight: 600; + font-size: 13px; + margin-bottom: 4px; + word-break: break-all; +} + +.selection-meta { + color: var(--fg2); + font-size: 11px; + font-family: 'SF Mono', 'Cascadia Mono', Consolas, monospace; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 4px 16px; +} + +.selection-meta .k { + color: var(--fg2); +} + +.selection-meta .v { + color: var(--fg1); +} + +/* -- stats table ----------------------------------------------------------- */ + +.view-stats { + overflow-y: auto; + padding: 12px; +} + +.stats-table { + width: 100%; + border-collapse: collapse; + font-size: 12px; +} + +.stats-table th { + text-align: left; + padding: 8px 10px; + background: var(--bg1); + color: var(--fg2); + font-weight: 600; + text-transform: uppercase; + font-size: 10px; + letter-spacing: 0.5px; + border-bottom: 1px solid var(--border); + cursor: pointer; + user-select: none; + position: sticky; + top: 0; +} + +.stats-table th.num { + text-align: right; +} + +.stats-table th:hover { + color: var(--fg0); +} + +.stats-table th.sorted::after { + content: ' ▾'; + color: var(--accent); +} + +.stats-table th.sorted.asc::after { + content: ' ▴'; +} + +.stats-table td { + padding: 5px 10px; + border-bottom: 1px solid var(--border-soft); + color: var(--fg1); +} + +.stats-table td.num { + text-align: right; + font-variant-numeric: tabular-nums; + color: var(--fg0); +} + +.stats-table tbody tr { + cursor: pointer; +} + +.stats-table tbody tr:hover { + background: var(--bg1); +} + +.stats-table tbody tr.selected { + background: var(--accent-soft); +} + +/* -- session view ---------------------------------------------------------- */ + +.view-session { + overflow-y: auto; + padding: 20px 24px; +} + +.session-content h2 { + font-size: 14px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--fg2); + margin: 24px 0 10px; + border-bottom: 1px solid var(--border-soft); + padding-bottom: 4px; +} + +.session-content h2:first-child { + margin-top: 0; +} + +.session-content dl { + display: grid; + grid-template-columns: 150px 1fr; + gap: 6px 16px; + margin: 0 0 12px; + font-size: 12px; +} + +.session-content dt { + color: var(--fg2); +} + +.session-content dd { + margin: 0; + color: var(--fg1); + font-family: 'SF Mono', 'Cascadia Mono', Consolas, monospace; + word-break: break-all; +} + +.session-content table { + width: 100%; + border-collapse: collapse; + font-size: 12px; +} + +.session-content table th { + text-align: left; + padding: 6px 10px; + color: var(--fg2); + font-weight: 600; + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.5px; + border-bottom: 1px solid var(--border-soft); +} + +.session-content table th.num { + text-align: right; +} + +.session-content table td { + padding: 4px 10px; + border-bottom: 1px solid var(--border-soft); + color: var(--fg1); +} + +.session-content table td.num { + text-align: right; + font-variant-numeric: tabular-nums; +} + +.chan-enabled { + color: var(--ok); +} + +.chan-disabled { + color: var(--fg2); +} + +.chan-readonly { + color: var(--warn); + font-size: 10px; + margin-left: 8px; +} + +/* -- logs view ------------------------------------------------------------- */ + +.view-logs { + display: flex; + flex-direction: column; + min-height: 0; +} + +#logs-content { + display: flex; + flex-direction: column; + flex: 1; + min-height: 0; +} + +.logs-toolbar { + display: flex; + align-items: center; + gap: 16px; + padding: 8px 12px; + background: var(--bg1); + border-bottom: 1px solid var(--border-soft); + flex-shrink: 0; +} + +.logs-filter { + display: flex; + align-items: center; + gap: 6px; +} + +.logs-filter-grow { + flex: 1; +} + +.logs-filter-label { + color: var(--fg2); + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.5px; + font-weight: 600; +} + +.logs-toolbar select, +.logs-toolbar input { + background: var(--bg2); + border: 1px solid var(--border); + color: var(--fg0); + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; +} + +.logs-toolbar select:focus, +.logs-toolbar input:focus { + outline: none; + border-color: var(--accent); +} + +.logs-toolbar input { + width: 100%; + box-sizing: border-box; +} + +.logs-count { + color: var(--fg2); + font-size: 11px; + font-family: 'SF Mono', 'Cascadia Mono', Consolas, monospace; + white-space: nowrap; +} + +.logs-list-wrap { + flex: 1; + overflow: auto; + min-height: 0; +} + +.logs-table { + width: 100%; + border-collapse: collapse; + font-size: 12px; +} + +.logs-table th { + text-align: left; + padding: 6px 10px; + background: var(--bg1); + color: var(--fg2); + font-weight: 600; + text-transform: uppercase; + font-size: 10px; + letter-spacing: 0.5px; + border-bottom: 1px solid var(--border); + position: sticky; + top: 0; + z-index: 1; +} + +.logs-table td { + padding: 4px 10px; + border-bottom: 1px solid var(--border-soft); + vertical-align: top; +} + +.logs-table .col-time { + white-space: nowrap; + color: var(--fg2); + width: 1%; +} + +.logs-table .col-verb { + white-space: nowrap; + width: 1%; + font-weight: 500; +} + +.logs-table .col-cat { + white-space: nowrap; + width: 1%; + color: var(--fg1); +} + +.logs-table .col-msg { + color: var(--fg0); + word-break: break-word; +} + +.logs-table .col-loc { + white-space: nowrap; + color: var(--fg2); + width: 1%; + font-size: 11px; +} + +.logs-table tr.vb-fatal td, +.logs-table tr.vb-error td { + color: var(--fail); +} + +.logs-table tr.vb-error .col-msg { + color: var(--fail); +} + +.logs-table tr.vb-warn .col-verb, +.logs-table tr.vb-warn .col-msg { + color: var(--warn); +} + +.logs-table tr.vb-display .col-verb { + color: var(--accent); +} + +.logs-table tr.vb-verbose .col-verb, +.logs-table tr.vb-verbose .col-msg { + color: var(--fg2); +} + +.logs-table tr.bm-row .col-verb { + color: #e3b341; + font-weight: 600; +} + +.logs-table tr.bm-row .col-msg { + color: #f0d078; +} + +.logs-table tr.bm-row .col-time { + color: #e3b341; +} + +.logs-empty, .logs-error { + padding: 20px; + text-align: center; + color: var(--fg2); +} + +.logs-error { + color: var(--fail); +} + +/* -- loading overlay ------------------------------------------------------- */ + +.loading { + position: fixed; + inset: 0; + background: var(--bg0); + color: var(--fg2); + display: flex; + align-items: center; + justify-content: center; + font-size: 14px; + z-index: 100; +} + +.loading.hidden { + display: none; +} + +/* ── CSV Stats view ───────────────────────────────────────────────── */ + +.view-csv { + display: flex; + flex-direction: column; + min-height: 0; +} + +#csv-content { + flex: 1; + display: flex; + flex-direction: column; + min-height: 0; +} + +.csv-layout { + display: flex; + flex: 1; + min-height: 0; +} + +.csv-tree-panel { + width: 240px; + flex-shrink: 0; + border-right: 1px solid var(--border); + display: flex; + flex-direction: column; + overflow-y: auto; + padding: 8px; +} + +.csv-chart-panel { + flex: 1; + position: relative; + min-width: 0; +} + +.csv-chart-canvas { + width: 100%; + height: 100%; + display: block; +} + +.csv-chart-tooltip { + position: absolute; + background: var(--bg1); + border: 1px solid var(--border); + border-radius: 4px; + padding: 6px 8px; + font-size: 11px; + color: var(--fg1); + pointer-events: none; + z-index: 10; + white-space: nowrap; +} + +.csv-cat-header { + font-size: 10px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--fg2); + padding: 8px 4px 2px; +} + +.csv-stat-row { + display: flex; + align-items: center; + gap: 6px; + padding: 2px 4px; + border-radius: 3px; + cursor: pointer; + font-size: 12px; + color: var(--fg1); +} + +.csv-stat-row:hover { + background: var(--bg2); +} + +.csv-stat-row input[type=checkbox] { + margin: 0; + accent-color: var(--accent); +} + +.csv-stat-name { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.csv-empty { + color: var(--fg2); + font-size: 12px; + padding: 12px 4px; +} + +/* -- memory view ---------------------------------------------------------- */ + +.view-memory { + overflow: auto; + padding: 16px; +} + +.memory-view { + display: flex; + flex-direction: column; + gap: 16px; +} + +.memory-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + gap: 12px; +} + +.memory-card, +.memory-panel { + background: var(--bg1); + border: 1px solid var(--border); + border-radius: 8px; +} + +.memory-card { + padding: 12px 14px; +} + +.memory-card-label, +.memory-panel-subtitle, +.memory-empty, +.memory-frame-path { + color: var(--fg2); +} + +.memory-chart-axis, +.memory-chart-text { + fill: var(--fg2); + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + font-size: 11px; +} + +.memory-card-label { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.4px; + margin-bottom: 6px; +} + +.memory-card-value { + font-size: 20px; + font-weight: 600; + color: var(--fg0); +} + +.memory-panel-header { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 12px; + padding: 12px 14px; + border-bottom: 1px solid var(--border-soft); +} + +.memory-panel-header-wrap { + flex-wrap: wrap; +} + +.memory-panel-title { + font-weight: 600; + color: var(--fg0); +} + +.memory-controls { + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; + color: var(--fg2); + font-size: 12px; +} + +.memory-controls label { + display: flex; + align-items: center; + gap: 6px; +} + +.memory-filter-input, +.memory-controls select { + background: var(--bg2); + color: var(--fg1); + border: 1px solid var(--border); + border-radius: 4px; + padding: 4px 6px; + font-size: 12px; +} + +.memory-filter-input { + min-width: 180px; +} + +.memory-direction-btn, +.memory-clear-btn { + background: var(--bg2); + color: var(--fg1); + border: 1px solid var(--border); + border-radius: 4px; + padding: 4px 8px; + font-size: 12px; + cursor: pointer; +} + +.memory-direction-btn:hover, +.memory-clear-btn:hover:not(:disabled) { + background: var(--bg3); +} + +.memory-clear-btn:disabled { + opacity: 0.5; + cursor: default; +} + +.memory-chart-wrap { + padding: 10px 12px 12px; +} + +.memory-chart { + display: block; + width: 100%; + height: 220px; +} + +.memory-chart-bg { + fill: var(--bg1); +} + +.memory-chart-grid { + stroke: var(--border-soft); + stroke-width: 1; +} + +.memory-chart-grid-vert { + stroke-opacity: 0.45; +} + +.memory-chart-line { + fill: none; + stroke: var(--accent); + stroke-width: 2; + stroke-linejoin: round; + stroke-linecap: round; +} + +.memory-histogram { + height: 260px; +} + +.memory-histogram-bar { + fill: var(--accent); + fill-opacity: 0.78; +} + +.memory-histogram-bar:hover { + fill-opacity: 1; +} + +.memory-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; +} + +.memory-callstack-panel { + grid-column: 1 / -1; +} + +.memory-table-wrap { + overflow: auto; + max-height: 360px; +} + +.memory-table { + width: 100%; + border-collapse: collapse; +} + +.memory-table th, +.memory-table td { + padding: 8px 10px; + border-bottom: 1px solid var(--border-soft); + text-align: left; + vertical-align: top; +} + +.memory-table th { + position: sticky; + top: 0; + background: var(--bg1); + z-index: 1; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.4px; + color: var(--fg2); +} + +.memory-table .num { + text-align: right; + white-space: nowrap; +} + +.memory-table tbody tr { + cursor: pointer; +} + +.memory-table tbody tr:hover { + background: var(--bg2); +} + +.memory-table tbody tr.selected { + background: var(--accent-soft); +} + +.memory-group-row td { + background: var(--bg2); + color: var(--fg2); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.4px; + font-weight: 600; +} + +.memory-summary-top-row { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 10px; +} + +.memory-summary-top { + color: var(--fg0); + font-weight: 500; + word-break: break-word; + min-width: 0; + flex: 1; +} + +.memory-summary-secondary { + margin-top: 3px; + color: var(--fg2); + font-size: 12px; + word-break: break-word; +} + +.memory-summary-badges { + display: flex; + flex-wrap: wrap; + gap: 6px; + justify-content: flex-end; + flex-shrink: 0; +} + +.memory-badge { + display: inline-block; + padding: 1px 6px; + border-radius: 999px; + background: var(--accent-soft); + color: var(--fg2); + font-size: 11px; +} + +.memory-mark { + background: color-mix(in srgb, var(--accent) 28%, transparent); + color: inherit; + padding: 0 1px; + border-radius: 2px; +} + +.memory-callstack-body { + padding: 12px 14px; + max-height: 320px; + overflow: auto; +} + +.memory-callstack-list { + margin: 0; + padding-left: 22px; +} + +.memory-callstack-list li { + margin: 0 0 8px; + word-break: break-word; +} + +.memory-frame-index { + color: var(--fg2); + margin-right: 8px; +} + +.memory-frame-display { + font-family: 'SF Mono', 'Cascadia Mono', Consolas, 'DejaVu Sans Mono', monospace; + color: var(--fg0); +} + +.memory-frame-path { + margin-left: 8px; + font-size: 12px; +} + +@media (max-width: 1200px) { + .memory-grid { + grid-template-columns: 1fr; + } +} diff --git a/src/zen/frontend/html/trace.js b/src/zen/frontend/html/trace.js new file mode 100644 index 000000000..2910da15d --- /dev/null +++ b/src/zen/frontend/html/trace.js @@ -0,0 +1,577 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// Entry point: boots the viewer, owns the model, wires tabs / sidebar / +// search / threads list / session panel. + +import { getSession, getThreads, getChannels, getScopeStats, getScopeNames, getLogCategories, getBookmarks, getRegions, getCsvCategories, getCsvStats } from "./api.js"; +import { Timeline } from "./timeline.js"; +import { StatsView } from "./stats.js"; +import { MemoryView } from "./memory.js"; +import { LogsView } from "./logs.js"; +import { CsvStatsView } from "./csvstats.js"; + +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, (c) => ({ + "&": "&", + "<": "<", + ">": ">", + "\"": """, + "'": "'", + }[c])); +} + +function formatTimeMs(us) { + if (us < 1000) return `${us} µs`; + if (us < 1_000_000) return `${(us / 1000).toFixed(2)} ms`; + return `${(us / 1_000_000).toFixed(2)} s`; +} + +function formatNum(n) { + return Number(n).toLocaleString(); +} + +function stripNul(s) { + return (s || "").replace(/\u0000/g, ""); +} + +function getThemePreference() { + const params = new URLSearchParams(window.location.search); + const theme = params.get("theme"); + if (theme === "dark" || theme === "light" || theme === "system") { + return theme; + } + const stored = window.localStorage.getItem("zen-trace-theme"); + if (stored === "dark" || stored === "light" || stored === "system") { + return stored; + } + return "system"; +} + +function applyTheme(theme) { + document.documentElement.setAttribute("data-theme", theme); + window.localStorage.setItem("zen-trace-theme", theme); + const url = new URL(window.location.href); + if (theme === "system") { + url.searchParams.delete("theme"); + } else { + url.searchParams.set("theme", theme); + } + window.history.replaceState({}, "", url); + const btn = document.getElementById("theme-toggle"); + if (btn) { + btn.textContent = theme === "dark" ? "Dark" : theme === "light" ? "Light" : "System"; + btn.title = `Theme: ${theme}. Click to cycle.`; + } +} + +function setupThemeToggle() { + const btn = document.getElementById("theme-toggle"); + if (!btn) { + return; + } + const themes = ["system", "dark", "light"]; + let theme = getThemePreference(); + applyTheme(theme); + btn.addEventListener("click", () => { + const index = themes.indexOf(theme); + theme = themes[(index + 1) % themes.length]; + applyTheme(theme); + }); +} + +async function main() { + setupThemeToggle(); + const loadingEl = document.getElementById("loading"); + try { + const [session, threads, channels, scopeStats, scopeNames, logCategories, bookmarks, regionsResponse, csvCategories, csvStats] = await Promise.all([ + getSession(), + getThreads(), + getChannels(), + getScopeStats(), + getScopeNames(), + getLogCategories(), + getBookmarks(), + getRegions(), + getCsvCategories(), + getCsvStats(), + ]); + + // Normalize strings (tourist sometimes leaves trailing NULs in FieldStr). + for (const t of threads) t.name = stripNul(t.name); + session.app_name = stripNul(session.app_name); + session.project_name = stripNul(session.project_name); + session.branch = stripNul(session.branch); + session.build_version = stripNul(session.build_version); + session.platform = stripNul(session.platform); + session.command_line = stripNul(session.command_line); + for (const s of scopeStats) s.name = stripNul(s.name); + for (let i = 0; i < scopeNames.length; i++) scopeNames[i] = stripNul(scopeNames[i]); + for (const c of logCategories) c.name = stripNul(c.name); + for (const b of bookmarks) { + b.text = stripNul(b.text); + b.file = stripNul(b.file); + } + const regionCategories = regionsResponse && regionsResponse.categories ? regionsResponse.categories : []; + for (const cat of regionCategories) { + cat.name = stripNul(cat.name); + for (const r of cat.regions) { + r.name = stripNul(r.name); + } + } + for (const cat of csvCategories) cat.name = stripNul(cat.name); + for (const s of csvStats) s.name = stripNul(s.name); + + // Precompute name → id for highlight lookups. + const scopeNameIds = new Map(); + for (let i = 0; i < scopeNames.length; i++) { + scopeNameIds.set(scopeNames[i], i); + } + + const model = { session, threads, channels, scopeStats, scopeNames, scopeNameIds, logCategories, bookmarks, regionCategories, csvCategories, csvStats }; + + renderHeader(model); + renderSessionView(model); + + const timeline = new Timeline({ + canvas: document.getElementById("timeline-canvas"), + tooltip: document.getElementById("tooltip"), + selectionEl: document.getElementById("selection-panel"), + viewportInfoEl: document.getElementById("viewport-info"), + zoomResetBtn: document.getElementById("zoom-reset"), + model, + onScopeSelect: (name) => { + timeline.setHighlightName(name); + stats.selectByName(name); + }, + }); + + const stats = new StatsView( + document.getElementById("stats-tbody"), + document.querySelector(".stats-table thead tr"), + model, + (name) => { + timeline.setHighlightName(name); + timeline.jumpToScopeName(name); + }, + ); + + const memoryView = new MemoryView(model, document.getElementById("memory-content")); + const logsView = new LogsView(model, document.getElementById("logs-content")); + const csvView = new CsvStatsView(model, document.getElementById("csv-content")); + + const threadsListApi = renderThreadsList(model, timeline); + renderRegionCategories(model, timeline); + setupTabs(memoryView, logsView, csvView); + setupSearch(model, timeline, stats); + + const bookmarksToggle = document.getElementById("bookmarks-toggle"); + bookmarksToggle.addEventListener("change", () => { + timeline.setBookmarksVisible(bookmarksToggle.checked); + }); + + const lodToggle = document.getElementById("lod-toggle"); + lodToggle.addEventListener("change", () => { + timeline.setLodEnabled(lodToggle.checked); + }); + + // Enable all threads that actually have captured scopes by default; if + // none do, enable every thread so the swimlanes still show up empty. + const withScopes = model.threads.filter((t) => t.scope_count > 0).map((t) => t.thread_id); + const initialEnabled = withScopes.length > 0 ? withScopes : model.threads.map((t) => t.thread_id); + for (const id of initialEnabled) { + const cb = document.querySelector(`.thread-row input[data-tid="${id}"]`); + if (cb) cb.checked = true; + } + threadsListApi.syncAllGroupCheckboxes(); + timeline.setEnabledThreads(initialEnabled); + + // "deselect all / select all" toggle for the Threads panel + const toggleAllBtn = document.getElementById("threads-toggle-all"); + const threadsList = document.getElementById("threads-list"); + toggleAllBtn.addEventListener("click", () => { + const allBoxes = threadsList.querySelectorAll(".thread-row input[type=checkbox]"); + const anyChecked = Array.from(allBoxes).some((cb) => cb.checked); + const newState = !anyChecked; + for (const cb of allBoxes) { + cb.checked = newState; + } + // Sync group checkboxes + for (const gcb of threadsList.querySelectorAll(".group-checkbox")) { + gcb.checked = newState; + gcb.indeterminate = false; + } + toggleAllBtn.textContent = newState ? "deselect all" : "select all"; + const enabled = []; + if (newState) { + for (const cb of allBoxes) { + enabled.push(Number(cb.dataset.tid)); + } + } + timeline.setEnabledThreads(enabled); + }); + + loadingEl.classList.add("hidden"); + } catch (e) { + loadingEl.textContent = `Failed to load trace: ${e.message}`; + console.error(e); + } +} + +function renderHeader(model) { + const { session } = model; + const hdrFile = document.getElementById("hdr-file"); + hdrFile.textContent = session.file_path || ""; + hdrFile.title = session.file_path || ""; + + const stats = document.getElementById("hdr-stats"); + stats.innerHTML = + `<span><span class="k">events:</span><span class="v"></span></span>` + + `<span><span class="k">threads:</span><span class="v"></span></span>` + + `<span><span class="k">duration:</span><span class="v"></span></span>` + + `<span><span class="k">parse:</span><span class="v"></span></span>`; + const vs = stats.querySelectorAll(".v"); + vs[0].textContent = formatNum(session.total_events || 0); + vs[1].textContent = formatNum(model.threads.length); + vs[2].textContent = formatTimeMs((session.trace_end_us || 0) - (session.trace_start_us || 0)); + vs[3].textContent = `${session.parse_time_ms} ms`; +} + +function renderSessionView(model) { + const { session, threads, channels } = model; + const el = document.getElementById("session-content"); + + const rows = []; + rows.push("<h2>Session</h2>"); + rows.push("<dl>"); + const row = (k, v) => v && rows.push(`<dt>${k}</dt><dd>${escapeHtml(v)}</dd>`); + row("File", session.file_path); + row("Size", `${formatNum(session.file_size)} bytes`); + row("Events", formatNum(session.total_events)); + row("Parse time", `${session.parse_time_ms} ms`); + row("Platform", session.platform); + row("App", session.app_name); + row("Project", session.project_name); + row("Branch", session.branch); + row("Build", session.build_version); + if (session.changelist) row("Changelist", String(session.changelist)); + row("Command line", session.command_line); + rows.push("</dl>"); + + rows.push("<h2>Threads</h2>"); + rows.push("<table><thead><tr>"); + rows.push(`<th>Name</th><th class="num">TID</th><th class="num">System ID</th><th class="num">Scopes</th>`); + rows.push("</tr></thead><tbody>"); + for (const t of threads) { + rows.push( + `<tr><td>${escapeHtml(t.name || "")}</td>` + + `<td class="num">${t.thread_id}</td>` + + `<td class="num">${t.system_id}</td>` + + `<td class="num">${formatNum(t.scope_count || 0)}</td></tr>`, + ); + } + rows.push("</tbody></table>"); + + if (channels && channels.length) { + rows.push("<h2>Trace channels</h2>"); + rows.push("<table><thead><tr><th>Name</th><th>State</th></tr></thead><tbody>"); + for (const c of channels) { + const cls = c.enabled ? "chan-enabled" : "chan-disabled"; + const ro = c.readonly ? `<span class="chan-readonly">read-only</span>` : ""; + rows.push(`<tr><td>${escapeHtml(c.name || "")}</td><td class="${cls}">${c.enabled ? "enabled" : "disabled"}${ro}</td></tr>`); + } + rows.push("</tbody></table>"); + } + + el.innerHTML = rows.join(""); +} + +function renderThreadsList(model, timeline) { + const list = document.getElementById("threads-list"); + const cmp = (a, b) => { + // SortHint first (UE sets low values for important threads like GameThread) + if (a.sort_hint !== b.sort_hint) return a.sort_hint - b.sort_hint; + // Then threads with scopes before empty ones + if ((a.scope_count > 0) !== (b.scope_count > 0)) return b.scope_count - a.scope_count; + // Then by thread ID (lower = created earlier, main thread is typically first) + if (a.thread_id !== b.thread_id) return a.thread_id - b.thread_id; + return (a.name || "").localeCompare(b.name || "", undefined, { numeric: true }); + }; + + // Build groups: ungrouped threads, named groups, and lanes + const lanes = model.threads.filter(t => t.is_lane).sort(cmp); + const grouped = new Map(); // groupName → [threads] + const ungrouped = []; + for (const t of model.threads) { + if (t.is_lane) continue; + const g = t.group || ""; + if (g) { + if (!grouped.has(g)) grouped.set(g, []); + grouped.get(g).push(t); + } else { + ungrouped.push(t); + } + } + ungrouped.sort(cmp); + for (const [, threads] of grouped) threads.sort(cmp); + + // Sort group names naturally + const groupNames = Array.from(grouped.keys()).sort((a, b) => + a.localeCompare(b, undefined, { numeric: true })); + + const collapsed = new Set(); + const parts = []; + + function renderGroup(label, threads, collapsible) { + if (threads.length === 0) return; + const collapseAttr = collapsible ? ` data-group="${label}"` : ""; + const chevron = collapsible ? `<span class="group-chevron">▾</span>` : ""; + const groupCb = collapsible ? `<input type="checkbox" class="group-checkbox" data-group-toggle="${label}">` : ""; + parts.push(`<div class="thread-group-header"${collapseAttr}>${groupCb}${chevron}${label} (${threads.length})</div>`); + for (const t of threads) { + const emptyCls = t.scope_count === 0 ? " empty" : ""; + const laneCls = t.is_lane ? " lane" : ""; + const groupAttr = collapsible ? ` data-group-member="${label}"` : ""; + parts.push( + `<label class="thread-row${emptyCls}${laneCls}"${groupAttr}>` + + `<input type="checkbox" data-tid="${t.thread_id}">` + + `<span class="thread-name"></span>` + + `<span class="thread-count">${formatNum(t.scope_count || 0)}</span>` + + `</label>`, + ); + } + } + + // Ungrouped threads first, then named groups, then lanes + if (ungrouped.length > 0 && (grouped.size > 0 || lanes.length > 0)) { + renderGroup("Threads", ungrouped, false); + } else { + // No groups at all — render without a header + for (const t of ungrouped) { + const emptyCls = t.scope_count === 0 ? " empty" : ""; + parts.push( + `<label class="thread-row${emptyCls}">` + + `<input type="checkbox" data-tid="${t.thread_id}">` + + `<span class="thread-name"></span>` + + `<span class="thread-count">${formatNum(t.scope_count || 0)}</span>` + + `</label>`, + ); + } + } + for (const name of groupNames) renderGroup(name, grouped.get(name), true); + renderGroup("Lanes", lanes, lanes.length > 0); + + list.innerHTML = parts.join(""); + + function syncTimeline() { + const enabled = new Set(); + for (const box of list.querySelectorAll(".thread-row input[type=checkbox]")) { + if (box.checked) enabled.add(Number(box.dataset.tid)); + } + timeline.setEnabledThreads(Array.from(enabled)); + } + + function syncGroupCheckbox(groupName) { + const members = list.querySelectorAll(`[data-group-member="${groupName}"] input[type=checkbox]`); + const gcb = list.querySelector(`.group-checkbox[data-group-toggle="${groupName}"]`); + if (!gcb || members.length === 0) return; + const checkedCount = Array.from(members).filter((c) => c.checked).length; + gcb.checked = checkedCount === members.length; + gcb.indeterminate = checkedCount > 0 && checkedCount < members.length; + } + + function syncAllGroupCheckboxes() { + for (const gcb of list.querySelectorAll(".group-checkbox")) { + syncGroupCheckbox(gcb.dataset.groupToggle); + } + } + + // Wire up collapsible group headers (click on the label area, not checkbox) + for (const hdr of list.querySelectorAll(".thread-group-header[data-group]")) { + hdr.addEventListener("click", (e) => { + // Don't collapse when clicking the group checkbox + if (e.target.classList.contains("group-checkbox")) return; + const group = hdr.dataset.group; + const isCollapsed = collapsed.has(group); + if (isCollapsed) { + collapsed.delete(group); + hdr.classList.remove("collapsed"); + } else { + collapsed.add(group); + hdr.classList.add("collapsed"); + } + for (const row of list.querySelectorAll(`[data-group-member="${group}"]`)) { + row.style.display = isCollapsed ? "" : "none"; + } + }); + } + + // Wire up group checkboxes — toggle all children + for (const gcb of list.querySelectorAll(".group-checkbox")) { + gcb.addEventListener("change", () => { + const group = gcb.dataset.groupToggle; + const checked = gcb.checked; + for (const row of list.querySelectorAll(`[data-group-member="${group}"]`)) { + const cb = row.querySelector("input[type=checkbox]"); + if (cb) cb.checked = checked; + } + syncTimeline(); + }); + } + + // Wire up thread checkboxes + for (const row of list.querySelectorAll(".thread-row")) { + const cb = row.querySelector("input"); + const nameEl = row.querySelector(".thread-name"); + const tid = Number(cb.dataset.tid); + const thread = model.threads.find((t) => t.thread_id === tid); + nameEl.textContent = (thread && thread.name) || `tid ${tid}`; + cb.addEventListener("change", () => { + const groupMember = row.dataset.groupMember; + if (groupMember) syncGroupCheckbox(groupMember); + syncTimeline(); + }); + } + + return { syncAllGroupCheckboxes }; +} + +function renderRegionCategories(model, timeline) { + const categories = model.regionCategories || []; + if (categories.length === 0) return; + + const panel = document.getElementById("regions-panel"); + panel.hidden = false; + + const list = document.getElementById("regions-list"); + const parts = []; + for (let i = 0; i < categories.length; i++) { + const cat = categories[i]; + const count = cat.regions ? cat.regions.length : 0; + parts.push( + `<label class="thread-row">` + + `<input type="checkbox" data-cat-idx="${i}" checked>` + + `<span class="thread-name"></span>` + + `<span class="thread-count">${formatNum(count)}</span>` + + `</label>`, + ); + } + list.innerHTML = parts.join(""); + + // Set label text via DOM to avoid XSS + for (const row of list.querySelectorAll(".thread-row")) { + const cb = row.querySelector("input"); + const nameEl = row.querySelector(".thread-name"); + const idx = Number(cb.dataset.catIdx); + nameEl.textContent = categories[idx].name || "Uncategorized"; + } + + function syncTimeline() { + const enabled = new Set(); + for (const cb of list.querySelectorAll("input[type=checkbox]")) { + if (cb.checked) enabled.add(Number(cb.dataset.catIdx)); + } + timeline.setEnabledRegionCategories(enabled); + } + + for (const cb of list.querySelectorAll("input[type=checkbox]")) { + cb.addEventListener("change", syncTimeline); + } + + // "deselect all / select all" toggle + const toggleBtn = document.getElementById("regions-toggle-all"); + toggleBtn.addEventListener("click", () => { + const allBoxes = list.querySelectorAll("input[type=checkbox]"); + const anyChecked = Array.from(allBoxes).some((cb) => cb.checked); + const newState = !anyChecked; + for (const cb of allBoxes) cb.checked = newState; + toggleBtn.textContent = newState ? "deselect all" : "select all"; + syncTimeline(); + }); + + // Initial state: all enabled + const allIndices = new Set(categories.map((_, i) => i)); + timeline.setEnabledRegionCategories(allIndices); +} + +function setupTabs(memoryView, logsView, csvView) { + const tabs = document.querySelectorAll(".tab"); + const views = document.querySelectorAll(".view"); + const validTabs = new Set(Array.from(tabs, (tab) => tab.dataset.tab)); + + function activateTab(key, updateUrl = true) { + for (const tab of tabs) { + tab.classList.toggle("active", tab.dataset.tab === key); + } + for (const view of views) { + view.hidden = view.dataset.view !== key; + } + if (updateUrl) { + const url = new URL(window.location.href); + url.searchParams.set("tab", key); + window.history.replaceState({}, "", url); + } + if (key === "memory" && memoryView) { + memoryView.ensureLoaded(); + } + if (key === "logs" && logsView) { + logsView.ensureLoaded(); + } + if (key === "csv" && csvView) { + csvView.ensureLoaded(); + } + } + + for (const tab of tabs) { + tab.addEventListener("click", () => activateTab(tab.dataset.tab)); + } + + const initialTab = new URLSearchParams(window.location.search).get("tab"); + if (initialTab && validTabs.has(initialTab)) { + activateTab(initialTab, false); + } +} + +function setupSearch(model, timeline, stats) { + const input = document.getElementById("search-input"); + const results = document.getElementById("search-results"); + + function render() { + const q = input.value.trim().toLowerCase(); + if (!q) { + results.innerHTML = ""; + timeline.setHighlightName(null); + return; + } + const matches = []; + for (const s of model.scopeStats) { + if (s.name.toLowerCase().includes(q)) { + matches.push(s); + if (matches.length >= 50) break; + } + } + const parts = []; + for (const m of matches) { + parts.push( + `<div class="hit" data-name="${escapeHtml(m.name)}">` + + `<span class="hit-name">${escapeHtml(m.name)}</span>` + + `<span class="hit-count">${formatNum(m.count)}</span>` + + `</div>`, + ); + } + results.innerHTML = parts.join(""); + for (const hit of results.querySelectorAll(".hit")) { + hit.addEventListener("click", () => { + const name = hit.dataset.name; + timeline.setHighlightName(name); + timeline.jumpToScopeName(name); + stats.selectByName(name); + }); + } + if (matches.length > 0) { + timeline.setHighlightName(matches[0].name); + } + } + + input.addEventListener("input", render); +} + +main(); diff --git a/src/zen/progressbar.cpp b/src/zen/progressbar.cpp deleted file mode 100644 index 6581cd116..000000000 --- a/src/zen/progressbar.cpp +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -// Zen command line client utility -// - -#include "progressbar.h" - -#include <zencore/logging.h> -#include <zencore/windows.h> -#include <zenremotestore/operationlogoutput.h> -#include <zenutil/consoletui.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <gsl/gsl-lite.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -////////////////////////////////////////////////////////////////////////// - -namespace zen { - -#if ZEN_PLATFORM_WINDOWS -static HANDLE -GetConsoleHandle() -{ - static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); - return hStdOut; -} -#endif - -static void -OutputToConsoleRaw(const char* String, size_t Length) -{ -#if ZEN_PLATFORM_WINDOWS - HANDLE hStdOut = GetConsoleHandle(); - if (TuiIsStdoutTty()) - { - WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); - } - else - { - ::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0); - } -#else - fwrite(String, 1, Length, stdout); -#endif -} - -static void -OutputToConsoleRaw(const std::string& String) -{ - OutputToConsoleRaw(String.c_str(), String.length()); -} - -static void -OutputToConsoleRaw(const StringBuilderBase& SB) -{ - OutputToConsoleRaw(SB.c_str(), SB.Size()); -} - -uint32_t -GetUpdateDelayMS(ProgressBar::Mode InMode) -{ - switch (InMode) - { - case ProgressBar::Mode::Plain: - return 5000; - case ProgressBar::Mode::Pretty: - return 200; - case ProgressBar::Mode::Log: - return 2000; - default: - ZEN_ASSERT(false); - return 0; - } -} - -void -ProgressBar::SetLogOperationName(Mode InMode, std::string_view Name) -{ - ZEN_ASSERT(Name.find('\"') == std::string_view::npos); - if (InMode == Mode::Log) - { - std::string String = fmt::format("@progress \"{}\"\n", Name); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount) -{ - if (InMode == Mode::Log) - { - const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u; - - std::string String = fmt::format("@progress {}%\n", PercentDone); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::PushLogOperation(Mode InMode, std::string_view Name) -{ - if (InMode == Mode::Log) - { - std::string String = fmt::format("@progress push \"{}\"\n", Name); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::PopLogOperation(Mode InMode) -{ - if (InMode == Mode::Log) - { - const std::string String("@progress pop\n"); - OutputToConsoleRaw(String); - } -} - -ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask) -: m_Mode((!TuiIsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) -, m_LastUpdateMS((uint64_t)-1) -, m_PausedMS(0) -, m_SubTask(InSubTask) -{ - ZEN_ASSERT(InSubTask.find('\"') == std::string_view::npos); - if (!m_SubTask.empty()) - { - PushLogOperation(InMode, m_SubTask); - } -} - -ProgressBar::~ProgressBar() -{ - try - { - ForceLinebreak(); - if (!m_SubTask.empty()) - { - PopLogOperation(m_Mode); - } - } - catch (const std::exception& Ex) - { - ZEN_ERROR("ProgressBar::~ProgressBar() failed with {}", Ex.what()); - } -} - -void -ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) -{ - ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount); - ZEN_ASSERT(NewState.Task.find('\"') == std::string::npos); - if (DoLinebreak == false && m_State == NewState) - { - return; - } - - uint64_t ElapsedTimeMS = NewState.OptionalElapsedTime == (uint64_t)-1 ? m_SW.GetElapsedTimeMs() : NewState.OptionalElapsedTime; - if (m_LastUpdateMS != (uint64_t)-1) - { - if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) && - ((m_LastUpdateMS + 200) > ElapsedTimeMS)) - { - return; - } - if (m_State.Status == State::EStatus::Paused) - { - uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS; - m_PausedMS += ElapsedSinceLast; - } - } - - m_LastUpdateMS = ElapsedTimeMS; - - std::string Task = NewState.Task; - switch (NewState.Status) - { - case State::EStatus::Aborted: - Task = "Aborting"; - break; - case State::EStatus::Paused: - Task = "Paused"; - break; - default: - break; - } - if (NewState.Task.length() > Task.length()) - { - Task += std::string(NewState.Task.length() - Task.length(), ' '); - } - - const size_t PercentDone = - NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; - - uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; - uint64_t ETAElapsedMS = ElapsedTimeMS - m_PausedMS; - uint64_t ETAMS = ((m_State.TotalCount == NewState.TotalCount) && (NewState.Status == State::EStatus::Running)) && (PercentDone > 5) - ? (ETAElapsedMS * NewState.RemainingCount) / Completed - : 0; - const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; - - if (m_Mode == Mode::Plain) - { - const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - const std::string Output = fmt::format("{} {}% {}{}{}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); - OutputToConsoleRaw(Output); - m_State = NewState; - } - else if (m_Mode == Mode::Pretty) - { - size_t ProgressBarSize = 20; - - size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; - - uint32_t ConsoleColumns = TuiConsoleColumns(1024); - - const std::string PercentString = fmt::format("{:#3}%", PercentDone); - - const std::string ProgressBarString = - fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' ')); - - const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS)); - - const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : ""; - - ExtendableStringBuilder<256> OutputBuilder; - - OutputBuilder << "\r" << Task << " " << PercentString; - if (OutputBuilder.Size() + 1 < ConsoleColumns) - { - size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); - bool ElapsedFits = RemainingSpace >= ElapsedString.length(); - RemainingSpace -= ElapsedString.length(); - bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length(); - RemainingSpace -= ETAString.length(); - bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length(); - RemainingSpace -= DetailsString.length(); - bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length(); - RemainingSpace -= ProgressBarString.length(); - - if (ProgressBarFits) - { - OutputBuilder << ProgressBarString; - } - if (ElapsedFits) - { - OutputBuilder << ElapsedString; - } - if (ETAFits) - { - OutputBuilder << ETAString; - } - if (DetailsFits) - { - OutputBuilder << DetailsString; - } - } - - std::string_view Output = OutputBuilder.ToView(); - std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0; - - ExtendableStringBuilder<256> LineToPrint; - - if (Output.length() + EraseLength >= ConsoleColumns) - { - if (m_LastOutputLength > 0) - { - LineToPrint << "\n"; - } - LineToPrint << Output.substr(1); - DoLinebreak = true; - } - else - { - LineToPrint << Output << std::string(EraseLength, ' '); - } - - if (DoLinebreak) - { - LineToPrint << "\n"; - } - - OutputToConsoleRaw(LineToPrint); - - m_LastOutputLength = DoLinebreak ? 0 : Output.length(); - m_State = NewState; - } - else if (m_Mode == Mode::Log) - { - if (m_State.Task != NewState.Task || - m_State.Details != NewState.Details) // TODO: Should we output just because details change? Will this spam the log collector? - { - std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - for (std::string::value_type& Char : Details) - { - if (Char == '"') - { - Char = '\''; - } - } - const std::string Message = - fmt::format("@progress \"{} {}{}{}\"\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); - OutputToConsoleRaw(Message); - } - - const size_t OldPercentDone = - m_State.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u; - - if (OldPercentDone != PercentDone) - { - const std::string Progress = fmt::format("@progress {}%\n", PercentDone); - OutputToConsoleRaw(Progress); - } - m_State = NewState; - } -} - -void -ProgressBar::ForceLinebreak() -{ - if (m_LastOutputLength > 0) - { - State NewState = m_State; - UpdateState(NewState, /*DoLinebreak*/ true); - } -} - -void -ProgressBar::Finish() -{ - if (m_LastOutputLength > 0 || m_State.RemainingCount > 0) - { - State NewState = m_State; - NewState.RemainingCount = 0; - NewState.Details = ""; - UpdateState(NewState, /*DoLinebreak*/ true); - } - m_State = State{}; - m_LastOutputLength = 0; - m_SW.Reset(); -} - -bool -ProgressBar::IsSameTask(std::string_view Task) const -{ - return Task == m_State.Task; -} - -bool -ProgressBar::HasActiveTask() const -{ - return !m_State.Task.empty(); -} - -class ConsoleOpLogProgressBar : public OperationLogOutput::ProgressBar -{ -public: - ConsoleOpLogProgressBar(zen::ProgressBar::Mode InMode, std::string_view InSubTask) : m_Inner(InMode, InSubTask) {} - - virtual void UpdateState(const State& NewState, bool DoLinebreak) - { - zen::ProgressBar::State State = {.Task = NewState.Task, - .Details = NewState.Details, - .TotalCount = NewState.TotalCount, - .RemainingCount = NewState.RemainingCount, - .Status = ConvertStatus(NewState.Status)}; - m_Inner.UpdateState(State, DoLinebreak); - } - virtual void Finish() { m_Inner.Finish(); } - -private: - zen::ProgressBar::State::EStatus ConvertStatus(State::EStatus Status) - { - switch (Status) - { - case State::EStatus::Running: - return zen::ProgressBar::State::EStatus::Running; - case State::EStatus::Aborted: - return zen::ProgressBar::State::EStatus::Aborted; - case State::EStatus::Paused: - return zen::ProgressBar::State::EStatus::Paused; - default: - return (zen::ProgressBar::State::EStatus)Status; - } - } - zen::ProgressBar m_Inner; -}; - -class ConsoleOpLogOutput : public OperationLogOutput -{ -public: - ConsoleOpLogOutput(zen::ProgressBar::Mode InMode) : m_Mode(InMode) {} - virtual void EmitLogMessage(const logging::LogPoint& Point, fmt::format_args Args) override - { - logging::EmitConsoleLogMessage(Point, Args); - } - - virtual void SetLogOperationName(std::string_view Name) override { zen::ProgressBar::SetLogOperationName(m_Mode, Name); } - virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override - { - zen::ProgressBar::SetLogOperationProgress(m_Mode, StepIndex, StepCount); - } - virtual uint32_t GetProgressUpdateDelayMS() override { return GetUpdateDelayMS(m_Mode); } - - virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override { return new ConsoleOpLogProgressBar(m_Mode, InSubTask); } - -private: - zen::ProgressBar::Mode m_Mode; -}; - -OperationLogOutput* -CreateConsoleLogOutput(ProgressBar::Mode InMode) -{ - return new ConsoleOpLogOutput(InMode); -} - -} // namespace zen diff --git a/src/zen/progressbar.h b/src/zen/progressbar.h deleted file mode 100644 index b54c009e1..000000000 --- a/src/zen/progressbar.h +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/timer.h> -#include <zencore/zencore.h> - -#include <string> - -namespace zen { - -class OperationLogOutput; - -class ProgressBar -{ -public: - struct State - { - bool operator==(const State&) const = default; - std::string Task; - std::string Details; - uint64_t TotalCount = 0; - uint64_t RemainingCount = 0; - uint64_t OptionalElapsedTime = (uint64_t)-1; - enum class EStatus - { - Running, - Aborted, - Paused - }; - EStatus Status = EStatus::Running; - - static EStatus CalculateStatus(bool IsAborted, bool IsPaused) - { - if (IsAborted) - { - return EStatus::Aborted; - } - if (IsPaused) - { - return EStatus::Paused; - } - return EStatus::Running; - } - }; - - enum class Mode - { - Plain, - Pretty, - Log, - Quiet - }; - - static void SetLogOperationName(Mode InMode, std::string_view Name); - static void SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount); - static void PushLogOperation(Mode InMode, std::string_view Name); - static void PopLogOperation(Mode InMode); - - explicit ProgressBar(Mode InMode, std::string_view InSubTask); - ~ProgressBar(); - - void UpdateState(const State& NewState, bool DoLinebreak); - void ForceLinebreak(); - void Finish(); - bool IsSameTask(std::string_view Task) const; - bool HasActiveTask() const; - -private: - const Mode m_Mode; - Stopwatch m_SW; - uint64_t m_LastUpdateMS; - uint64_t m_PausedMS; - State m_State; - const std::string m_SubTask; - size_t m_LastOutputLength = 0; -}; - -uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode); - -OperationLogOutput* CreateConsoleLogOutput(ProgressBar::Mode InMode); - -} // namespace zen diff --git a/src/zen/trace/callstack_formatter.cpp b/src/zen/trace/callstack_formatter.cpp new file mode 100644 index 000000000..0c601d5c0 --- /dev/null +++ b/src/zen/trace/callstack_formatter.cpp @@ -0,0 +1,251 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "callstack_formatter.h" + +#include <zencore/fmtutils.h> +#include <zencore/string.h> +#include <zenutil/wildcard.h> + +#include <algorithm> + +namespace zen::trace_detail { + +namespace { + + static bool IsKnownRuntimeModuleName(std::string_view ModuleName) + { + std::string LowerName = zen::ToLower(ModuleName); + return LowerName == "ntdll.dll" || LowerName == "kernel32.dll" || LowerName == "kernelbase.dll" || LowerName == "ucrtbase.dll" || + LowerName == "ucrtbased.dll" || LowerName.starts_with("vcruntime") || LowerName.starts_with("msvcp") || + LowerName.starts_with("api-ms-win-") || LowerName == "libc.so" || LowerName.starts_with("libc.so.") || + LowerName == "libstdc++.so" || LowerName.starts_with("libstdc++.so.") || LowerName == "libgcc_s.so" || + LowerName.starts_with("libgcc_s.so.") || LowerName == "libpthread.so" || LowerName.starts_with("libpthread.so.") || + LowerName == "libm.so" || LowerName.starts_with("libm.so.") || LowerName == "ld-linux.so" || + LowerName.starts_with("ld-linux") || LowerName == "libsystem_kernel.dylib" || LowerName == "libsystem_malloc.dylib" || + LowerName == "libsystem_pthread.dylib" || LowerName == "libdyld.dylib"; + } + + static bool PathLooksThirdParty(std::string_view ModulePath) + { + std::string LowerPath = zen::ToLower(ModulePath); + return LowerPath.find("/thirdparty/") != std::string::npos || LowerPath.find("\\thirdparty\\") != std::string::npos || + LowerPath.find("/third-party/") != std::string::npos || LowerPath.find("\\third-party\\") != std::string::npos || + LowerPath.find("/external/") != std::string::npos || LowerPath.find("\\external\\") != std::string::npos || + LowerPath.find("/extern/") != std::string::npos || LowerPath.find("\\extern\\") != std::string::npos || + LowerPath.find("/engine/binaries/thirdparty/") != std::string::npos || + LowerPath.find("\\engine\\binaries\\thirdparty\\") != std::string::npos || + LowerPath.find("c:\\windows\\system32\\") != std::string::npos || + LowerPath.find("c:\\windows\\syswow64\\") != std::string::npos || LowerPath.starts_with("/usr/lib/") || + LowerPath.starts_with("/lib/") || LowerPath.starts_with("/system/"); + } + + static bool MatchesAnyPattern(std::string_view Text, const std::vector<std::string>& Patterns) + { + for (const std::string& Pattern : Patterns) + { + if (zen::MatchWildcard(Pattern, Text, /*CaseSensitive=*/false)) + { + return true; + } + } + return false; + } + + static bool ShouldSkipFrameByPattern(const CallstackFilterOptions& Options, + const TraceModel& Model, + const ResolvedFrame& Frame, + std::string_view Description) + { + if (MatchesAnyPattern(Description, Options.SkipPatterns)) + { + return true; + } + if (Frame.ModuleIndex != ~0u && Frame.ModuleIndex < Model.Modules.size()) + { + const ModuleInfo& Module = Model.Modules[Frame.ModuleIndex]; + if (MatchesAnyPattern(Module.Name, Options.SkipPatterns) || MatchesAnyPattern(Module.FullPath, Options.SkipPatterns)) + { + return true; + } + } + + static const std::vector<std::string> kDefaultSkipPatterns = { + "zen::MemoryTrace_*", + "*mi_*", + "*_mi_*", + "*rpmalloc*", + "*mimalloc*", + "*je_malloc*", + "*je_free*", + "*malloc*", + "*free*", + "*realloc*", + }; + return MatchesAnyPattern(Description, kDefaultSkipPatterns); + } + + static bool IsThirdPartyFrame(const TraceModel& Model, const ResolvedFrame& Frame, std::string_view Description) + { + if (Description.starts_with("std::")) + { + return true; + } + if (Frame.ModuleIndex == ~0u || Frame.ModuleIndex >= Model.Modules.size()) + { + return false; + } + + const ModuleInfo& Module = Model.Modules[Frame.ModuleIndex]; + return IsKnownRuntimeModuleName(Module.Name) || PathLooksThirdParty(Module.FullPath); + } + +} // namespace + +CallstackFormatter::CallstackFormatter(const TraceModel& InModel, const SymbolResolver* InSymbols) : m_Model(InModel), m_Symbols(InSymbols) +{ +} + +const CallstackEntry* +CallstackFormatter::FindCallstackEntry(uint32_t CallstackId) const +{ + auto It = + eastl::lower_bound(m_Model.Callstacks.begin(), m_Model.Callstacks.end(), CallstackId, [](const CallstackEntry& E, uint32_t Id) { + return E.Id < Id; + }); + if (It == m_Model.Callstacks.end() || It->Id != CallstackId) + { + return nullptr; + } + return &*It; +} + +const std::string& +CallstackFormatter::Describe(const ResolvedFrame& Frame) +{ + auto It = m_Cache.find(Frame.Address); + if (It != m_Cache.end()) + { + return It->second; + } + + std::string Result = m_Symbols ? m_Symbols->Resolve(Frame.Address) : std::string{}; + if (Result.empty()) + { + if (Frame.ModuleIndex != ~0u && Frame.ModuleIndex < m_Model.Modules.size()) + { + Result = fmt::format("{} + 0x{:X}", m_Model.Modules[Frame.ModuleIndex].Name, Frame.Offset); + } + else + { + Result = fmt::format("0x{:X}", Frame.Address); + } + } + + auto [InsertedIt, Inserted] = m_Cache.emplace(Frame.Address, std::move(Result)); + ZEN_UNUSED(Inserted); + return InsertedIt->second; +} + +FilteredCallstackView +CallstackFormatter::BuildView(const CallstackEntry& Entry, const CallstackFilterOptions& Options) +{ + FilteredCallstackView Result; + Result.Frames.reserve(Entry.Frames.size()); + if (Entry.Frames.empty()) + { + return Result; + } + + eastl::vector<bool> ExplicitSkip; + eastl::vector<bool> ThirdParty; + ExplicitSkip.reserve(Entry.Frames.size()); + ThirdParty.reserve(Entry.Frames.size()); + + for (const ResolvedFrame& Frame : Entry.Frames) + { + const std::string& Description = Describe(Frame); + ExplicitSkip.push_back(ShouldSkipFrameByPattern(Options, m_Model, Frame, Description)); + ThirdParty.push_back(IsThirdPartyFrame(m_Model, Frame, Description)); + } + + eastl::vector<size_t> VisibleFrameIndices; + VisibleFrameIndices.reserve(Entry.Frames.size()); + + if (!Options.EnableHeuristic) + { + for (size_t Index = 0; Index < Entry.Frames.size(); ++Index) + { + if (!ExplicitSkip[Index]) + { + VisibleFrameIndices.push_back(Index); + } + } + if (VisibleFrameIndices.empty()) + { + VisibleFrameIndices.push_back(0); + } + } + else + { + size_t FirstProgramIndex = Entry.Frames.size(); + size_t BoundaryThirdPartyIndex = Entry.Frames.size(); + for (size_t Index = 0; Index < Entry.Frames.size(); ++Index) + { + if (ExplicitSkip[Index]) + { + continue; + } + if (ThirdParty[Index]) + { + BoundaryThirdPartyIndex = Index; + continue; + } + FirstProgramIndex = Index; + break; + } + + if (FirstProgramIndex == Entry.Frames.size()) + { + for (size_t Index = 0; Index < Entry.Frames.size(); ++Index) + { + if (!ExplicitSkip[Index]) + { + VisibleFrameIndices.push_back(Index); + } + } + if (VisibleFrameIndices.empty()) + { + VisibleFrameIndices.push_back(0); + } + } + else + { + if (BoundaryThirdPartyIndex < Entry.Frames.size()) + { + VisibleFrameIndices.push_back(BoundaryThirdPartyIndex); + Result.IncludedThirdPartyBoundary = true; + } + for (size_t Index = FirstProgramIndex; Index < Entry.Frames.size(); ++Index) + { + if (!ExplicitSkip[Index]) + { + VisibleFrameIndices.push_back(Index); + } + } + if (VisibleFrameIndices.empty()) + { + VisibleFrameIndices.push_back(FirstProgramIndex); + } + } + } + + Result.HiddenPrefixCount = uint32_t(VisibleFrameIndices.front()); + for (size_t FrameIndex : VisibleFrameIndices) + { + Result.Frames.push_back( + {.OriginalIndex = FrameIndex, .Frame = &Entry.Frames[FrameIndex], .Display = Describe(Entry.Frames[FrameIndex])}); + } + return Result; +} + +} // namespace zen::trace_detail diff --git a/src/zen/trace/callstack_formatter.h b/src/zen/trace/callstack_formatter.h new file mode 100644 index 000000000..067985f25 --- /dev/null +++ b/src/zen/trace/callstack_formatter.h @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "symbol_resolver.h" +#include "trace_model.h" + +#include <string> +#include <vector> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/hash_map.h> +#include <EASTL/vector.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::trace_detail { + +struct CallstackFilterOptions +{ + bool EnableHeuristic = true; + std::vector<std::string> SkipPatterns; +}; + +struct FilteredCallstackFrame +{ + size_t OriginalIndex = 0; + const ResolvedFrame* Frame = nullptr; + std::string Display; +}; + +struct FilteredCallstackView +{ + eastl::vector<FilteredCallstackFrame> Frames; + uint32_t HiddenPrefixCount = 0; + bool IncludedThirdPartyBoundary = false; +}; + +class CallstackFormatter +{ +public: + CallstackFormatter(const TraceModel& InModel, const SymbolResolver* InSymbols); + + const eastl::hash_map<uint64_t, std::string>& GetResolvedCache() const { return m_Cache; } + + const CallstackEntry* FindCallstackEntry(uint32_t CallstackId) const; + const std::string& Describe(const ResolvedFrame& Frame); + FilteredCallstackView BuildView(const CallstackEntry& Entry, const CallstackFilterOptions& Options); + +private: + const TraceModel& m_Model; + const SymbolResolver* m_Symbols = nullptr; + eastl::hash_map<uint64_t, std::string> m_Cache; +}; + +} // namespace zen::trace_detail diff --git a/src/zen/trace/symbol_resolver.cpp b/src/zen/trace/symbol_resolver.cpp new file mode 100644 index 000000000..53374cd64 --- /dev/null +++ b/src/zen/trace/symbol_resolver.cpp @@ -0,0 +1,1631 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "symbol_resolver.h" + +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/process.h> +#include <zencore/string.h> +#include <zenhttp/httpclient.h> + +#include <algorithm> +#include <filesystem> +#include <mutex> +#include <unordered_map> +#include <vector> + +#if !ZEN_PLATFORM_WINDOWS +# include <cerrno> +# include <unistd.h> +#endif + +#if ZEN_PLATFORM_WINDOWS + +ZEN_THIRD_PARTY_INCLUDES_START +# include <Foundation/PDB_PointerUtil.h> +# include <PDB.h> +# include <PDB_CoalescedMSFStream.h> +# include <PDB_DBIStream.h> +# include <PDB_ImageSectionStream.h> +# include <PDB_InfoStream.h> +# include <PDB_ModuleInfoStream.h> +# include <PDB_ModuleLineStream.h> +# include <PDB_ModuleSymbolStream.h> +# include <PDB_PublicSymbolStream.h> +# include <PDB_RawFile.h> +ZEN_THIRD_PARTY_INCLUDES_END + +# include <zencore/windows.h> + +ZEN_THIRD_PARTY_INCLUDES_START +# include <DbgHelp.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#endif // ZEN_PLATFORM_WINDOWS + +namespace zen::trace_detail { + +////////////////////////////////////////////////////////////////////////////// +// Null resolver (used when symbolication is off or unsupported) + +class NullSymbolResolver final : public SymbolResolver +{ +public: + void LoadModule(const ModuleInfo&) override {} + std::string Resolve(uint64_t) const override { return {}; } +}; + +#if ZEN_PLATFORM_WINDOWS + +////////////////////////////////////////////////////////////////////////////// +// Helpers shared by Windows backends + +static std::string +FormatSymbol(std::string_view Name, uint64_t Displacement) +{ + if (Displacement == 0) + { + return std::string(Name); + } + return fmt::format("{} + 0x{:X}", Name, Displacement); +} + +////////////////////////////////////////////////////////////////////////////// +// Memory-mapped file helper + +namespace { + + struct MappedFile + { + const void* Data = nullptr; + size_t Size = 0; + HANDLE FileHandle = INVALID_HANDLE_VALUE; + HANDLE MappingHandle = nullptr; + + MappedFile() = default; + ~MappedFile() { Close(); } + + bool Open(const std::filesystem::path& Path) + { + FileHandle = CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (FileHandle == INVALID_HANDLE_VALUE) + { + return false; + } + + LARGE_INTEGER FileSize; + if (!GetFileSizeEx(FileHandle, &FileSize) || FileSize.QuadPart == 0) + { + Close(); + return false; + } + + MappingHandle = CreateFileMappingW(FileHandle, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (MappingHandle == nullptr) + { + Close(); + return false; + } + + Data = MapViewOfFile(MappingHandle, FILE_MAP_READ, 0, 0, 0); + if (Data == nullptr) + { + Close(); + return false; + } + + Size = size_t(FileSize.QuadPart); + return true; + } + + void Close() + { + if (Data != nullptr) + { + UnmapViewOfFile(Data); + Data = nullptr; + } + if (MappingHandle != nullptr) + { + CloseHandle(MappingHandle); + MappingHandle = nullptr; + } + if (FileHandle != INVALID_HANDLE_VALUE) + { + CloseHandle(FileHandle); + FileHandle = INVALID_HANDLE_VALUE; + } + Size = 0; + } + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + }; + + // Format an ImageId (16-byte GUID + 4-byte Age) as the hex key used in + // symbol server URLs: <GUID_no_dashes><Age_hex>, e.g. "A1B2C3...1". + std::string FormatImageIdKey(const eastl::vector<uint8_t>& ImageId) + { + if (ImageId.size() < 20) + { + return {}; + } + + // GUID bytes are stored as {Data1 LE, Data2 LE, Data3 LE, Data4[8]}. + // The symbol server key encodes Data1/2/3 as big-endian hex. + const uint8_t* G = ImageId.data(); + + uint32_t Data1; + uint16_t Data2; + uint16_t Data3; + memcpy(&Data1, G + 0, 4); + memcpy(&Data2, G + 4, 2); + memcpy(&Data3, G + 6, 2); + + uint32_t Age; + memcpy(&Age, ImageId.data() + 16, 4); + + return fmt::format("{:08X}{:04X}{:04X}{:02X}{:02X}{:02X}{:02X}{:02X}{:02X}{:02X}{:02X}{:x}", + Data1, + Data2, + Data3, + G[8], + G[9], + G[10], + G[11], + G[12], + G[13], + G[14], + G[15], + Age); + } + + // PdbName originates from module metadata in an untrusted trace file and is used + // to build both a filesystem path and an HTTP request path. Restrict it to a safe + // subset so a malicious trace cannot traverse out of the cache dir, inject URL + // syntax, or trip cross-platform path parsing quirks (e.g. `\` is a separator on + // Windows but not POSIX, so filename() doesn't always strip it). + bool IsSafePdbName(std::string_view Name) + { + constexpr AsciiSet SafeChars( + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + "._-+"); + + if (Name.empty() || Name.size() > 255 || Name == "." || Name == "..") + { + return false; + } + return AsciiSet::HasOnly(Name, SafeChars); + } + + const std::filesystem::path& GetSymbolCacheDir() + { + // Use %TEMP%/zen-symbols as the default cache location. + static const std::filesystem::path s_CacheDir = [] { + std::filesystem::path TempDir = std::filesystem::temp_directory_path(); + return TempDir / "zen-symbols"; + }(); + return s_CacheDir; + } + + // Try to download a PDB from a symbol server URL. + // Returns the local cache path on success, empty path on failure. + std::filesystem::path DownloadPdb(std::string_view ServerUrl, + std::string_view PdbName, + const std::string& ImageIdKey, + const std::filesystem::path& CacheDir) + { + // Cache path mirrors the server structure + std::filesystem::path CachePath = CacheDir / PdbName / ImageIdKey / PdbName; + + // Already cached? + std::error_code Ec; + if (std::filesystem::exists(CachePath, Ec)) + { + return CachePath; + } + + ZEN_INFO("Downloading {} from symbol server...", PdbName); + + try + { + std::string RequestPath = fmt::format("/{}/{}/{}", PdbName, ImageIdKey, PdbName); + + zen::HttpClientSettings Settings; + Settings.Timeout = std::chrono::milliseconds(30000); + Settings.ConnectTimeout = std::chrono::milliseconds(5000); + Settings.FollowRedirects = true; + Settings.ExpectedErrorCodes = {zen::HttpResponseCode::NotFound}; + + zen::HttpClient Http(ServerUrl, Settings); + + zen::HttpClient::Response Response = Http.Get(RequestPath); + + if (!Response) + { + ZEN_DEBUG("Symbol server: {} not found (HTTP {})", PdbName, int(Response.StatusCode)); + return {}; + } + + // Write to cache using zencore file I/O + std::filesystem::create_directories(CachePath.parent_path(), Ec); + zen::WriteFile(CachePath, Response.ResponsePayload); + + ZEN_INFO("Cached {} ({})", PdbName, zen::NiceBytes(Response.ResponsePayload.GetSize())); + return CachePath; + } + catch (const std::exception& Ex) + { + ZEN_WARN("Symbol server download failed for {}: {}", PdbName, Ex.what()); + return {}; + } + } + + // Parse _NT_SYMBOL_PATH to extract symbol server URLs. + // Format: "srv*<cache>*<url>" or "symsrv*symsrv.dll*<cache>*<url>" or just a URL. + // Returns a list of server URLs to try. + const std::vector<std::string>& ParseSymbolPath() + { + static const std::vector<std::string> s_Servers = [] { + std::vector<std::string> Servers; + + const char* EnvPath = std::getenv("_NT_SYMBOL_PATH"); + if (EnvPath == nullptr || EnvPath[0] == '\0') + { + // Default to Microsoft public symbol server + Servers.push_back("https://msdl.microsoft.com/download/symbols"); + return Servers; + } + + std::string_view Path(EnvPath); + + // Split on ';' for multiple entries + while (!Path.empty()) + { + size_t Semi = Path.find(';'); + std::string_view Entry = (Semi != std::string_view::npos) ? Path.substr(0, Semi) : Path; + Path = (Semi != std::string_view::npos) ? Path.substr(Semi + 1) : std::string_view{}; + + // Look for srv* or symsrv* prefix — the last '*'-delimited token is the server URL. + if (Entry.substr(0, 4) == "srv*" || Entry.substr(0, 7) == "symsrv*") + { + size_t LastStar = Entry.rfind('*'); + if (LastStar != std::string_view::npos && LastStar + 1 < Entry.size()) + { + std::string_view Url = Entry.substr(LastStar + 1); + if (Url.substr(0, 4) == "http") + { + Servers.emplace_back(Url); + } + } + } + } + + if (Servers.empty()) + { + Servers.push_back("https://msdl.microsoft.com/download/symbols"); + } + + return Servers; + }(); + return s_Servers; + } + + // Copy a local PDB into the symbol cache so that future analysis of traces + // from this build succeeds even after the binary is recompiled. + void CacheLocalPdb(const std::filesystem::path& PdbPath, + std::string_view PdbName, + const std::string& ImageIdKey, + const std::filesystem::path& CacheDir) + { + std::filesystem::path CachePath = CacheDir / PdbName / ImageIdKey / PdbName; + std::error_code Ec; + if (std::filesystem::exists(CachePath, Ec)) + { + return; + } + + std::filesystem::create_directories(CachePath.parent_path(), Ec); + if (Ec) + { + return; + } + + std::filesystem::copy_file(PdbPath, CachePath, std::filesystem::copy_options::skip_existing, Ec); + if (!Ec) + { + uint64_t Size = std::filesystem::file_size(PdbPath, Ec); + ZEN_INFO("Cached local PDB {} ({})", PdbName, zen::NiceBytes(Size)); + } + } + + // Look for a PDB in the local symbol cache or download from symbol servers. + // Returns the cache path on success, empty path on failure. + std::filesystem::path FindPdbInCacheOrServer(std::string_view PdbName, + const std::string& ImageIdKey, + const std::filesystem::path& CacheDir) + { + if (ImageIdKey.empty()) + { + return {}; + } + + // Check local cache first (includes previously cached local PDBs and + // earlier symbol server downloads). + std::filesystem::path CachePath = CacheDir / PdbName / ImageIdKey / PdbName; + std::error_code Ec; + if (std::filesystem::exists(CachePath, Ec)) + { + return CachePath; + } + + // Try symbol servers + const std::vector<std::string>& Servers = ParseSymbolPath(); + for (const std::string& Server : Servers) + { + std::filesystem::path Downloaded = DownloadPdb(Server, PdbName, ImageIdKey, CacheDir); + if (!Downloaded.empty()) + { + return Downloaded; + } + } + + return {}; + } + +} // namespace + +////////////////////////////////////////////////////////////////////////////// +// RawPdb backend — reads PDB files directly + +class PdbSymbolResolver final : public SymbolResolver +{ +public: + void LoadModule(const ModuleInfo& Module) override; + std::string Resolve(uint64_t Address) const override; + +private: + struct FunctionEntry + { + uint64_t Address; + uint32_t Size; + std::string Name; + }; + + struct LineEntry + { + uint64_t Address; + uint32_t CodeSize; + uint32_t Line; + std::string File; // shortened: basename only + }; + + std::vector<FunctionEntry> m_Functions; + std::vector<LineEntry> m_Lines; +}; + +void +PdbSymbolResolver::LoadModule(const ModuleInfo& Module) +{ + if (Module.FullPath.empty() || Module.Base == 0) + { + return; + } + + std::string ImageIdKey = FormatImageIdKey(Module.ImageId); + std::string PdbName = std::filesystem::path(Module.FullPath).filename().replace_extension(".pdb").string(); + const std::filesystem::path& CacheDir = GetSymbolCacheDir(); + + if (!IsSafePdbName(PdbName)) + { + ZEN_WARN("Rejecting unsafe PDB name from trace: '{}'", PdbName); + return; + } + + // Try local PDB first (next to the binary) + std::filesystem::path PdbPath(Module.FullPath); + PdbPath.replace_extension(".pdb"); + + std::error_code Ec; + bool FromLocal = false; + + if (std::filesystem::exists(PdbPath, Ec)) + { + FromLocal = true; + } + else + { + // Try symbol cache / symbol server download + PdbPath = FindPdbInCacheOrServer(PdbName, ImageIdKey, CacheDir); + if (PdbPath.empty()) + { + ZEN_DEBUG("PDB not found locally or on symbol server: {}", PdbName); + return; + } + } + + MappedFile File; + if (!File.Open(PdbPath)) + { + ZEN_DEBUG("Failed to open PDB: {}", PdbPath.string()); + return; + } + + if (PDB::ValidateFile(File.Data, File.Size) != PDB::ErrorCode::Success) + { + ZEN_DEBUG("Invalid PDB file: {}", PdbPath.string()); + return; + } + + PDB::RawFile RawFile = PDB::CreateRawFile(File.Data); + PDB::InfoStream PdbInfoStream(RawFile); + + // Verify the PDB matches the traced module by comparing GUID + Age. + // The trace stores ImageId as 16 bytes GUID followed by 4 bytes Age. + if (Module.ImageId.size() >= 20) + { + const PDB::Header* PdbHeader = PdbInfoStream.GetHeader(); + + // Only compare the GUID, not the age. The symbol server may return a + // PDB with a higher age (from incremental linking) which is compatible. + if (memcmp(&PdbHeader->guid, Module.ImageId.data(), 16) != 0) + { + if (FromLocal) + { + // The local PDB no longer matches — the binary was recompiled + // since the trace was taken. Try the symbol cache / servers for + // the original PDB. + File.Close(); + PdbPath = FindPdbInCacheOrServer(PdbName, ImageIdKey, CacheDir); + if (PdbPath.empty()) + { + ZEN_WARN("PDB mismatch for {} — binary was recompiled and no cached symbols available", Module.Name); + return; + } + + FromLocal = false; + + if (!File.Open(PdbPath)) + { + ZEN_DEBUG("Failed to open cached PDB: {}", PdbPath.string()); + return; + } + + if (PDB::ValidateFile(File.Data, File.Size) != PDB::ErrorCode::Success) + { + ZEN_DEBUG("Invalid cached PDB: {}", PdbPath.string()); + return; + } + + RawFile = PDB::CreateRawFile(File.Data); + PdbInfoStream = PDB::InfoStream(RawFile); + + const PDB::Header* CachedHeader = PdbInfoStream.GetHeader(); + if (memcmp(&CachedHeader->guid, Module.ImageId.data(), 16) != 0) + { + ZEN_WARN("PDB GUID mismatch for {} — skipping", Module.Name); + return; + } + } + else + { + ZEN_WARN("PDB GUID mismatch for {} — skipping", Module.Name); + return; + } + } + } + + // Cache the local PDB so that future analysis of traces from this build + // succeeds even after the binary is recompiled. + if (FromLocal && !ImageIdKey.empty()) + { + CacheLocalPdb(PdbPath, PdbName, ImageIdKey, CacheDir); + } + + if (PDB::HasValidDBIStream(RawFile) != PDB::ErrorCode::Success) + { + return; + } + + const PDB::DBIStream DbiStream = PDB::CreateDBIStream(RawFile); + if (DbiStream.HasValidImageSectionStream(RawFile) != PDB::ErrorCode::Success) + { + return; + } + + const PDB::ImageSectionStream ImageSections = DbiStream.CreateImageSectionStream(RawFile); + uint64_t ModuleBase = Module.Base; + uint32_t SkippedModules = 0; + size_t FunctionCountBeforeModuleSymbols = m_Functions.size(); + + // Collect functions from module symbol streams (S_GPROC32 / S_LPROC32) + { + const PDB::ModuleInfoStream ModInfoStream = DbiStream.CreateModuleInfoStream(RawFile); + const PDB::ArrayView<PDB::ModuleInfoStream::Module> Modules = ModInfoStream.GetModules(); + for (const PDB::ModuleInfoStream::Module& Mod : Modules) + { + if (!Mod.HasSymbolStream()) + { + ++SkippedModules; + continue; + } + + const PDB::ModuleSymbolStream SymStream = Mod.CreateSymbolStream(RawFile); + + SymStream.ForEachSymbol([&](const PDB::CodeView::DBI::Record* Record) { + using Kind = PDB::CodeView::DBI::SymbolRecordKind; + const Kind K = Record->header.kind; + const auto& Data = Record->data; + + if (K == Kind::S_GPROC32 || K == Kind::S_LPROC32 || K == Kind::S_GPROC32_ID || K == Kind::S_LPROC32_ID) + { + uint32_t Rva = ImageSections.ConvertSectionOffsetToRVA(Data.S_GPROC32.section, Data.S_GPROC32.offset); + if (Rva != 0) + { + m_Functions.push_back({ModuleBase + Rva, Data.S_GPROC32.codeSize, Data.S_GPROC32.name}); + } + } + }); + } + } + + // Public symbols as fallback only when module symbol streams did not yield any + // functions. Building the coalesced symbol-record stream is expensive and can + // allocate tens of megabytes for large PDBs. + if (FunctionCountBeforeModuleSymbols == m_Functions.size() && DbiStream.HasValidPublicSymbolStream(RawFile) == PDB::ErrorCode::Success) + { + const PDB::PublicSymbolStream PubStream = DbiStream.CreatePublicSymbolStream(RawFile); + const PDB::CoalescedMSFStream SymRecords = DbiStream.CreateSymbolRecordStream(RawFile); + + for (const PDB::HashRecord& Hash : PubStream.GetRecords()) + { + const PDB::CodeView::DBI::Record* Record = SymRecords.GetDataAtOffset<PDB::CodeView::DBI::Record>(Hash.offset); + if (Record->header.kind == PDB::CodeView::DBI::SymbolRecordKind::S_PUB32) + { + uint32_t Rva = ImageSections.ConvertSectionOffsetToRVA(Record->data.S_PUB32.section, Record->data.S_PUB32.offset); + if (Rva != 0) + { + m_Functions.push_back({ModuleBase + Rva, 0, Record->data.S_PUB32.name}); + } + } + } + } + + // Collect line information from module line streams + if (PdbInfoStream.HasNamesStream()) + { + const PDB::NamesStream NamesStream = PdbInfoStream.CreateNamesStream(RawFile); + const PDB::ModuleInfoStream ModInfoStream2 = DbiStream.CreateModuleInfoStream(RawFile); + + for (const PDB::ModuleInfoStream::Module& Mod : ModInfoStream2.GetModules()) + { + if (!Mod.HasLineStream()) + { + continue; + } + + const PDB::ModuleLineStream LineStream = Mod.CreateLineStream(RawFile); + + // Two passes: first find the checksums section, then process lines. + const PDB::CodeView::DBI::FileChecksumHeader* ModuleChecksumBase = nullptr; + + LineStream.ForEachSection([&](const PDB::CodeView::DBI::LineSection* Section) { + if (Section->header.kind == PDB::CodeView::DBI::DebugSubsectionKind::S_FILECHECKSUMS) + { + ModuleChecksumBase = &Section->checksumHeader; + } + }); + + if (ModuleChecksumBase == nullptr) + { + continue; + } + + LineStream.ForEachSection([&](const PDB::CodeView::DBI::LineSection* Section) { + if (Section->header.kind != PDB::CodeView::DBI::DebugSubsectionKind::S_LINES) + { + return; + } + + uint16_t SecIdx = Section->linesHeader.sectionIndex; + uint32_t SecOff = Section->linesHeader.sectionOffset; + + LineStream.ForEachLinesBlock(Section, + [&](const PDB::CodeView::DBI::LinesFileBlockHeader* Block, + const PDB::CodeView::DBI::Line* Lines, + const PDB::CodeView::DBI::Column*) { + if (Block->numLines == 0) + { + return; + } + + // Resolve filename for this block + const auto* Checksum = PDB::Pointer::Offset<const PDB::CodeView::DBI::FileChecksumHeader*>( + ModuleChecksumBase, + Block->fileChecksumOffset); + const char* FullFile = NamesStream.GetFilename(Checksum->filenameOffset); + + // Extract basename + std::string_view FileView(FullFile); + size_t Cut = FileView.find_last_of("\\/"); + std::string Basename(Cut != std::string_view::npos ? FileView.substr(Cut + 1) : FileView); + + for (uint32_t I = 0; I < Block->numLines; ++I) + { + uint32_t Rva = + ImageSections.ConvertSectionOffsetToRVA(SecIdx, SecOff + Lines[I].offset); + if (Rva == 0) + { + continue; + } + + uint32_t CodeSize = 0; + if (I + 1 < Block->numLines) + { + CodeSize = Lines[I + 1].offset - Lines[I].offset; + } + else + { + CodeSize = Section->linesHeader.codeSize - Lines[I].offset; + } + + m_Lines.push_back({ModuleBase + Rva, CodeSize, Lines[I].linenumStart, Basename}); + } + }); + }); + } + } + + std::sort(m_Functions.begin(), m_Functions.end(), [](const FunctionEntry& A, const FunctionEntry& B) { return A.Address < B.Address; }); + + std::sort(m_Lines.begin(), m_Lines.end(), [](const LineEntry& A, const LineEntry& B) { return A.Address < B.Address; }); + + if (SkippedModules > 0) + { + ZEN_INFO("Loaded {} symbols, {} line records from {} ({} modules without embedded debug info)", + m_Functions.size(), + m_Lines.size(), + Module.Name, + SkippedModules); + } + else + { + ZEN_INFO("Loaded {} symbols, {} line records from {}", m_Functions.size(), m_Lines.size(), Module.Name); + } +} + +std::string +PdbSymbolResolver::Resolve(uint64_t Address) const +{ + if (m_Functions.empty()) + { + return {}; + } + + // Resolve function name + auto FnIt = std::upper_bound(m_Functions.begin(), m_Functions.end(), Address, [](uint64_t Addr, const FunctionEntry& E) { + return Addr < E.Address; + }); + + if (FnIt == m_Functions.begin()) + { + return {}; + } + + --FnIt; + + if (FnIt->Size > 0 && Address >= FnIt->Address + FnIt->Size) + { + return {}; + } + + std::string Result = FormatSymbol(FnIt->Name, Address - FnIt->Address); + + // Resolve file:line + if (!m_Lines.empty()) + { + auto LineIt = + std::upper_bound(m_Lines.begin(), m_Lines.end(), Address, [](uint64_t Addr, const LineEntry& E) { return Addr < E.Address; }); + + if (LineIt != m_Lines.begin()) + { + --LineIt; + if (LineIt->CodeSize == 0 || Address < LineIt->Address + LineIt->CodeSize) + { + Result += fmt::format(" [{}:{}]", LineIt->File, LineIt->Line); + } + } + } + + return Result; +} + +////////////////////////////////////////////////////////////////////////////// +// DbgHelp backend — uses Windows symbol API, supports _NT_SYMBOL_PATH + +class DbgHelpSymbolResolver final : public SymbolResolver +{ +public: + DbgHelpSymbolResolver(); + ~DbgHelpSymbolResolver() override; + + void LoadModule(const ModuleInfo& Module) override; + std::string Resolve(uint64_t Address) const override; + +private: + // Map trace addresses to DbgHelp addresses when the loaded base differs. + struct ModuleMapping + { + uint64_t TraceBase; + uint64_t TraceEnd; + int64_t Delta; // DbgHelpBase - TraceBase + }; + + HANDLE m_Process = nullptr; + std::vector<ModuleMapping> m_Mappings; + // DbgHelp is not thread-safe; its API functions require serialized access. This + // mutex covers every DbgHelp call (SymInitialize/SymLoadModuleExW/SymFromAddr/ + // SymGetLineFromAddr64) and therefore serializes all parallel symbol lookups in + // trace_analyze. For workloads where lookup throughput matters, prefer + // PdbSymbolResolver, which parses PDBs directly and is lock-free per-module. + mutable std::mutex m_Mutex; +}; + +DbgHelpSymbolResolver::DbgHelpSymbolResolver() +{ + std::lock_guard Lock(m_Mutex); + + // Use a unique pseudo-handle so we don't conflict with the runtime + // symbol handler used by callstack.cpp / crashhandler.cpp. + m_Process = reinterpret_cast<HANDLE>(static_cast<uintptr_t>(0xDEAD0042)); + + // NULL search path lets DbgHelp use _NT_SYMBOL_PATH and its defaults. + if (!SymInitialize(m_Process, nullptr, FALSE)) + { + ZEN_WARN("DbgHelp: SymInitialize failed (error {})", GetLastError()); + m_Process = nullptr; + } +} + +DbgHelpSymbolResolver::~DbgHelpSymbolResolver() +{ + std::lock_guard Lock(m_Mutex); + + if (m_Process != nullptr) + { + SymCleanup(m_Process); + } +} + +void +DbgHelpSymbolResolver::LoadModule(const ModuleInfo& Module) +{ + std::lock_guard Lock(m_Mutex); + + if (m_Process == nullptr || Module.FullPath.empty() || Module.Base == 0) + { + return; + } + + std::filesystem::path ModulePath(Module.FullPath); + std::wstring WidePath = ModulePath.wstring(); + + DWORD64 LoadedBase = SymLoadModuleExW(m_Process, nullptr, WidePath.c_str(), nullptr, Module.Base, Module.Size, nullptr, 0); + + if (LoadedBase == 0) + { + DWORD Err = GetLastError(); + if (Err != ERROR_SUCCESS) + { + ZEN_DEBUG("DbgHelp: failed to load {}: error {}", Module.Name, Err); + } + return; + } + + int64_t Delta = int64_t(LoadedBase) - int64_t(Module.Base); + if (Delta != 0) + { + ZEN_DEBUG("DbgHelp: {} loaded at 0x{:X} (trace base 0x{:X}, delta {:+})", Module.Name, LoadedBase, Module.Base, Delta); + } + + uint64_t TraceEnd = Module.Base + (Module.Size > 0 ? Module.Size : 0x1000000); + m_Mappings.push_back({Module.Base, TraceEnd, Delta}); + + ZEN_INFO("DbgHelp: loaded symbols for {}", Module.Name); +} + +std::string +DbgHelpSymbolResolver::Resolve(uint64_t Address) const +{ + std::lock_guard Lock(m_Mutex); + + if (m_Process == nullptr) + { + return {}; + } + + // Translate the trace address to the DbgHelp address space + uint64_t DbgAddr = Address; + for (const ModuleMapping& M : m_Mappings) + { + if (Address >= M.TraceBase && Address < M.TraceEnd) + { + DbgAddr = uint64_t(int64_t(Address) + M.Delta); + break; + } + } + + alignas(SYMBOL_INFO) char Buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME]; + SYMBOL_INFO* SymInfo = reinterpret_cast<SYMBOL_INFO*>(Buffer); + SymInfo->SizeOfStruct = sizeof(SYMBOL_INFO); + SymInfo->MaxNameLen = MAX_SYM_NAME; + + DWORD64 Displacement = 0; + if (!SymFromAddr(m_Process, DbgAddr, &Displacement, SymInfo)) + { + return {}; + } + + std::string Result = FormatSymbol(std::string_view(SymInfo->Name, SymInfo->NameLen), Displacement); + + IMAGEHLP_LINE64 LineInfo = {}; + LineInfo.SizeOfStruct = sizeof(IMAGEHLP_LINE64); + DWORD LineDisplacement = 0; + if (SymGetLineFromAddr64(m_Process, DbgAddr, &LineDisplacement, &LineInfo)) + { + std::string_view FileView(LineInfo.FileName); + size_t Cut = FileView.find_last_of("\\/"); + std::string_view Basename = (Cut != std::string_view::npos) ? FileView.substr(Cut + 1) : FileView; + Result += fmt::format(" [{}:{}]", Basename, LineInfo.LineNumber); + } + + return Result; +} + +#endif // ZEN_PLATFORM_WINDOWS + +////////////////////////////////////////////////////////////////////////////// +// Shared helpers for subprocess-based backends + +namespace { + + // CreateProc parses the command line as space-separated tokens, so argv[0] + // must be quoted if the resolved executable path contains spaces (e.g. an + // Xcode toolchain location on macOS). + std::string QuoteIfNeeded(std::string_view Path) + { + if (Path.find(' ') == std::string_view::npos) + { + return std::string(Path); + } + return fmt::format("\"{}\"", Path); + } + +} // namespace + +////////////////////////////////////////////////////////////////////////////// +// llvm-symbolizer backend — cross-platform, shells out to `llvm-symbolizer` +// and speaks its interactive protocol over pipes. +// +// Protocol (one request / response): +// We write: "<path-to-binary> 0x<relative-address>\n" +// It replies: "FunctionName\n" +// "file:line:col\n" +// "\n" <-- blank line terminates the record +// +// Launch flags: +// --demangle demangle C++ names (default, but explicit) +// --output-style=LLVM stable two-line format described above +// --functions=linkage keep template arguments visible +// --relative-address treat the address as an offset from module base +// --inlining=false emit one frame per address (no inline expansion) + +class LlvmSymbolizerResolver final : public SymbolResolver +{ +public: + LlvmSymbolizerResolver() = default; + ~LlvmSymbolizerResolver() override; + + void LoadModule(const ModuleInfo& Module) override; + std::string Resolve(uint64_t Address) const override; + +private: + struct Module + { + std::string FullPath; + uint64_t Base = 0; + uint64_t End = 0; + }; + + const Module* FindModule(uint64_t Address) const; + bool EnsureProcess() const; + bool ReadLine(std::string& Out) const; + std::string DoQuery(const Module& M, uint64_t RelAddress) const; + + std::vector<Module> m_Modules; + + // Subprocess + IO state. All accesses serialized under m_Mutex. + mutable std::mutex m_Mutex; + mutable bool m_Attempted = false; + mutable bool m_Alive = false; + mutable zen::ProcessHandle m_Process; + mutable zen::StdinPipeHandles m_StdinPipe; + mutable zen::StdoutPipeHandles m_StdoutPipe; + mutable std::string m_ReadBuffer; + + // Cache resolved addresses (same mutex). + mutable std::unordered_map<uint64_t, std::string> m_Cache; +}; + +LlvmSymbolizerResolver::~LlvmSymbolizerResolver() +{ + std::lock_guard Lock(m_Mutex); + if (m_Alive) + { + // Closing stdin lets llvm-symbolizer exit cleanly on EOF. + m_StdinPipe.CloseWriteEnd(); + m_Process.Wait(2000); + if (m_Process.IsRunning()) + { + m_Process.Terminate(0); + } + } +} + +void +LlvmSymbolizerResolver::LoadModule(const ModuleInfo& Mod) +{ + if (Mod.FullPath.empty() || Mod.Base == 0) + { + return; + } + + // llvm-symbolizer auto-discovers adjacent debug info (Foo.dSYM on Mac, + // .gnu_debuglink / build-id sources on Linux, Foo.pdb on Windows). If the + // binary itself isn't present locally, there's nothing we can do. + std::error_code Ec; + if (!std::filesystem::exists(Mod.FullPath, Ec)) + { + ZEN_DEBUG("llvm-symbolizer: binary not found for {} at {}", Mod.Name, Mod.FullPath); + return; + } + + uint64_t End = Mod.Base + (Mod.Size > 0 ? Mod.Size : 0x1000000); + + std::lock_guard Lock(m_Mutex); + m_Modules.push_back({Mod.FullPath, Mod.Base, End}); + ZEN_INFO("llvm-symbolizer: registered {} [0x{:X}..0x{:X})", Mod.Name, Mod.Base, End); +} + +std::string +LlvmSymbolizerResolver::Resolve(uint64_t Address) const +{ + std::lock_guard Lock(m_Mutex); + + auto CacheIt = m_Cache.find(Address); + if (CacheIt != m_Cache.end()) + { + return CacheIt->second; + } + + const Module* M = FindModule(Address); + if (M == nullptr) + { + m_Cache.emplace(Address, std::string{}); + return {}; + } + + std::string Result = DoQuery(*M, Address - M->Base); + m_Cache.emplace(Address, Result); + return Result; +} + +const LlvmSymbolizerResolver::Module* +LlvmSymbolizerResolver::FindModule(uint64_t Address) const +{ + for (const Module& M : m_Modules) + { + if (Address >= M.Base && Address < M.End) + { + return &M; + } + } + return nullptr; +} + +bool +LlvmSymbolizerResolver::EnsureProcess() const +{ + if (m_Attempted) + { + return m_Alive; + } + m_Attempted = true; + + std::filesystem::path Executable = SearchPathForExecutable("llvm-symbolizer"); + + if (!CreateStdinPipe(m_StdinPipe) || !CreateStdoutPipe(m_StdoutPipe)) + { + ZEN_WARN("llvm-symbolizer: failed to create pipes"); + return false; + } + + // Build the command line. CommandLine begins with the executable name (arg[0]). + std::string CommandLine = fmt::format("{} --demangle --output-style=LLVM --functions=linkage --relative-address --inlining=false", + QuoteIfNeeded(Executable.string())); + + CreateProcOptions Options; + Options.StdinPipe = &m_StdinPipe; + Options.StdoutPipe = &m_StdoutPipe; + + CreateProcResult Handle = CreateProc(Executable, CommandLine, Options); + +#if ZEN_PLATFORM_WINDOWS + if (Handle == nullptr) +#else + if (Handle <= 0) +#endif + { + ZEN_WARN("llvm-symbolizer: failed to launch '{}' - install LLVM or add to PATH", Executable.string()); + m_StdinPipe.Close(); + m_StdoutPipe.Close(); + return false; + } + +#if ZEN_PLATFORM_WINDOWS + m_Process.Initialize(Handle); +#else + std::error_code Ec; + m_Process.Initialize(int(Handle), Ec); + if (Ec) + { + ZEN_WARN("llvm-symbolizer: ProcessHandle init failed: {}", Ec.message()); + m_StdinPipe.Close(); + m_StdoutPipe.Close(); + return false; + } +#endif + + // Close the child-side handles in the parent. + m_StdinPipe.CloseReadEnd(); + m_StdoutPipe.CloseWriteEnd(); + + m_Alive = true; + return true; +} + +bool +LlvmSymbolizerResolver::ReadLine(std::string& Out) const +{ + // Search for a newline already in the buffer; if not, read more. + for (;;) + { + size_t NewlinePos = m_ReadBuffer.find('\n'); + if (NewlinePos != std::string::npos) + { + Out.assign(m_ReadBuffer, 0, NewlinePos); + m_ReadBuffer.erase(0, NewlinePos + 1); + // Trim a trailing \r (in case of CRLF line endings). + if (!Out.empty() && Out.back() == '\r') + { + Out.pop_back(); + } + return true; + } + + char Buffer[1024]; +#if ZEN_PLATFORM_WINDOWS + DWORD BytesRead = 0; + if (!::ReadFile(m_StdoutPipe.ReadHandle, Buffer, sizeof(Buffer), &BytesRead, nullptr) || BytesRead == 0) + { + return false; + } + m_ReadBuffer.append(Buffer, BytesRead); +#else + ssize_t BytesRead = ::read(m_StdoutPipe.ReadFd, Buffer, sizeof(Buffer)); + if (BytesRead <= 0) + { + if (BytesRead < 0 && errno == EINTR) + { + continue; + } + return false; + } + m_ReadBuffer.append(Buffer, static_cast<size_t>(BytesRead)); +#endif + } +} + +std::string +LlvmSymbolizerResolver::DoQuery(const Module& M, uint64_t RelAddress) const +{ + if (!EnsureProcess()) + { + return {}; + } + + // Write "<path> 0x<addr>\n". Paths with spaces must be quoted for llvm-symbolizer + // interactive input; it accepts double quotes. + std::string Line; + if (M.FullPath.find(' ') != std::string::npos) + { + Line = fmt::format("\"{}\" 0x{:X}\n", M.FullPath, RelAddress); + } + else + { + Line = fmt::format("{} 0x{:X}\n", M.FullPath, RelAddress); + } + +#if ZEN_PLATFORM_WINDOWS + DWORD BytesWritten = 0; + if (!::WriteFile(m_StdinPipe.WriteHandle, Line.data(), static_cast<DWORD>(Line.size()), &BytesWritten, nullptr) || + BytesWritten != Line.size()) + { + ZEN_WARN("llvm-symbolizer: write failed, disabling backend"); + m_Alive = false; + return {}; + } +#else + const char* Ptr = Line.data(); + size_t Remaining = Line.size(); + while (Remaining > 0) + { + ssize_t N = ::write(m_StdinPipe.WriteFd, Ptr, Remaining); + if (N <= 0) + { + if (N < 0 && errno == EINTR) + { + continue; + } + ZEN_WARN("llvm-symbolizer: write failed, disabling backend"); + m_Alive = false; + return {}; + } + Ptr += N; + Remaining -= static_cast<size_t>(N); + } +#endif + + // Read lines until a blank line terminates the record. + std::string Function; + std::string Location; + std::string Buf; + int LineIdx = 0; + while (ReadLine(Buf)) + { + if (Buf.empty()) + { + break; + } + if (LineIdx == 0) + { + Function = Buf; + } + else if (LineIdx == 1) + { + Location = Buf; + } + // Additional lines would be inline frames (--inlining=false suppresses them); ignore. + ++LineIdx; + } + + if (Function.empty() || Function == "??") + { + return {}; + } + + std::string Result = std::move(Function); + if (!Location.empty() && Location != "??:0:0") + { + // Location is "path:line:col" — trim to "basename:line" to match Windows output. + std::string_view LocView(Location); + size_t LastColon = LocView.find_last_of(':'); + if (LastColon != std::string_view::npos) + { + LocView = LocView.substr(0, LastColon); + } + size_t Slash = LocView.find_last_of("/\\"); + std::string_view FileLine = (Slash == std::string_view::npos) ? LocView : LocView.substr(Slash + 1); + Result += fmt::format(" [{}]", FileLine); + } + return Result; +} + +////////////////////////////////////////////////////////////////////////////// +// atos backend — macOS only. Apple's symbolizer; ships with Xcode + the CLT. +// +// Unlike llvm-symbolizer, atos accepts only one binary per process. We keep +// one subprocess per loaded module and demultiplex queries by module path. +// +// Protocol (one request / response): +// We write: "0x<absolute-address>\n" +// It replies: "Function (in Binary) (file.cpp:NN)\n" +// or "Function (in Binary) + 0x<disp>\n" (no debug info) +// or "0x<address>\n" (nothing known) +// +// Launched with: atos -o <binary> -l 0x<module-base> +// atos subtracts -l from each input address to get the file offset. + +#if ZEN_PLATFORM_MAC + +class AtosSymbolizerResolver final : public SymbolResolver +{ +public: + AtosSymbolizerResolver() = default; + ~AtosSymbolizerResolver() override; + + void LoadModule(const ModuleInfo& Module) override; + std::string Resolve(uint64_t Address) const override; + +private: + struct Module + { + std::string FullPath; + uint64_t Base = 0; + uint64_t End = 0; + }; + + // One atos subprocess per loaded module (atos is single-binary). + struct AtosProcess + { + zen::ProcessHandle Process; + zen::StdinPipeHandles StdinPipe; + zen::StdoutPipeHandles StdoutPipe; + std::string ReadBuffer; + bool Alive = false; + }; + + const Module* FindModule(uint64_t Address) const; + AtosProcess* EnsureProcessFor(const Module& M) const; + bool ReadLine(AtosProcess& P, std::string& Out) const; + std::string DoQuery(const Module& M, uint64_t Address) const; + + std::vector<Module> m_Modules; + + mutable std::mutex m_Mutex; + mutable std::unordered_map<std::string, std::unique_ptr<AtosProcess>> m_Processes; + mutable std::unordered_map<uint64_t, std::string> m_Cache; +}; + +AtosSymbolizerResolver::~AtosSymbolizerResolver() +{ + std::lock_guard Lock(m_Mutex); + for (auto& [Path, P] : m_Processes) + { + if (P && P->Alive) + { + P->StdinPipe.CloseWriteEnd(); + P->Process.Wait(2000); + if (P->Process.IsRunning()) + { + P->Process.Terminate(0); + } + } + } +} + +void +AtosSymbolizerResolver::LoadModule(const ModuleInfo& Mod) +{ + if (Mod.FullPath.empty() || Mod.Base == 0) + { + return; + } + + std::error_code Ec; + if (!std::filesystem::exists(Mod.FullPath, Ec)) + { + ZEN_DEBUG("atos: binary not found for {} at {}", Mod.Name, Mod.FullPath); + return; + } + + uint64_t End = Mod.Base + (Mod.Size > 0 ? Mod.Size : 0x1000000); + + std::lock_guard Lock(m_Mutex); + m_Modules.push_back({Mod.FullPath, Mod.Base, End}); + ZEN_INFO("atos: registered {} [0x{:X}..0x{:X})", Mod.Name, Mod.Base, End); +} + +std::string +AtosSymbolizerResolver::Resolve(uint64_t Address) const +{ + std::lock_guard Lock(m_Mutex); + + auto CacheIt = m_Cache.find(Address); + if (CacheIt != m_Cache.end()) + { + return CacheIt->second; + } + + const Module* M = FindModule(Address); + if (M == nullptr) + { + m_Cache.emplace(Address, std::string{}); + return {}; + } + + std::string Result = DoQuery(*M, Address); + m_Cache.emplace(Address, Result); + return Result; +} + +const AtosSymbolizerResolver::Module* +AtosSymbolizerResolver::FindModule(uint64_t Address) const +{ + for (const Module& M : m_Modules) + { + if (Address >= M.Base && Address < M.End) + { + return &M; + } + } + return nullptr; +} + +AtosSymbolizerResolver::AtosProcess* +AtosSymbolizerResolver::EnsureProcessFor(const Module& M) const +{ + auto It = m_Processes.find(M.FullPath); + if (It != m_Processes.end()) + { + return It->second.get(); + } + + auto P = std::make_unique<AtosProcess>(); + + if (!CreateStdinPipe(P->StdinPipe) || !CreateStdoutPipe(P->StdoutPipe)) + { + ZEN_WARN("atos: failed to create pipes for {}", M.FullPath); + auto [Ins, _] = m_Processes.emplace(M.FullPath, std::move(P)); + return Ins->second.get(); // Alive = false + } + + std::filesystem::path Executable = SearchPathForExecutable("atos"); + std::string CommandLine = fmt::format("{} -o \"{}\" -l 0x{:X}", QuoteIfNeeded(Executable.string()), M.FullPath, M.Base); + + CreateProcOptions Options; + Options.StdinPipe = &P->StdinPipe; + Options.StdoutPipe = &P->StdoutPipe; + + CreateProcResult Handle = CreateProc(Executable, CommandLine, Options); + if (Handle <= 0) + { + ZEN_WARN("atos: failed to launch for {} - `atos` should be on PATH on macOS", M.FullPath); + P->StdinPipe.Close(); + P->StdoutPipe.Close(); + auto [Ins, _] = m_Processes.emplace(M.FullPath, std::move(P)); + return Ins->second.get(); // Alive = false + } + + std::error_code Ec; + P->Process.Initialize(int(Handle), Ec); + if (Ec) + { + ZEN_WARN("atos: ProcessHandle init failed for {}: {}", M.FullPath, Ec.message()); + P->StdinPipe.Close(); + P->StdoutPipe.Close(); + auto [Ins, _] = m_Processes.emplace(M.FullPath, std::move(P)); + return Ins->second.get(); // Alive = false + } + + P->StdinPipe.CloseReadEnd(); + P->StdoutPipe.CloseWriteEnd(); + P->Alive = true; + + auto [Ins, _] = m_Processes.emplace(M.FullPath, std::move(P)); + return Ins->second.get(); +} + +bool +AtosSymbolizerResolver::ReadLine(AtosProcess& P, std::string& Out) const +{ + for (;;) + { + size_t NewlinePos = P.ReadBuffer.find('\n'); + if (NewlinePos != std::string::npos) + { + Out.assign(P.ReadBuffer, 0, NewlinePos); + P.ReadBuffer.erase(0, NewlinePos + 1); + return true; + } + + char Buffer[1024]; + ssize_t BytesRead = ::read(P.StdoutPipe.ReadFd, Buffer, sizeof(Buffer)); + if (BytesRead <= 0) + { + if (BytesRead < 0 && errno == EINTR) + { + continue; + } + return false; + } + P.ReadBuffer.append(Buffer, static_cast<size_t>(BytesRead)); + } +} + +std::string +AtosSymbolizerResolver::DoQuery(const Module& M, uint64_t Address) const +{ + AtosProcess* P = EnsureProcessFor(M); + if (P == nullptr || !P->Alive) + { + return {}; + } + + std::string Line = fmt::format("0x{:X}\n", Address); + + const char* Ptr = Line.data(); + size_t Remaining = Line.size(); + while (Remaining > 0) + { + ssize_t N = ::write(P->StdinPipe.WriteFd, Ptr, Remaining); + if (N <= 0) + { + if (N < 0 && errno == EINTR) + { + continue; + } + ZEN_WARN("atos: write failed for {}, disabling", M.FullPath); + P->Alive = false; + return {}; + } + Ptr += N; + Remaining -= static_cast<size_t>(N); + } + + std::string Reply; + if (!ReadLine(*P, Reply) || Reply.empty()) + { + return {}; + } + + // Parse "Function (in Binary) (file.cpp:NN)" or "... + 0xNN" or just "0xADDR". + // Extract everything before " (in " as the function name. + size_t InPos = Reply.find(" (in "); + if (InPos == std::string::npos) + { + // No match — either raw "0xADDR" (no info) or an error message. Skip. + return {}; + } + + std::string_view Function(Reply.data(), InPos); + + // Look for a trailing "(file:line)" after the "(in ...)" block. + std::string_view LocationView; + size_t AfterIn = Reply.find(')', InPos); + if (AfterIn != std::string::npos) + { + size_t OpenParen = Reply.find('(', AfterIn); + if (OpenParen != std::string::npos) + { + size_t CloseParen = Reply.find(')', OpenParen); + if (CloseParen != std::string::npos && CloseParen > OpenParen + 1) + { + LocationView = std::string_view(Reply).substr(OpenParen + 1, CloseParen - OpenParen - 1); + } + } + } + + std::string Result(Function); + if (!LocationView.empty()) + { + // atos gives us "file.cpp:NN" directly — no need to strip a directory. + Result += fmt::format(" [{}]", LocationView); + } + return Result; +} + +#endif // ZEN_PLATFORM_MAC + +////////////////////////////////////////////////////////////////////////////// +// Factory + +namespace { + +#if ZEN_PLATFORM_MAC + // Probe PATH for a tool and return true if something usable was found. + // SearchPathForExecutable returns the input unchanged if the tool can't be + // found, so we compare against the filesystem to detect a hit. + bool ToolIsOnPath(std::string_view Name) + { + std::filesystem::path Resolved = SearchPathForExecutable(Name); + std::error_code Ec; + return std::filesystem::exists(Resolved, Ec) && std::filesystem::is_regular_file(Resolved, Ec); + } +#endif + + SymbolBackend ResolveAutoBackend() + { +#if ZEN_PLATFORM_WINDOWS + return SymbolBackend::Pdb; +#elif ZEN_PLATFORM_MAC + if (ToolIsOnPath("llvm-symbolizer")) + { + return SymbolBackend::LlvmSymbolizer; + } + return SymbolBackend::Atos; +#else + // Linux: llvm-symbolizer is the only backend we ship. + return SymbolBackend::LlvmSymbolizer; +#endif + } + +} // namespace + +std::unique_ptr<SymbolResolver> +CreateSymbolResolver(SymbolBackend Backend) +{ + if (Backend == SymbolBackend::Auto) + { + Backend = ResolveAutoBackend(); + } + + if (Backend == SymbolBackend::Off) + { + return std::make_unique<NullSymbolResolver>(); + } + + if (Backend == SymbolBackend::LlvmSymbolizer) + { + return std::make_unique<LlvmSymbolizerResolver>(); + } + +#if ZEN_PLATFORM_MAC + if (Backend == SymbolBackend::Atos) + { + return std::make_unique<AtosSymbolizerResolver>(); + } +#else + if (Backend == SymbolBackend::Atos) + { + ZEN_WARN("atos backend is macOS-only; falling back to llvm-symbolizer"); + return std::make_unique<LlvmSymbolizerResolver>(); + } +#endif + +#if ZEN_PLATFORM_WINDOWS + if (Backend == SymbolBackend::DbgHelp) + { + return std::make_unique<DbgHelpSymbolResolver>(); + } + return std::make_unique<PdbSymbolResolver>(); +#else + // Pdb / DbgHelp aren't available on non-Windows; any other request falls back to llvm-symbolizer. + return std::make_unique<LlvmSymbolizerResolver>(); +#endif +} + +SymbolBackend +ParseSymbolBackend(std::string_view Name) +{ + if (Name == "auto") + { + return SymbolBackend::Auto; + } + if (Name == "pdb") + { + return SymbolBackend::Pdb; + } + if (Name == "dbghelp") + { + return SymbolBackend::DbgHelp; + } + if (Name == "llvm" || Name == "llvm-symbolizer") + { + return SymbolBackend::LlvmSymbolizer; + } + if (Name == "atos") + { + return SymbolBackend::Atos; + } + if (Name == "off") + { + return SymbolBackend::Off; + } + return SymbolBackend::Off; +} + +} // namespace zen::trace_detail diff --git a/src/zen/trace/symbol_resolver.h b/src/zen/trace/symbol_resolver.h new file mode 100644 index 000000000..4acdaf95e --- /dev/null +++ b/src/zen/trace/symbol_resolver.h @@ -0,0 +1,45 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "trace_model.h" + +#include <cstdint> +#include <memory> +#include <string> + +namespace zen::trace_detail { + +enum class SymbolBackend : uint8_t +{ + Off, + Auto, // Probe PATH and pick the best available backend for the platform + Pdb, // Windows only: RawPdb — fast, reads PDB files directly + DbgHelp, // Windows only: DbgHelp API — supports symbol servers and _NT_SYMBOL_PATH + LlvmSymbolizer, // Any platform: shells out to `llvm-symbolizer`, resolves dSYM (Mac) / DWARF (Linux) / PDB (Windows) + Atos // macOS only: shells out to `atos`, resolves Mach-O binaries and adjacent .dSYM bundles +}; + +// Resolves virtual addresses captured in a trace to function names. +// Use CreateSymbolResolver() to obtain a concrete implementation. +class SymbolResolver +{ +public: + virtual ~SymbolResolver() = default; + + // Load symbols for a module. + virtual void LoadModule(const ModuleInfo& Module) = 0; + + // Resolve an absolute virtual address to "FunctionName + 0xNN" (or just + // "FunctionName" when the displacement is zero). Returns an empty string + // when the address cannot be resolved. + virtual std::string Resolve(uint64_t Address) const = 0; +}; + +std::unique_ptr<SymbolResolver> CreateSymbolResolver(SymbolBackend Backend); + +// Parse a string ("auto", "pdb", "dbghelp", "llvm", "llvm-symbolizer", +// "atos", "off") into a SymbolBackend enum. Returns Off on unrecognised input. +SymbolBackend ParseSymbolBackend(std::string_view Name); + +} // namespace zen::trace_detail diff --git a/src/zen/trace/timeline_query.cpp b/src/zen/trace/timeline_query.cpp new file mode 100644 index 000000000..d90c79a29 --- /dev/null +++ b/src/zen/trace/timeline_query.cpp @@ -0,0 +1,123 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "timeline_query.h" + +#include <algorithm> + +namespace zen::trace_detail { + +namespace { + + // Pick the LOD level a given resolution should read from. Mirrors the + // historical selection in trace_viewer_service.cpp: resolution 0 reads the + // raw LOD 0; otherwise the smallest LOD whose ResolutionUs >= the request + // wins, falling back to the coarsest level if none qualify. + // + // Returned values: 0 == raw scopes (LOD 0), 1..kTimelineLodCount == DetailLevels[lod-1]. + size_t SelectLodIndex(uint32_t ResolutionUs) + { + if (ResolutionUs == 0) + { + return 0; + } + for (size_t I = 0; I < kTimelineLodCount; ++I) + { + if (kTimelineLodResolutions[I] >= ResolutionUs) + { + return I + 1; + } + } + return kTimelineLodCount; + } + + const eastl::vector<TimelineScope>& LodScopes(const ThreadTimeline& Timeline, size_t LodIndex) + { + if (LodIndex == 0) + { + return Timeline.Scopes; + } + return Timeline.DetailLevels[LodIndex - 1].Scopes; + } + + void ExtractScopesInto(const ThreadTimeline& Timeline, const TimelineQueryRequest& Req, std::vector<TimelineScopeView>& Out) + { + const eastl::vector<TimelineScope>& Scopes = LodScopes(Timeline, SelectLodIndex(Req.ResolutionUs)); + + auto MidIt = + std::lower_bound(Scopes.begin(), Scopes.end(), Req.StartUs, [](const TimelineScope& S, uint32_t V) { return S.BeginUs < V; }); + + for (auto It = Scopes.begin(); It != MidIt; ++It) + { + if ((It->BeginUs + It->DurationUs) < Req.StartUs || It->DurationUs < Req.MinDurUs) + { + continue; + } + Out.push_back({It->BeginUs, It->DurationUs, It->NameId, It->Depth, It->MergeCount}); + } + for (auto It = MidIt; It != Scopes.end(); ++It) + { + if (It->BeginUs > Req.EndUs) + { + break; + } + if (It->DurationUs < Req.MinDurUs) + { + continue; + } + Out.push_back({It->BeginUs, It->DurationUs, It->NameId, It->Depth, It->MergeCount}); + } + } + + const ThreadTimeline* FindThread(const TraceModel& Model, uint32_t ThreadId) + { + auto It = std::find_if(Model.Timelines.begin(), Model.Timelines.end(), [ThreadId](const ThreadTimeline& T) { + return T.ThreadId == ThreadId; + }); + return (It != Model.Timelines.end()) ? &*It : nullptr; + } + + class InMemoryTimelineQuery final : public TimelineQuery + { + public: + explicit InMemoryTimelineQuery(const TraceModel& Model) : m_Model(Model) {} + + void QueryThread(uint32_t ThreadId, const TimelineQueryRequest& Req, std::vector<TimelineScopeView>& Out) const override + { + const ThreadTimeline* Timeline = FindThread(m_Model, ThreadId); + if (Timeline) + { + ExtractScopesInto(*Timeline, Req, Out); + } + } + + void QueryBatch(std::span<const uint32_t> ThreadIds, const TimelineQueryRequest& Req, BatchResult& Out) const override + { + Out.Scopes.clear(); + Out.Ranges.clear(); + Out.Ranges.reserve(ThreadIds.size()); + + for (uint32_t ThreadId : ThreadIds) + { + const uint32_t Begin = uint32_t(Out.Scopes.size()); + const ThreadTimeline* Timeline = FindThread(m_Model, ThreadId); + if (Timeline) + { + ExtractScopesInto(*Timeline, Req, Out.Scopes); + } + Out.Ranges.push_back({Begin, uint32_t(Out.Scopes.size())}); + } + } + + private: + const TraceModel& m_Model; + }; + +} // namespace + +std::unique_ptr<TimelineQuery> +MakeInMemoryTimelineQuery(const TraceModel& Model) +{ + return std::make_unique<InMemoryTimelineQuery>(Model); +} + +} // namespace zen::trace_detail diff --git a/src/zen/trace/timeline_query.h b/src/zen/trace/timeline_query.h new file mode 100644 index 000000000..f773d8e58 --- /dev/null +++ b/src/zen/trace/timeline_query.h @@ -0,0 +1,69 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "trace_model.h" + +#include <cstdint> +#include <memory> +#include <span> +#include <vector> + +namespace zen::trace_detail { + +// Plain-data view of a single timeline scope returned by a TimelineQuery. +// Mirrors the on-disk TimelineScope but is intentionally decoupled from the +// in-memory model so that alternative backends can share the same result type. +struct TimelineScopeView +{ + uint32_t BeginUs; + uint32_t DurationUs; + uint32_t NameId; + uint16_t Depth; + uint16_t MergeCount; // 0 == raw LOD 0, N>0 == N merged scopes +}; + +// Common parameters for a viewport-style timeline query. +struct TimelineQueryRequest +{ + uint32_t StartUs; + uint32_t EndUs; + uint32_t MinDurUs; + uint32_t ResolutionUs; // 0 == LOD 0 (raw); >0 picks the smallest LOD with ResolutionUs >= this +}; + +// Backend-agnostic interface for serving timeline scope data to the trace +// viewer HTTP handlers. Currently only the in-memory implementation exists, +// but the abstraction is preserved as a clean swap point if a different +// backend (e.g. on-disk indexed store) ever becomes useful. +class TimelineQuery +{ +public: + virtual ~TimelineQuery() = default; + + // Append all scopes for a single thread matching the request to Out. + // Out is not cleared; callers can chain queries into the same buffer. + virtual void QueryThread(uint32_t ThreadId, const TimelineQueryRequest& Req, std::vector<TimelineScopeView>& Out) const = 0; + + // Result of a batch query: a single flat scope vector plus per-thread + // ranges into it. Ranges[i] corresponds to ThreadIds[i] from the request. + struct BatchResult + { + struct Range + { + uint32_t Begin; + uint32_t End; + }; + std::vector<TimelineScopeView> Scopes; + std::vector<Range> Ranges; + }; + + // Query several threads in one call. Out is cleared before being filled. + virtual void QueryBatch(std::span<const uint32_t> ThreadIds, const TimelineQueryRequest& Req, BatchResult& Out) const = 0; +}; + +// In-memory implementation. Holds a reference to Model — the model must +// outlive the returned object. +std::unique_ptr<TimelineQuery> MakeInMemoryTimelineQuery(const TraceModel& Model); + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_analyze.cpp b/src/zen/trace/trace_analyze.cpp new file mode 100644 index 000000000..ff168cd9c --- /dev/null +++ b/src/zen/trace/trace_analyze.cpp @@ -0,0 +1,812 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_analyze.h" + +#include "callstack_formatter.h" +#include "trace_cache.h" +#include "zen.h" + +#include <zencore/basicfile.h> +#include <zencore/fmtutils.h> +#include <zencore/iobuffer.h> +#include <zencore/logging.h> +#include <zencore/scopeguard.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <zencore/workthreadpool.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/hash_map.h> +#include <EASTL/hash_set.h> +#include <EASTL/vector.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <algorithm> + +namespace { + +using namespace zen::trace_detail; + +static void +AppendHtmlEscaped(zen::StringBuilderBase& Out, std::string_view Text) +{ + for (char Ch : Text) + { + switch (Ch) + { + case '&': + Out << "&"; + break; + case '<': + Out << "<"; + break; + case '>': + Out << ">"; + break; + case '"': + Out << """; + break; + case '\'': + Out << "'"; + break; + default: + Out.Append(Ch); + break; + } + } +} + +static CallstackFilterOptions +BuildCallstackFilterOptions(const AnalyzeOptions& Options) +{ + CallstackFilterOptions Result; + Result.EnableHeuristic = Options.EnableCallstackHeuristic; + Result.SkipPatterns = Options.CallstackSkipPatterns; + return Result; +} + +static std::string +BuildThreadSummary(const TraceModel& Model, const eastl::fixed_vector<uint32_t, 4, true>& ThreadIds) +{ + std::string Result; + for (uint32_t Tid : ThreadIds) + { + if (!Result.empty()) + { + Result += ", "; + } + auto TIt = std::find_if(Model.Threads.begin(), Model.Threads.end(), [Tid](const ThreadInfoEntry& T) { return T.ThreadId == Tid; }); + if (TIt != Model.Threads.end() && !TIt->Name.empty()) + { + Result += TIt->Name; + } + else + { + Result += fmt::format("tid:{}", Tid); + } + } + return Result; +} + +static void +AppendHtmlCallstack(zen::StringBuilderBase& Out, const AnalyzeOptions& Options, CallstackFormatter& Formatter, uint32_t CallstackId) +{ + const CallstackEntry* Entry = Formatter.FindCallstackEntry(CallstackId); + if (Entry == nullptr || Entry->Frames.empty()) + { + Out << "<div class=\"muted\">No callstack frames recorded.</div>"; + return; + } + + FilteredCallstackView Filtered = Formatter.BuildView(*Entry, BuildCallstackFilterOptions(Options)); + if (Filtered.HiddenPrefixCount > 0) + { + Out << "<div class=\"muted\">Skipped " << uint64_t(Filtered.HiddenPrefixCount) << " leading frame(s)"; + if (Filtered.IncludedThirdPartyBoundary) + { + Out << "; kept boundary third-party callsite"; + } + Out << ".</div>"; + } + + Out << "<ol class=\"frames\">"; + for (const FilteredCallstackFrame& Frame : Filtered.Frames) + { + Out << "<li>"; + AppendHtmlEscaped(Out, Frame.Display); + Out << "</li>"; + } + Out << "</ol>"; +} + +static std::string_view +FindHeapName(const TraceModel& Model, uint32_t HeapId) +{ + for (const HeapInfo& Heap : Model.Heaps) + { + if (Heap.Id == HeapId && !Heap.Name.empty()) + { + return Heap.Name; + } + } + return "unknown"; +} + +static bool +PassesChurnThreshold(const AnalyzeOptions& Options, const CallstackChurnStat& Stat) +{ + return Stat.MeanDistance <= double(Options.ChurnDistanceThreshold); +} + +static uint64_t +CountShownChurnSites(const TraceModel& Model, const AnalyzeOptions& Options, uint64_t Limit = 100) +{ + uint64_t Result = 0; + for (const CallstackChurnStat& Stat : Model.ChurnStats) + { + if (PassesChurnThreshold(Options, Stat) && Result < Limit) + { + ++Result; + } + } + return Result; +} + +class ConsoleAnalyzeWriter +{ +public: + ConsoleAnalyzeWriter(const TraceModel& InModel, + const AnalyzeOptions& InOptions, + const std::filesystem::path& InFilePath, + CallstackFormatter& InFrameFormatter) + : m_Model(InModel) + , m_Options(InOptions) + , m_FilePath(InFilePath) + , m_FrameFormatter(InFrameFormatter) + { + } + + void Write() const + { + AppendSession(); + AppendGeneralSummary(); + AppendEventTypes(); + AppendThreads(); + AppendChannels(); + AppendCpuScopeStats(); + AppendMemorySummary(); + AppendLiveAllocationCallstacks(); + AppendChurnCallstacks(); + } + +private: + void AppendSession() const + { + const SessionInfo& Session = m_Model.Session; + if (!Session.HasSession) + { + return; + } + + ZEN_CONSOLE("Session:"); + if (!Session.Platform.empty()) + { + ZEN_CONSOLE(" Platform: {}", Session.Platform); + } + if (!Session.AppName.empty()) + { + ZEN_CONSOLE(" App: {}", Session.AppName); + } + if (!Session.ProjectName.empty()) + { + ZEN_CONSOLE(" Project: {}", Session.ProjectName); + } + if (!Session.Branch.empty()) + { + ZEN_CONSOLE(" Branch: {}", Session.Branch); + } + if (!Session.BuildVersion.empty()) + { + ZEN_CONSOLE(" Build: {}", Session.BuildVersion); + } + if (Session.ConfigurationType != 0) + { + constexpr const char* kConfigNames[] = {"Unknown", "Debug", "DebugGame", "Development", "Shipping", "Test"}; + uint8_t Idx = Session.ConfigurationType; + const char* Name = (Idx < std::size(kConfigNames)) ? kConfigNames[Idx] : "Unknown"; + ZEN_CONSOLE(" Config: {}", Name); + } + if (Session.Changelist != 0) + { + ZEN_CONSOLE(" CL: {}", Session.Changelist); + } + if (!Session.CommandLine.empty()) + { + ZEN_CONSOLE(" Cmd: {}", Session.CommandLine); + } + ZEN_CONSOLE(""); + } + + void AppendGeneralSummary() const + { + uint64_t DurationUs = (m_Model.TraceEndUs > m_Model.TraceStartUs) ? (m_Model.TraceEndUs - m_Model.TraceStartUs) : 0; + + ZEN_CONSOLE("Trace: {}", m_FilePath); + ZEN_CONSOLE("Size: {}", zen::NiceBytes(m_Model.FileSize)); + ZEN_CONSOLE("Events: {}", zen::ThousandsNum(m_Model.TotalEvents)); + ZEN_CONSOLE("Duration: {}", zen::NiceTimeSpanMs((DurationUs + 500) / 1000)); + ZEN_CONSOLE("Threads: {}", m_Model.Threads.size()); + ZEN_CONSOLE("Modules: {}", m_Model.Modules.size()); + ZEN_CONSOLE("Parsed: {}", zen::NiceTimeSpanMs(m_Model.ParseTimeMs)); + if (m_Model.ParseTimeMs > 0) + { + ZEN_CONSOLE("Rate: {} events/s", zen::ThousandsNum(m_Model.TotalEvents * 1000 / m_Model.ParseTimeMs)); + } + ZEN_CONSOLE(""); + } + + void AppendEventTypes() const + { + if (m_Model.EventTypeCounts.empty()) + { + return; + } + + size_t MaxNameLen = 10; + for (const auto& Entry : m_Model.EventTypeCounts) + { + MaxNameLen = std::max(MaxNameLen, Entry.Name.size()); + } + + ZEN_CONSOLE("{:<{}} {:>14}", "Event Type", MaxNameLen, "Count"); + ZEN_CONSOLE("{:-<{}}", "", MaxNameLen + 16); + for (const auto& Entry : m_Model.EventTypeCounts) + { + ZEN_CONSOLE("{:<{}} {:>14}", Entry.Name, MaxNameLen, zen::ThousandsNum(Entry.Count)); + } + ZEN_CONSOLE(""); + } + + void AppendThreads() const + { + if (m_Model.Threads.empty()) + { + return; + } + + ZEN_CONSOLE("Threads:"); + for (const ThreadInfoEntry& Thread : m_Model.Threads) + { + auto TimelineIt = std::find_if(m_Model.Timelines.begin(), + m_Model.Timelines.end(), + [Tid = Thread.ThreadId](const ThreadTimeline& T) { return T.ThreadId == Tid; }); + uint64_t ScopeCount = (TimelineIt != m_Model.Timelines.end()) ? TimelineIt->Scopes.size() : 0; + + if (!Thread.Name.empty()) + { + ZEN_CONSOLE(" {:>5} {:<32} {} scopes", Thread.ThreadId, Thread.Name, zen::ThousandsNum(ScopeCount)); + } + } + ZEN_CONSOLE(""); + } + + void AppendChannels() const + { + if (m_Model.Channels.empty()) + { + return; + } + + ZEN_CONSOLE("Channels:"); + for (const ChannelInfo& Channel : m_Model.Channels) + { + ZEN_CONSOLE(" {:<32} {}", Channel.Name, Channel.Enabled ? "enabled" : "disabled"); + } + ZEN_CONSOLE(""); + } + + void AppendCpuScopeStats() const + { + if (m_Model.ScopeStats.empty()) + { + return; + } + + ZEN_CONSOLE("CPU Profiling Scopes:"); + ZEN_CONSOLE(""); + ZEN_CONSOLE("{:<48} {:>8} {:>9} {:>9} {:>9} {:>9}", "Scope", "Count", "Min(ms)", "Mean(ms)", "Max(ms)", "SD(ms)"); + ZEN_CONSOLE("{:-<{}}", "", 48 + 8 + 9 + 9 + 9 + 9 + 5); + + constexpr double UsToMs = 1.0 / 1000.0; + for (const CpuScopeStat& Stat : m_Model.ScopeStats) + { + if (Stat.MaxUs < 500) + { + continue; + } + + ZEN_CONSOLE("{:<48.48} {:>8} {:>9.3f} {:>9.3f} {:>9.3f} {:>9.3f}", + Stat.Name, + zen::ThousandsNum(Stat.Count), + double(Stat.MinUs) * UsToMs, + Stat.MeanUs * UsToMs, + double(Stat.MaxUs) * UsToMs, + Stat.StdDevUs * UsToMs); + } + ZEN_CONSOLE(""); + } + + void AppendMemorySummary() const + { + const AllocationSummary& AllocSummary = m_Model.AllocSummary; + if (!AllocSummary.HasMemoryData) + { + return; + } + + ZEN_CONSOLE("Memory Allocations:"); + ZEN_CONSOLE(""); + ZEN_CONSOLE(" Allocs: {}", zen::ThousandsNum(AllocSummary.TotalAllocs)); + ZEN_CONSOLE(" Frees: {}", zen::ThousandsNum(AllocSummary.TotalFrees)); + ZEN_CONSOLE(" Reallocs: {} alloc / {} free", + zen::ThousandsNum(AllocSummary.TotalReallocAllocs), + zen::ThousandsNum(AllocSummary.TotalReallocFrees)); + ZEN_CONSOLE(" Peak: {}", zen::NiceBytes(uint64_t(AllocSummary.PeakBytes))); + ZEN_CONSOLE(" End: {}", zen::NiceBytes(uint64_t(AllocSummary.EndBytes))); + ZEN_CONSOLE(" Live allocs: {}", zen::ThousandsNum(AllocSummary.LiveAllocations)); + + if (!m_Model.HeapStats.empty()) + { + ZEN_CONSOLE(""); + ZEN_CONSOLE(" {:<20} {:>14} {:>14} {:>10} {:>10}", "Heap", "Current", "Peak", "Allocs", "Frees"); + ZEN_CONSOLE(" {:-<{}}", "", 20 + 14 + 14 + 10 + 10 + 4); + + for (const HeapStat& Stat : m_Model.HeapStats) + { + std::string_view HeapName = FindHeapName(m_Model, Stat.HeapId); + + ZEN_CONSOLE(" {:<20.20} {:>14} {:>14} {:>10} {:>10}", + HeapName, + zen::NiceBytes(uint64_t(Stat.CurrentBytes)), + zen::NiceBytes(uint64_t(Stat.PeakBytes)), + zen::ThousandsNum(Stat.AllocCount), + zen::ThousandsNum(Stat.FreeCount)); + } + } + ZEN_CONSOLE(""); + } + + void PrintCallstack(uint32_t CallstackId) const + { + const CallstackEntry* Entry = m_FrameFormatter.FindCallstackEntry(CallstackId); + if (Entry == nullptr) + { + return; + } + + FilteredCallstackView Filtered = m_FrameFormatter.BuildView(*Entry, BuildCallstackFilterOptions(m_Options)); + if (Filtered.HiddenPrefixCount > 0) + { + if (Filtered.IncludedThirdPartyBoundary) + { + ZEN_CONSOLE(" [skipped {} leading frame(s); kept boundary third-party callsite]", Filtered.HiddenPrefixCount); + } + else + { + ZEN_CONSOLE(" [skipped {} leading frame(s)]", Filtered.HiddenPrefixCount); + } + } + for (const FilteredCallstackFrame& Frame : Filtered.Frames) + { + ZEN_CONSOLE(" {}", Frame.Display); + } + } + + void AppendLiveAllocationCallstacks() const + { + if (m_Options.LiveAllocsLimit <= 0 || m_Model.CallstackStats.empty()) + { + return; + } + + size_t Count = std::min(size_t(m_Options.LiveAllocsLimit), m_Model.CallstackStats.size()); + ZEN_CONSOLE("Live Allocation Callstacks (top {} by bytes):", Count); + ZEN_CONSOLE(""); + + for (size_t I = 0; I < Count; ++I) + { + const CallstackAllocStat& Stat = m_Model.CallstackStats[I]; + std::string ThreadInfo = BuildThreadSummary(m_Model, Stat.ThreadIds); + ZEN_CONSOLE(" #{} {} in {} allocation(s) [callstack {}, {}]", + I + 1, + zen::NiceBytes(uint64_t(Stat.LiveBytes)), + zen::ThousandsNum(Stat.LiveCount), + Stat.CallstackId, + ThreadInfo); + PrintCallstack(Stat.CallstackId); + ZEN_CONSOLE(""); + } + } + + void AppendChurnCallstacks() const + { + if (m_Options.ChurnLimit <= 0 || m_Model.ChurnStats.empty()) + { + return; + } + + size_t Emitted = 0; + size_t Limit = size_t(m_Options.ChurnLimit); + ZEN_CONSOLE("Allocation Churn (top {}, event distance <= {}):", Limit, m_Options.ChurnDistanceThreshold); + ZEN_CONSOLE(""); + + for (const CallstackChurnStat& Stat : m_Model.ChurnStats) + { + if (Emitted >= Limit) + { + break; + } + if (!PassesChurnThreshold(m_Options, Stat)) + { + continue; + } + + ZEN_CONSOLE(" #{} {} short-lived allocs ({} total), {} churned, avg distance {:.0f} events [callstack {}]", + Emitted + 1, + zen::ThousandsNum(Stat.ChurnAllocs), + zen::ThousandsNum(Stat.TotalAllocs), + zen::NiceBytes(Stat.ChurnBytes), + Stat.MeanDistance, + Stat.CallstackId); + PrintCallstack(Stat.CallstackId); + ZEN_CONSOLE(""); + ++Emitted; + } + } + + const TraceModel& m_Model; + const AnalyzeOptions& m_Options; + const std::filesystem::path& m_FilePath; + CallstackFormatter& m_FrameFormatter; +}; + +class HtmlReportWriter +{ +public: + HtmlReportWriter(const TraceModel& InModel, + const AnalyzeOptions& InOptions, + const std::filesystem::path& InFilePath, + CallstackFormatter& InFrameFormatter) + : m_Model(InModel) + , m_Options(InOptions) + , m_FilePath(InFilePath) + , m_FrameFormatter(InFrameFormatter) + { + } + + void Write(const std::filesystem::path& OutputPath) + { + AppendDocument(); + zen::WriteFile(OutputPath, zen::IoBuffer(zen::IoBuffer::Wrap, m_Html.Data(), m_Html.Size())); + } + +private: + void AppendDocument() + { + m_Html << "<!doctype html><html><head><meta charset=\"utf-8\"><title>zen trace analyze report</title>"; + AppendStyles(); + m_Html << "</head><body>"; + AppendHeader(); + AppendSummaryCards(); + AppendLeaksSection(); + AppendChurnSection(); + m_Html << "</body></html>"; + } + + void AppendStyles() + { + m_Html << "<style>body{font:14px/1.45 system-ui,-apple-system,Segoe " + "UI,Roboto,sans-serif;margin:24px;color:#1f2937;background:#f8fafc;}"; + m_Html << "h1,h2{margin:0 0 12px;}h1{font-size:28px;}h2{font-size:20px;margin-top:28px;}"; + m_Html << ".meta,.card,details{background:#fff;border:1px solid #dbe2ea;border-radius:10px;box-shadow:0 1px 2px rgba(0,0,0,.04);}"; + m_Html << ".meta,.card{padding:16px;margin-bottom:16px;}.report-table{width:100%;border-collapse:collapse;background:#fff;border:" + "1px solid #dbe2ea;border-radius:10px;overflow:hidden;table-layout:fixed;}"; + m_Html << "th,td{padding:10px 12px;border-bottom:1px solid " + "#e5e7eb;vertical-align:top;text-align:left;}th{background:#f1f5f9;font-weight:600;}tr:last-child td{border-bottom:0;}"; + m_Html + << "th.num,td.num{text-align:right;white-space:nowrap;font-variant-numeric:tabular-nums;}.muted{color:#64748b;}.pill{display:" + "inline-block;padding:2px 8px;border-radius:999px;background:#e2e8f0;color:#334155;font-size:12px;margin-right:6px;}"; + m_Html << ".col-rank{width:56px;}.col-live-bytes,.col-alloc-count,.col-short-lived,.col-churn-bytes,.col-total-allocs,.col-avg-" + "distance{width:132px;}.col-threads{width:260px;}.col-callstack{width:auto;}"; + m_Html << ".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:12px;margin-bottom:18px;}details{" + "display:block;width:100%;box-sizing:border-box;padding:12px " + "14px;margin:0;}summary{cursor:pointer;font-weight:600;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;}"; + m_Html << ".callstack-cell{width:100%;}.frames{margin:10px 0 0 " + "20px;padding:0;font-family:ui-monospace,SFMono-Regular,Consolas,monospace;font-size:12px;}.frames li{margin:4px " + "0;overflow-wrap:anywhere;word-break:break-word;}code{font-family:ui-monospace,SFMono-Regular,Consolas,monospace;}a{" + "color:#2563eb;text-decoration:none;}a:hover{text-decoration:underline;}</style>"; + } + + void AppendHeader() + { + m_Html << "<h1>zen trace analyze memory report</h1>"; + m_Html << "<div class=\"meta\"><div><span class=\"pill\">offline HTML</span><span class=\"pill\">top 100 churn " + "sites</span></div><p><strong>Trace:</strong> <code>"; + AppendHtmlEscaped(m_Html, m_FilePath.string()); + m_Html << "</code></p>"; + if (m_Model.Session.HasSession && !m_Model.Session.AppName.empty()) + { + m_Html << "<p><strong>App:</strong> "; + AppendHtmlEscaped(m_Html, m_Model.Session.AppName); + m_Html << "</p>"; + } + m_Html << "<p class=\"muted\">Generated by zen trace analyze. Churn threshold: "; + AppendHtmlEscaped(m_Html, fmt::format("{} events", m_Options.ChurnDistanceThreshold)); + m_Html << "</p></div>"; + } + + void AppendSummaryCards() + { + uint64_t DurationUs = (m_Model.TraceEndUs > m_Model.TraceStartUs) ? (m_Model.TraceEndUs - m_Model.TraceStartUs) : 0; + m_Html << "<div class=\"grid\">"; + m_Html << "<div class=\"card\"><div class=\"muted\">Trace size</div><div><strong>" << zen::NiceBytes(m_Model.FileSize) + << "</strong></div></div>"; + m_Html << "<div class=\"card\"><div class=\"muted\">Duration</div><div><strong>" << zen::NiceTimeSpanMs((DurationUs + 500) / 1000) + << "</strong></div></div>"; + m_Html << "<div class=\"card\"><div class=\"muted\">Peak memory</div><div><strong>" + << zen::NiceBytes(uint64_t(m_Model.AllocSummary.PeakBytes)) << "</strong></div></div>"; + m_Html << "<div class=\"card\"><div class=\"muted\">End memory</div><div><strong>" + << zen::NiceBytes(uint64_t(m_Model.AllocSummary.EndBytes)) << "</strong></div></div>"; + m_Html << "<div class=\"card\"><div class=\"muted\">Live allocations</div><div><strong>" + << zen::ThousandsNum(m_Model.AllocSummary.LiveAllocations) << "</strong></div></div>"; + m_Html << "<div class=\"card\"><div class=\"muted\">Leak callstacks</div><div><strong>" + << zen::ThousandsNum(m_Model.CallstackStats.size()) << "</strong></div></div>"; + m_Html << "<div class=\"card\"><div class=\"muted\">Churn sites shown</div><div><strong>" + << zen::ThousandsNum(::CountShownChurnSites(m_Model, m_Options)) << "</strong></div></div>"; + m_Html << "</div>"; + } + + void AppendLeaksSection() + { + m_Html << "<h2 id=\"leaks\">Memory leaks (all live-allocation callstacks)</h2>"; + if (m_Model.CallstackStats.empty()) + { + m_Html << "<div class=\"card muted\">No live allocation callstacks were present at the end of the trace.</div>"; + return; + } + + m_Html << "<table class=\"report-table\"><colgroup><col class=\"col-rank\"><col class=\"col-live-bytes\"><col " + "class=\"col-alloc-count\"><col class=\"col-threads\"><col class=\"col-callstack\"></colgroup><thead><tr><th " + "class=\"num\">#</th><th class=\"num\">Live bytes</th><th class=\"num\">Alloc " + "count</th><th>Threads</th><th>Callstack</th></tr></thead><tbody>"; + for (size_t I = 0; I < m_Model.CallstackStats.size(); ++I) + { + const CallstackAllocStat& Stat = m_Model.CallstackStats[I]; + std::string ThreadInfo = BuildThreadSummary(m_Model, Stat.ThreadIds); + m_Html << "<tr><td class=\"num\">" << uint64_t(I + 1) << "</td><td class=\"num\">" << zen::NiceBytes(uint64_t(Stat.LiveBytes)) + << "</td><td class=\"num\">" << zen::ThousandsNum(Stat.LiveCount) << "</td><td>"; + AppendHtmlEscaped(m_Html, ThreadInfo); + m_Html << "</td><td class=\"callstack-cell\"><details><summary>Callstack " << Stat.CallstackId << "</summary>"; + AppendHtmlCallstack(m_Html, m_Options, m_FrameFormatter, Stat.CallstackId); + m_Html << "</details></td></tr>"; + } + m_Html << "</tbody></table>"; + } + + void AppendChurnSection() + { + m_Html << "<h2 id=\"churn\">Allocation churn sites (top 100)</h2>"; + if (m_Model.ChurnStats.empty()) + { + m_Html << "<div class=\"card muted\">No churn statistics were available in this trace.</div>"; + return; + } + + m_Html << "<table class=\"report-table\"><colgroup><col class=\"col-rank\"><col class=\"col-short-lived\"><col " + "class=\"col-churn-bytes\"><col class=\"col-total-allocs\"><col class=\"col-avg-distance\"><col " + "class=\"col-callstack\"></colgroup><thead><tr><th class=\"num\">#</th><th class=\"num\">Short-lived allocs</th><th " + "class=\"num\">Churn bytes</th><th class=\"num\">Total allocs</th><th class=\"num\">Avg " + "distance</th><th>Callstack</th></tr></thead><tbody>"; + size_t Emitted = 0; + for (const CallstackChurnStat& Stat : m_Model.ChurnStats) + { + if (Emitted >= 100) + { + break; + } + if (!PassesChurnThreshold(m_Options, Stat)) + { + continue; + } + m_Html << "<tr><td class=\"num\">" << uint64_t(Emitted + 1) << "</td><td class=\"num\">" << zen::ThousandsNum(Stat.ChurnAllocs) + << "</td><td class=\"num\">" << zen::NiceBytes(Stat.ChurnBytes) << "</td><td class=\"num\">" + << zen::ThousandsNum(Stat.TotalAllocs) << "</td><td class=\"num\">" << fmt::format("{:.0f} events", Stat.MeanDistance) + << "</td><td class=\"callstack-cell\"><details><summary>Callstack " << Stat.CallstackId << "</summary>"; + AppendHtmlCallstack(m_Html, m_Options, m_FrameFormatter, Stat.CallstackId); + m_Html << "</details></td></tr>"; + ++Emitted; + } + m_Html << "</tbody></table>"; + } + + const TraceModel& m_Model; + const AnalyzeOptions& m_Options; + const std::filesystem::path& m_FilePath; + CallstackFormatter& m_FrameFormatter; + zen::ExtendableStringBuilder<32768> m_Html; +}; + +static void +WriteAnalyzeHtmlReport(const TraceModel& Model, + const AnalyzeOptions& Options, + const std::filesystem::path& FilePath, + CallstackFormatter& FrameFormatter) +{ + std::filesystem::path OutputPath = std::filesystem::absolute(Options.HtmlReportPath); + if (OutputPath.empty()) + { + return; + } + + std::error_code Ec; + std::filesystem::path ParentPath = OutputPath.parent_path(); + if (!ParentPath.empty()) + { + std::filesystem::create_directories(ParentPath, Ec); + } + + HtmlReportWriter Writer(Model, Options, FilePath, FrameFormatter); + Writer.Write(OutputPath); + ZEN_CONSOLE("HTML report: {}", OutputPath.string()); +} + +} // namespace + +namespace zen::trace_detail { + +void +RunAnalyze(const std::filesystem::path& FilePath, const AnalyzeOptions& Options) +{ + std::filesystem::path CachePath = FilePath; + CachePath.replace_extension(".ucache_z"); + + TraceModel Model; + std::unique_ptr<SymbolResolver> Symbols; + bool LoadedFromCache = false; + + // Try loading from cache + if (!Options.NoCache) + { + std::optional<CachedAnalysis> Cached = TryLoadAnalyzeCache(CachePath, FilePath); + if (Cached) + { + Model = std::move(Cached->Model); + Symbols = std::move(Cached->Symbols); + LoadedFromCache = true; + } + } + + if (!LoadedFromCache) + { + WorkerThreadPool ThreadPool(gsl::narrow<int>(GetHardwareConcurrency())); + Model = BuildTraceModel(FilePath, ThreadPool); + + if (Options.Symbols != SymbolBackend::Off) + { + Symbols = CreateSymbolResolver(Options.Symbols); + for (const ModuleInfo& Mod : Model.Modules) + { + Symbols->LoadModule(Mod); + } + } + } + + CallstackFormatter FrameFormatter(Model, Symbols.get()); + ConsoleAnalyzeWriter ConsoleWriter(Model, Options, FilePath, FrameFormatter); + ConsoleWriter.Write(); + + if (!Options.HtmlReportPath.empty()) + { + WriteAnalyzeHtmlReport(Model, Options, FilePath, FrameFormatter); + } + + // Write cache on fresh parse + if (!LoadedFromCache && !Options.NoCache) + { + // Build the complete symbol map for the cache. Start with whatever + // the formatter already resolved during display, then resolve every + // remaining callstack address in parallel. + eastl::hash_map<uint64_t, std::string> AllSymbols = FrameFormatter.GetResolvedCache(); + + // Collect unique addresses that still need resolving. + eastl::hash_set<uint64_t> Needed; + for (const CallstackEntry& CS : Model.Callstacks) + { + for (const ResolvedFrame& Frame : CS.Frames) + { + if (AllSymbols.find(Frame.Address) == AllSymbols.end()) + { + Needed.insert(Frame.Address); + } + } + } + + if (!Needed.empty() && Symbols) + { + // Flatten to a vector so we can partition into chunks. + eastl::vector<uint64_t> Addresses(Needed.begin(), Needed.end()); + Needed.clear(); + + uint32_t ThreadCount = gsl::narrow<uint32_t>(GetHardwareConcurrency()); + WorkerThreadPool ResolvePool(gsl::narrow<int>(ThreadCount)); + + // Each worker resolves a chunk and writes into its own local map. + eastl::vector<eastl::hash_map<uint64_t, std::string>> PerThread(ThreadCount); + uint32_t ChunkSize = uint32_t((Addresses.size() + ThreadCount - 1) / ThreadCount); + + Latch Done(ThreadCount); + for (uint32_t T = 0; T < ThreadCount; ++T) + { + uint32_t Begin = T * ChunkSize; + uint32_t End = std::min(Begin + ChunkSize, uint32_t(Addresses.size())); + if (Begin >= End) + { + Done.CountDown(); + continue; + } + + ResolvePool.ScheduleWork( + [&Addresses, &PerThread, &Model, &Symbols, &Done, T, Begin, End]() { + auto _ = MakeGuard([&Done]() { Done.CountDown(); }); + for (uint32_t I = Begin; I < End; ++I) + { + uint64_t Addr = Addresses[I]; + std::string Symbol = Symbols->Resolve(Addr); + if (!Symbol.empty()) + { + PerThread[T].emplace(Addr, std::move(Symbol)); + } + } + }, + WorkerThreadPool::EMode::EnableBacklog); + } + Done.Wait(); + + // Merge per-thread results. + for (auto& Map : PerThread) + { + for (auto& [Addr, Sym] : Map) + { + AllSymbols.emplace(Addr, std::move(Sym)); + } + } + } + + // Fill in module-name fallbacks for any addresses not resolved by the + // symbol resolver (same logic as CallstackFormatter::Describe). + for (const CallstackEntry& CS : Model.Callstacks) + { + for (const ResolvedFrame& Frame : CS.Frames) + { + if (AllSymbols.find(Frame.Address) != AllSymbols.end()) + { + continue; + } + std::string Fallback; + if (Frame.ModuleIndex != ~0u && Frame.ModuleIndex < Model.Modules.size()) + { + Fallback = fmt::format("{} + 0x{:X}", Model.Modules[Frame.ModuleIndex].Name, Frame.Offset); + } + else + { + Fallback = fmt::format("0x{:X}", Frame.Address); + } + AllSymbols.emplace(Frame.Address, std::move(Fallback)); + } + } + + WriteAnalyzeCache(CachePath, FilePath, Model, AllSymbols); + } +} + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_analyze.h b/src/zen/trace/trace_analyze.h new file mode 100644 index 000000000..7b6f4fccd --- /dev/null +++ b/src/zen/trace/trace_analyze.h @@ -0,0 +1,29 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "symbol_resolver.h" +#include "trace_model.h" + +#include <cstdint> +#include <filesystem> +#include <string> +#include <vector> + +namespace zen::trace_detail { + +struct AnalyzeOptions +{ + int LiveAllocsLimit = 50; // 0 = off + int ChurnLimit = 0; // 0 = off; top N churny callstacks + uint64_t ChurnDistanceThreshold = 1000; // event distance: allocs freed within N events are "churny" + SymbolBackend Symbols = SymbolBackend(1); // Pdb (default) + std::filesystem::path HtmlReportPath; // empty = off; standalone offline memory HTML report + bool NoCache = false; // skip reading/writing the .ucache_z cache + bool EnableCallstackHeuristic = true; // skip leading low-level / third-party frames while keeping the boundary callsite + std::vector<std::string> CallstackSkipPatterns; // wildcard patterns matched against symbol, module name, and module path +}; + +void RunAnalyze(const std::filesystem::path& FilePath, const AnalyzeOptions& Options = {}); + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_cache.cpp b/src/zen/trace/trace_cache.cpp new file mode 100644 index 000000000..165c1eecf --- /dev/null +++ b/src/zen/trace/trace_cache.cpp @@ -0,0 +1,1104 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_cache.h" + +#include <zencore/basicfile.h> +#include <zencore/compress.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/iohash.h> +#include <zencore/logging.h> +#include <zencore/stream.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/sort.h> +#include <EASTL/vector.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <filesystem> + +namespace zen::trace_detail { + +// =========================================================================== +// StringTableBuilder — write-path helper that deduplicates and packs strings +// +// Strings are appended back-to-back (null-terminated) in a single contiguous +// block. Deduplication is keyed by (offset, length) pairs into that block so +// no separate string copies are made. To look up an incoming string_view we +// speculatively append it, build a key, and look it up. On duplicate the +// append is rolled back by truncating the buffer. +// =========================================================================== + +class StringTableBuilder +{ +public: + StringTableBuilder() : m_IndexMap(0, StringHash{&m_Packed}, StringEq{&m_Packed}) { m_Packed.reserve(4096); } + + // Intern a string and return its index. Deduplicates across calls. + uint32_t Intern(std::string_view Str) + { + // Speculatively append the string so that the hash/eq functors can + // read it from the packed buffer (avoids dangling string_view keys). + uint32_t SpecOffset = uint32_t(m_Packed.size()); + uint32_t SpecLength = uint32_t(Str.size()); + + m_Packed.resize(m_Packed.size() + Str.size() + 1); + if (!Str.empty()) + { + memcpy(m_Packed.data() + SpecOffset, Str.data(), Str.size()); + } + m_Packed[SpecOffset + Str.size()] = '\0'; + + StringKey Key{SpecOffset, SpecLength}; + auto It = m_IndexMap.find(Key); + if (It != m_IndexMap.end()) + { + // Duplicate — roll back the speculative append. + m_Packed.resize(SpecOffset); + return It->second; + } + + // New string — keep the append and record its index. + uint32_t Index = uint32_t(m_Offsets.size()); + m_Offsets.push_back(SpecOffset); + m_IndexMap.emplace(Key, Index); + return Index; + } + + // Serialize: [uint32_t count][uint32_t offsets[count]][packed strings] + SharedBuffer Serialize() const + { + BinaryWriter W; + uint32_t Count = uint32_t(m_Offsets.size()); + W.Write(&Count, sizeof(Count)); + if (Count > 0) + { + W.Write(m_Offsets.data(), m_Offsets.size() * sizeof(uint32_t)); + } + if (!m_Packed.empty()) + { + W.Write(m_Packed.data(), m_Packed.size()); + } + return SharedBuffer(IoBuffer(IoBuffer::Clone, W.Data(), W.Size())); + } + +private: + struct StringKey + { + uint32_t Offset; + uint32_t Length; + }; + + struct StringHash + { + const eastl::vector<uint8_t>* Packed; + size_t operator()(const StringKey& K) const + { + std::string_view Sv(reinterpret_cast<const char*>(Packed->data()) + K.Offset, K.Length); + return std::hash<std::string_view>{}(Sv); + } + }; + + struct StringEq + { + const eastl::vector<uint8_t>* Packed; + bool operator()(const StringKey& A, const StringKey& B) const + { + if (A.Length != B.Length) + { + return false; + } + return memcmp(Packed->data() + A.Offset, Packed->data() + B.Offset, A.Length) == 0; + } + }; + + eastl::vector<uint8_t> m_Packed; // null-terminated strings back-to-back + eastl::vector<uint32_t> m_Offsets; // byte offset into m_Packed for each string + + // Dedup map: StringKey (offset+length into m_Packed) → string index. + // Hash/eq functors hold a pointer to m_Packed (stable address) and read + // via data() at call time, so reallocation of m_Packed is safe. + eastl::hash_map<StringKey, uint32_t, StringHash, StringEq> m_IndexMap; +}; + +// =========================================================================== +// StringTableReader — read-path helper for O(1) string lookup by index +// =========================================================================== + +class StringTableReader +{ +public: + bool Init(const SharedBuffer& Data) + { + if (Data.GetSize() < sizeof(uint32_t)) + { + return false; + } + + const uint8_t* Base = reinterpret_cast<const uint8_t*>(Data.GetData()); + memcpy(&m_Count, Base, sizeof(uint32_t)); + + size_t RequiredHeader = sizeof(uint32_t) + size_t(m_Count) * sizeof(uint32_t); + if (Data.GetSize() < RequiredHeader) + { + return false; + } + + m_Offsets = reinterpret_cast<const uint32_t*>(Base + sizeof(uint32_t)); + m_PackedBase = reinterpret_cast<const char*>(Base + RequiredHeader); + m_PackedSize = Data.GetSize() - RequiredHeader; + m_OwningBuffer = Data; + return true; + } + + std::string_view Get(uint32_t Index) const + { + if (Index >= m_Count) + { + return {}; + } + uint32_t Off = m_Offsets[Index]; + if (Off >= m_PackedSize) + { + return {}; + } + return std::string_view(m_PackedBase + Off); + } + + uint32_t Count() const { return m_Count; } + +private: + uint32_t m_Count = 0; + const uint32_t* m_Offsets = nullptr; + const char* m_PackedBase = nullptr; + size_t m_PackedSize = 0; + SharedBuffer m_OwningBuffer; // keeps the decompressed data alive +}; + +// =========================================================================== +// CachedSymbolResolver — SymbolResolver backed by cache data +// =========================================================================== + +class CachedSymbolResolver final : public SymbolResolver +{ +public: + void LoadModule(const ModuleInfo&) override {} + std::string Resolve(uint64_t Address) const override + { + auto It = m_Symbols.find(Address); + if (It != m_Symbols.end()) + { + return It->second; + } + return {}; + } + + eastl::hash_map<uint64_t, std::string> m_Symbols; +}; + +// =========================================================================== +// Section writers (model → binary blob) +// =========================================================================== + +namespace { + + template<typename T> + void WritePod(BinaryWriter& W, const T& Value) + { + W.Write(&Value, sizeof(T)); + } + + template<typename T> + void WriteCount(BinaryWriter& W, uint32_t Count) + { + W.Write(&Count, sizeof(Count)); + } + + SharedBuffer ToSharedBuffer(const BinaryWriter& W) { return SharedBuffer(IoBuffer(IoBuffer::Clone, W.Data(), W.Size())); } + + // -- Metadata section -- + + SharedBuffer WriteMetadataSection(const TraceModel& Model, StringTableBuilder& Strings) + { + BinaryWriter W; + + MetadataPod M = {}; + M.FileSize = Model.FileSize; + M.TotalEvents = Model.TotalEvents; + M.ParseTimeMs = Model.ParseTimeMs; + M.TraceStartUs = Model.TraceStartUs; + M.TraceEndUs = Model.TraceEndUs; + + M.SessionPlatform = Strings.Intern(Model.Session.Platform); + M.SessionAppName = Strings.Intern(Model.Session.AppName); + M.SessionProjectName = Strings.Intern(Model.Session.ProjectName); + M.SessionCommandLine = Strings.Intern(Model.Session.CommandLine); + M.SessionBranch = Strings.Intern(Model.Session.Branch); + M.SessionBuildVersion = Strings.Intern(Model.Session.BuildVersion); + M.SessionChangelist = Model.Session.Changelist; + M.SessionConfigType = Model.Session.ConfigurationType; + M.SessionHasSession = Model.Session.HasSession ? 1 : 0; + WritePod(W, M); + + // Threads + uint32_t ThreadCount = uint32_t(Model.Threads.size()); + WritePod(W, ThreadCount); + for (const ThreadInfoEntry& T : Model.Threads) + { + ThreadInfoPod P = {}; + P.ThreadId = T.ThreadId; + P.Name = Strings.Intern(T.Name); + P.GroupName = Strings.Intern(T.GroupName); + P.SystemId = T.SystemId; + P.SortHint = T.SortHint; + WritePod(W, P); + } + + // Channels + uint32_t ChannelCount = uint32_t(Model.Channels.size()); + WritePod(W, ChannelCount); + for (const ChannelInfo& C : Model.Channels) + { + ChannelInfoPod P = {}; + P.Name = Strings.Intern(C.Name); + P.Enabled = C.Enabled ? 1 : 0; + P.ReadOnly = C.ReadOnly ? 1 : 0; + WritePod(W, P); + } + + // Modules + uint32_t ModuleCount = uint32_t(Model.Modules.size()); + WritePod(W, ModuleCount); + + // First pass: compute ImageId blob layout + eastl::vector<uint32_t> ImageIdOffsets(ModuleCount); + uint32_t ImageIdBlobSize = 0; + for (uint32_t I = 0; I < ModuleCount; ++I) + { + ImageIdOffsets[I] = ImageIdBlobSize; + ImageIdBlobSize += uint32_t(Model.Modules[I].ImageId.size()); + } + + for (uint32_t I = 0; I < ModuleCount; ++I) + { + const ModuleInfo& Mod = Model.Modules[I]; + ModuleInfoPod P = {}; + P.Name = Strings.Intern(Mod.Name); + P.FullPath = Strings.Intern(Mod.FullPath); + P.Base = Mod.Base; + P.Size = Mod.Size; + P.ImageIdSize = uint32_t(Mod.ImageId.size()); + P.ImageIdOffset = ImageIdOffsets[I]; + WritePod(W, P); + } + + // ImageId blob + for (const ModuleInfo& Mod : Model.Modules) + { + if (!Mod.ImageId.empty()) + { + W.Write(Mod.ImageId.data(), Mod.ImageId.size()); + } + } + + // EventTypeCounts + uint32_t EventTypeCount = uint32_t(Model.EventTypeCounts.size()); + WritePod(W, EventTypeCount); + for (const TraceModel::EventTypeCount& E : Model.EventTypeCounts) + { + EventTypeCountPod P = {}; + P.Name = Strings.Intern(E.Name); + P.Count = E.Count; + WritePod(W, P); + } + + // ScopeStats + uint32_t ScopeStatCount = uint32_t(Model.ScopeStats.size()); + WritePod(W, ScopeStatCount); + for (const CpuScopeStat& S : Model.ScopeStats) + { + CpuScopeStatPod P = {}; + P.Name = Strings.Intern(S.Name); + P.MinUs = S.MinUs; + P.MaxUs = S.MaxUs; + P.Count = S.Count; + P.MeanUs = S.MeanUs; + P.StdDevUs = S.StdDevUs; + WritePod(W, P); + } + + return ToSharedBuffer(W); + } + + // -- Memory section -- + + SharedBuffer WriteMemorySection(const TraceModel& Model, StringTableBuilder& Strings) + { + BinaryWriter W; + + // AllocSummary + AllocSummaryPod A = {}; + A.HasMemoryData = Model.AllocSummary.HasMemoryData ? 1 : 0; + A.PeakTimeUs = Model.AllocSummary.PeakTimeUs; + A.LiveAllocations = Model.AllocSummary.LiveAllocations; + A.TotalAllocs = Model.AllocSummary.TotalAllocs; + A.TotalFrees = Model.AllocSummary.TotalFrees; + A.TotalReallocAllocs = Model.AllocSummary.TotalReallocAllocs; + A.TotalReallocFrees = Model.AllocSummary.TotalReallocFrees; + A.PeakBytes = Model.AllocSummary.PeakBytes; + A.EndBytes = Model.AllocSummary.EndBytes; + WritePod(W, A); + + // Heaps + uint32_t HeapCount = uint32_t(Model.Heaps.size()); + WritePod(W, HeapCount); + for (const HeapInfo& H : Model.Heaps) + { + HeapInfoPod P = {}; + P.Id = H.Id; + P.ParentId = H.ParentId; + P.Flags = H.Flags; + P.Name = Strings.Intern(H.Name); + WritePod(W, P); + } + + // HeapStats + uint32_t HeapStatCount = uint32_t(Model.HeapStats.size()); + WritePod(W, HeapStatCount); + for (const HeapStat& S : Model.HeapStats) + { + HeapStatPod P = {}; + P.HeapId = S.HeapId; + P.CurrentBytes = S.CurrentBytes; + P.PeakBytes = S.PeakBytes; + P.AllocCount = S.AllocCount; + P.FreeCount = S.FreeCount; + WritePod(W, P); + } + + // CallstackAllocStats + uint32_t AllocStatCount = uint32_t(Model.CallstackStats.size()); + WritePod(W, AllocStatCount); + for (const CallstackAllocStat& S : Model.CallstackStats) + { + CallstackAllocStatPod P = {}; + P.CallstackId = S.CallstackId; + P.LiveCount = S.LiveCount; + P.LiveBytes = S.LiveBytes; + P.ThreadIdCount = uint32_t(std::min(S.ThreadIds.size(), size_t(4))); + for (uint32_t I = 0; I < P.ThreadIdCount; ++I) + { + P.ThreadIds[I] = S.ThreadIds[I]; + } + WritePod(W, P); + } + + // ChurnStats + uint32_t ChurnCount = uint32_t(Model.ChurnStats.size()); + WritePod(W, ChurnCount); + for (const CallstackChurnStat& S : Model.ChurnStats) + { + CallstackChurnStatPod P = {}; + P.CallstackId = S.CallstackId; + P.ChurnAllocs = S.ChurnAllocs; + P.ChurnBytes = S.ChurnBytes; + P.TotalAllocs = S.TotalAllocs; + P.TotalBytes = S.TotalBytes; + P.MeanDistance = S.MeanDistance; + WritePod(W, P); + } + + return ToSharedBuffer(W); + } + + // -- Callstacks section -- + + SharedBuffer WriteCallstacksSection(const TraceModel& Model) + { + BinaryWriter W; + + uint32_t Count = uint32_t(Model.Callstacks.size()); + WritePod(W, Count); + + // Compute frame offsets + uint32_t FrameOffset = 0; + for (const CallstackEntry& CS : Model.Callstacks) + { + CallstackHeaderPod H = {}; + H.Id = CS.Id; + H.FrameCount = uint32_t(CS.Frames.size()); + H.FrameOffset = FrameOffset; + WritePod(W, H); + FrameOffset += H.FrameCount; + } + + // Write all frames + for (const CallstackEntry& CS : Model.Callstacks) + { + for (const ResolvedFrame& F : CS.Frames) + { + ResolvedFramePod P = {}; + P.Address = F.Address; + P.ModuleIndex = F.ModuleIndex; + P.Offset = F.Offset; + WritePod(W, P); + } + } + + return ToSharedBuffer(W); + } + + // -- Symbols section -- + + SharedBuffer WriteSymbolsSection(const eastl::hash_map<uint64_t, std::string>& ResolvedSymbols, StringTableBuilder& Strings) + { + BinaryWriter W; + + // Collect and sort entries by address for binary search on read + eastl::vector<SymbolEntryPod> Entries; + Entries.reserve(ResolvedSymbols.size()); + for (const auto& [Address, SymbolStr] : ResolvedSymbols) + { + SymbolEntryPod E = {}; + E.Address = Address; + E.StringIdx = Strings.Intern(SymbolStr); + Entries.push_back(E); + } + eastl::sort(Entries.begin(), Entries.end(), [](const SymbolEntryPod& A, const SymbolEntryPod& B) { return A.Address < B.Address; }); + + uint32_t Count = uint32_t(Entries.size()); + WritePod(W, Count); + if (!Entries.empty()) + { + W.Write(Entries.data(), Entries.size() * sizeof(SymbolEntryPod)); + } + + return ToSharedBuffer(W); + } + + // -- Compression helper -- + + CompressedBuffer CompressSection(const SharedBuffer& Raw) + { + return CompressedBuffer::Compress(Raw, OodleCompressor::Mermaid, OodleCompressionLevel::VeryFast); + } + + // =========================================================================== + // Section readers (binary blob → model) + // =========================================================================== + + template<typename T> + bool ReadPod(BinaryReader& R, T& Out) + { + if (R.Remaining() < sizeof(T)) + { + return false; + } + R.Read(&Out, sizeof(T)); + return true; + } + + bool ReadUint32(BinaryReader& R, uint32_t& Out) { return ReadPod(R, Out); } + + bool ReadMetadataSection(const SharedBuffer& Data, const StringTableReader& Strings, TraceModel& Model) + { + BinaryReader R(Data.GetData(), Data.GetSize()); + + MetadataPod M; + if (!ReadPod(R, M)) + { + return false; + } + Model.FileSize = M.FileSize; + Model.TotalEvents = M.TotalEvents; + Model.ParseTimeMs = M.ParseTimeMs; + Model.TraceStartUs = M.TraceStartUs; + Model.TraceEndUs = M.TraceEndUs; + + Model.Session.Platform = std::string(Strings.Get(M.SessionPlatform)); + Model.Session.AppName = std::string(Strings.Get(M.SessionAppName)); + Model.Session.ProjectName = std::string(Strings.Get(M.SessionProjectName)); + Model.Session.CommandLine = std::string(Strings.Get(M.SessionCommandLine)); + Model.Session.Branch = std::string(Strings.Get(M.SessionBranch)); + Model.Session.BuildVersion = std::string(Strings.Get(M.SessionBuildVersion)); + Model.Session.Changelist = M.SessionChangelist; + Model.Session.ConfigurationType = M.SessionConfigType; + Model.Session.HasSession = (M.SessionHasSession != 0); + + // Threads + uint32_t ThreadCount = 0; + if (!ReadUint32(R, ThreadCount)) + { + return false; + } + Model.Threads.resize(ThreadCount); + for (uint32_t I = 0; I < ThreadCount; ++I) + { + ThreadInfoPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.Threads[I].ThreadId = P.ThreadId; + Model.Threads[I].Name = std::string(Strings.Get(P.Name)); + Model.Threads[I].GroupName = std::string(Strings.Get(P.GroupName)); + Model.Threads[I].SystemId = P.SystemId; + Model.Threads[I].SortHint = P.SortHint; + } + + // Channels + uint32_t ChannelCount = 0; + if (!ReadUint32(R, ChannelCount)) + { + return false; + } + Model.Channels.resize(ChannelCount); + for (uint32_t I = 0; I < ChannelCount; ++I) + { + ChannelInfoPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.Channels[I].Name = std::string(Strings.Get(P.Name)); + Model.Channels[I].Enabled = (P.Enabled != 0); + Model.Channels[I].ReadOnly = (P.ReadOnly != 0); + } + + // Modules + uint32_t ModuleCount = 0; + if (!ReadUint32(R, ModuleCount)) + { + return false; + } + + // Read ModuleInfoPod entries first, then the ImageId blob + eastl::vector<ModuleInfoPod> ModulePods(ModuleCount); + for (uint32_t I = 0; I < ModuleCount; ++I) + { + if (!ReadPod(R, ModulePods[I])) + { + return false; + } + } + + // Compute total ImageId blob size + uint32_t TotalImageIdSize = 0; + for (const ModuleInfoPod& MP : ModulePods) + { + uint32_t End = MP.ImageIdOffset + MP.ImageIdSize; + if (End > TotalImageIdSize) + { + TotalImageIdSize = End; + } + } + + const uint8_t* ImageIdBlobBase = nullptr; + if (TotalImageIdSize > 0) + { + if (R.Remaining() < TotalImageIdSize) + { + return false; + } + ImageIdBlobBase = reinterpret_cast<const uint8_t*>(R.GetView(TotalImageIdSize).GetData()); + R.Skip(TotalImageIdSize); + } + + Model.Modules.resize(ModuleCount); + for (uint32_t I = 0; I < ModuleCount; ++I) + { + const ModuleInfoPod& MP = ModulePods[I]; + ModuleInfo& Mod = Model.Modules[I]; + Mod.Name = std::string(Strings.Get(MP.Name)); + Mod.FullPath = std::string(Strings.Get(MP.FullPath)); + Mod.Base = MP.Base; + Mod.Size = MP.Size; + if (MP.ImageIdSize > 0 && ImageIdBlobBase != nullptr) + { + Mod.ImageId.assign(ImageIdBlobBase + MP.ImageIdOffset, ImageIdBlobBase + MP.ImageIdOffset + MP.ImageIdSize); + } + } + + // EventTypeCounts + uint32_t EventTypeCount = 0; + if (!ReadUint32(R, EventTypeCount)) + { + return false; + } + Model.EventTypeCounts.resize(EventTypeCount); + for (uint32_t I = 0; I < EventTypeCount; ++I) + { + EventTypeCountPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.EventTypeCounts[I].Name = std::string(Strings.Get(P.Name)); + Model.EventTypeCounts[I].Count = P.Count; + } + + // ScopeStats + uint32_t ScopeStatCount = 0; + if (!ReadUint32(R, ScopeStatCount)) + { + return false; + } + Model.ScopeStats.resize(ScopeStatCount); + for (uint32_t I = 0; I < ScopeStatCount; ++I) + { + CpuScopeStatPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.ScopeStats[I].Name = std::string(Strings.Get(P.Name)); + Model.ScopeStats[I].MinUs = P.MinUs; + Model.ScopeStats[I].MaxUs = P.MaxUs; + Model.ScopeStats[I].Count = P.Count; + Model.ScopeStats[I].MeanUs = P.MeanUs; + Model.ScopeStats[I].StdDevUs = P.StdDevUs; + } + + return true; + } + + bool ReadMemorySection(const SharedBuffer& Data, const StringTableReader& Strings, TraceModel& Model) + { + BinaryReader R(Data.GetData(), Data.GetSize()); + + // AllocSummary + AllocSummaryPod A; + if (!ReadPod(R, A)) + { + return false; + } + Model.AllocSummary.HasMemoryData = (A.HasMemoryData != 0); + Model.AllocSummary.PeakTimeUs = A.PeakTimeUs; + Model.AllocSummary.LiveAllocations = A.LiveAllocations; + Model.AllocSummary.TotalAllocs = A.TotalAllocs; + Model.AllocSummary.TotalFrees = A.TotalFrees; + Model.AllocSummary.TotalReallocAllocs = A.TotalReallocAllocs; + Model.AllocSummary.TotalReallocFrees = A.TotalReallocFrees; + Model.AllocSummary.PeakBytes = A.PeakBytes; + Model.AllocSummary.EndBytes = A.EndBytes; + + // Heaps + uint32_t HeapCount = 0; + if (!ReadUint32(R, HeapCount)) + { + return false; + } + Model.Heaps.resize(HeapCount); + for (uint32_t I = 0; I < HeapCount; ++I) + { + HeapInfoPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.Heaps[I].Id = P.Id; + Model.Heaps[I].ParentId = P.ParentId; + Model.Heaps[I].Flags = P.Flags; + Model.Heaps[I].Name = std::string(Strings.Get(P.Name)); + } + + // HeapStats + uint32_t HeapStatCount = 0; + if (!ReadUint32(R, HeapStatCount)) + { + return false; + } + Model.HeapStats.resize(HeapStatCount); + for (uint32_t I = 0; I < HeapStatCount; ++I) + { + HeapStatPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.HeapStats[I].HeapId = P.HeapId; + Model.HeapStats[I].CurrentBytes = P.CurrentBytes; + Model.HeapStats[I].PeakBytes = P.PeakBytes; + Model.HeapStats[I].AllocCount = P.AllocCount; + Model.HeapStats[I].FreeCount = P.FreeCount; + } + + // CallstackAllocStats + uint32_t AllocStatCount = 0; + if (!ReadUint32(R, AllocStatCount)) + { + return false; + } + Model.CallstackStats.resize(AllocStatCount); + for (uint32_t I = 0; I < AllocStatCount; ++I) + { + CallstackAllocStatPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.CallstackStats[I].CallstackId = P.CallstackId; + Model.CallstackStats[I].LiveCount = P.LiveCount; + Model.CallstackStats[I].LiveBytes = P.LiveBytes; + for (uint32_t J = 0; J < P.ThreadIdCount && J < 4; ++J) + { + Model.CallstackStats[I].ThreadIds.push_back(P.ThreadIds[J]); + } + } + + // ChurnStats + uint32_t ChurnCount = 0; + if (!ReadUint32(R, ChurnCount)) + { + return false; + } + Model.ChurnStats.resize(ChurnCount); + for (uint32_t I = 0; I < ChurnCount; ++I) + { + CallstackChurnStatPod P; + if (!ReadPod(R, P)) + { + return false; + } + Model.ChurnStats[I].CallstackId = P.CallstackId; + Model.ChurnStats[I].ChurnAllocs = P.ChurnAllocs; + Model.ChurnStats[I].ChurnBytes = P.ChurnBytes; + Model.ChurnStats[I].TotalAllocs = P.TotalAllocs; + Model.ChurnStats[I].TotalBytes = P.TotalBytes; + Model.ChurnStats[I].MeanDistance = P.MeanDistance; + } + + return true; + } + + bool ReadCallstacksSection(const SharedBuffer& Data, TraceModel& Model) + { + BinaryReader R(Data.GetData(), Data.GetSize()); + + uint32_t Count = 0; + if (!ReadUint32(R, Count)) + { + return false; + } + + // Read headers + eastl::vector<CallstackHeaderPod> Headers(Count); + for (uint32_t I = 0; I < Count; ++I) + { + if (!ReadPod(R, Headers[I])) + { + return false; + } + } + + // Compute total frame count + uint32_t TotalFrames = 0; + for (const CallstackHeaderPod& H : Headers) + { + TotalFrames = std::max(TotalFrames, H.FrameOffset + H.FrameCount); + } + + if (R.Remaining() < TotalFrames * sizeof(ResolvedFramePod)) + { + return false; + } + + // Read all frames + eastl::vector<ResolvedFramePod> AllFrames(TotalFrames); + for (uint32_t I = 0; I < TotalFrames; ++I) + { + if (!ReadPod(R, AllFrames[I])) + { + return false; + } + } + + // Build CallstackEntry vector + Model.Callstacks.resize(Count); + for (uint32_t I = 0; I < Count; ++I) + { + const CallstackHeaderPod& H = Headers[I]; + CallstackEntry& CS = Model.Callstacks[I]; + CS.Id = H.Id; + CS.Frames.resize(H.FrameCount); + for (uint32_t J = 0; J < H.FrameCount; ++J) + { + const ResolvedFramePod& FP = AllFrames[H.FrameOffset + J]; + CS.Frames[J].Address = FP.Address; + CS.Frames[J].ModuleIndex = FP.ModuleIndex; + CS.Frames[J].Offset = FP.Offset; + } + } + + return true; + } + + bool ReadSymbolsSection(const SharedBuffer& Data, const StringTableReader& Strings, CachedSymbolResolver& Resolver) + { + BinaryReader R(Data.GetData(), Data.GetSize()); + + uint32_t Count = 0; + if (!ReadUint32(R, Count)) + { + return false; + } + + for (uint32_t I = 0; I < Count; ++I) + { + SymbolEntryPod E; + if (!ReadPod(R, E)) + { + return false; + } + std::string_view Str = Strings.Get(E.StringIdx); + if (!Str.empty()) + { + Resolver.m_Symbols.emplace(E.Address, std::string(Str)); + } + } + + return true; + } + + // =========================================================================== + // File-level helpers + // =========================================================================== + + int64_t GetFileModTimeNs(const std::filesystem::path& Path) + { + std::error_code Ec; + auto ModTime = std::filesystem::last_write_time(Path, Ec); + if (Ec) + { + return 0; + } + auto Duration = ModTime.time_since_epoch(); + return std::chrono::duration_cast<std::chrono::nanoseconds>(Duration).count(); + } + + SharedBuffer DecompressSection(const uint8_t* FileBase, const SectionDirectoryEntry& Dir) + { + IoBuffer CompressedIo(IoBuffer::Wrap, FileBase + Dir.FileOffset, Dir.CompressedSize); + + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CB = CompressedBuffer::FromCompressed(SharedBuffer(std::move(CompressedIo)), RawHash, RawSize); + if (CB.IsNull()) + { + return {}; + } + return CB.Decompress(); + } + +} // namespace + +// =========================================================================== +// Public API +// =========================================================================== + +void +WriteAnalyzeCache(const std::filesystem::path& CachePath, + const std::filesystem::path& SourcePath, + const TraceModel& Model, + const eastl::hash_map<uint64_t, std::string>& ResolvedSymbols) +{ + try + { + StringTableBuilder Strings; + + // Build section payloads (order matters: Symbols and Metadata/Memory + // intern strings, so StringTable must be serialized LAST after all + // interning is done). + SharedBuffer MetadataRaw = WriteMetadataSection(Model, Strings); + SharedBuffer MemoryRaw = WriteMemorySection(Model, Strings); + SharedBuffer CallstacksRaw = WriteCallstacksSection(Model); + SharedBuffer SymbolsRaw = WriteSymbolsSection(ResolvedSymbols, Strings); + SharedBuffer StringTableRaw = Strings.Serialize(); + + // Compress each section + CompressedBuffer Sections[uint32_t(CacheSectionId::Count)]; + Sections[uint32_t(CacheSectionId::StringTable)] = CompressSection(StringTableRaw); + Sections[uint32_t(CacheSectionId::Metadata)] = CompressSection(MetadataRaw); + Sections[uint32_t(CacheSectionId::Memory)] = CompressSection(MemoryRaw); + Sections[uint32_t(CacheSectionId::Callstacks)] = CompressSection(CallstacksRaw); + Sections[uint32_t(CacheSectionId::Symbols)] = CompressSection(SymbolsRaw); + + // Build file header + CacheFileHeader Header = {}; + Header.Magic = kCacheMagic; + Header.Version = kCacheVersion; + + std::error_code Ec; + Header.SourceFileSize = std::filesystem::file_size(SourcePath, Ec); + Header.SourceModTimeNs = GetFileModTimeNs(SourcePath); + + uint32_t SectionCount = uint32_t(CacheSectionId::Count); + + // Compute section directory + uint64_t DataOffset = sizeof(CacheFileHeader) + SectionCount * sizeof(SectionDirectoryEntry); + + SectionDirectoryEntry Directory[uint32_t(CacheSectionId::Count)]; + for (uint32_t I = 0; I < SectionCount; ++I) + { + Directory[I].SectionId = I; + Directory[I].Reserved = 0; + Directory[I].FileOffset = DataOffset; + Directory[I].CompressedSize = Sections[I].GetCompressedSize(); + DataOffset += Directory[I].CompressedSize; + } + + // Assemble and write the file + BinaryWriter FileWriter; + FileWriter.Write(&Header, sizeof(Header)); + FileWriter.Write(Directory, sizeof(Directory)); + + // Append compressed blobs + for (uint32_t I = 0; I < SectionCount; ++I) + { + SharedBuffer Flat = std::move(Sections[I]).GetCompressed().Flatten(); + FileWriter.Write(Flat.GetData(), Flat.GetSize()); + } + + zen::TemporaryFile::SafeWriteFile(CachePath, FileWriter.GetView()); + + ZEN_INFO("Wrote analysis cache {} ({})", CachePath.filename().string(), zen::NiceBytes(FileWriter.Size())); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed to write analysis cache: {}", Ex.what()); + } +} + +std::optional<CachedAnalysis> +TryLoadAnalyzeCache(const std::filesystem::path& CachePath, const std::filesystem::path& SourcePath) +{ + std::error_code Ec; + if (!std::filesystem::exists(CachePath, Ec)) + { + return std::nullopt; + } + + try + { + FileContents Contents = zen::ReadFile(CachePath); + if (!Contents) + { + return std::nullopt; + } + + IoBuffer FileData = Contents.Flatten(); + if (FileData.Size() < sizeof(CacheFileHeader)) + { + return std::nullopt; + } + + const uint8_t* Base = reinterpret_cast<const uint8_t*>(FileData.Data()); + + // Validate header + CacheFileHeader Header; + memcpy(&Header, Base, sizeof(Header)); + + if (Header.Magic != kCacheMagic) + { + ZEN_DEBUG("Analysis cache: bad magic"); + return std::nullopt; + } + + if (Header.Version != kCacheVersion) + { + ZEN_DEBUG("Analysis cache: version mismatch ({} vs {})", Header.Version, kCacheVersion); + return std::nullopt; + } + + // Validate source file hasn't changed + uint64_t CurrentSize = std::filesystem::file_size(SourcePath, Ec); + int64_t CurrentModTime = GetFileModTimeNs(SourcePath); + + if (Header.SourceFileSize != CurrentSize || Header.SourceModTimeNs != CurrentModTime) + { + ZEN_DEBUG("Analysis cache: source file changed, invalidating"); + return std::nullopt; + } + + // Parse section directory + uint32_t SectionCount = uint32_t(CacheSectionId::Count); + size_t DirSize = SectionCount * sizeof(SectionDirectoryEntry); + if (FileData.Size() < sizeof(CacheFileHeader) + DirSize) + { + return std::nullopt; + } + + SectionDirectoryEntry Directory[uint32_t(CacheSectionId::Count)]; + memcpy(Directory, Base + sizeof(CacheFileHeader), DirSize); + + // Validate all sections fit in the file + for (uint32_t I = 0; I < SectionCount; ++I) + { + if (Directory[I].FileOffset + Directory[I].CompressedSize > FileData.Size()) + { + ZEN_DEBUG("Analysis cache: section {} truncated", I); + return std::nullopt; + } + } + + // Decompress string table first + SharedBuffer StringTableData = DecompressSection(Base, Directory[uint32_t(CacheSectionId::StringTable)]); + if (StringTableData.IsNull()) + { + ZEN_DEBUG("Analysis cache: failed to decompress string table"); + return std::nullopt; + } + + StringTableReader Strings; + if (!Strings.Init(StringTableData)) + { + ZEN_DEBUG("Analysis cache: invalid string table"); + return std::nullopt; + } + + CachedAnalysis Result; + Result.Model.FilePath = SourcePath; + + // Decompress and read each section + SharedBuffer MetaData = DecompressSection(Base, Directory[uint32_t(CacheSectionId::Metadata)]); + if (MetaData.IsNull() || !ReadMetadataSection(MetaData, Strings, Result.Model)) + { + ZEN_DEBUG("Analysis cache: failed to read metadata section"); + return std::nullopt; + } + + SharedBuffer MemData = DecompressSection(Base, Directory[uint32_t(CacheSectionId::Memory)]); + if (MemData.IsNull() || !ReadMemorySection(MemData, Strings, Result.Model)) + { + ZEN_DEBUG("Analysis cache: failed to read memory section"); + return std::nullopt; + } + + SharedBuffer CsData = DecompressSection(Base, Directory[uint32_t(CacheSectionId::Callstacks)]); + if (CsData.IsNull() || !ReadCallstacksSection(CsData, Result.Model)) + { + ZEN_DEBUG("Analysis cache: failed to read callstacks section"); + return std::nullopt; + } + + SharedBuffer SymData = DecompressSection(Base, Directory[uint32_t(CacheSectionId::Symbols)]); + if (!SymData.IsNull()) + { + auto Resolver = std::make_unique<CachedSymbolResolver>(); + if (ReadSymbolsSection(SymData, Strings, *Resolver)) + { + Result.Symbols = std::move(Resolver); + } + } + + ZEN_INFO("Loaded analysis from cache ({})", zen::NiceBytes(FileData.Size())); + return Result; + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("Analysis cache load failed: {}", Ex.what()); + return std::nullopt; + } +} + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_cache.h b/src/zen/trace/trace_cache.h new file mode 100644 index 000000000..88778a020 --- /dev/null +++ b/src/zen/trace/trace_cache.h @@ -0,0 +1,253 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "symbol_resolver.h" +#include "trace_model.h" + +#include <zencore/sharedbuffer.h> + +#include <cstdint> +#include <filesystem> +#include <memory> +#include <optional> +#include <string_view> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/hash_map.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::trace_detail { + +// --------------------------------------------------------------------------- +// File format constants +// --------------------------------------------------------------------------- + +static constexpr uint32_t kCacheMagic = 0x005A4355; // "UCZ\0" +static constexpr uint32_t kCacheVersion = 1; + +enum class CacheSectionId : uint32_t +{ + StringTable = 0, + Metadata = 1, + Memory = 2, + Callstacks = 3, + Symbols = 4, + Count +}; + +// --------------------------------------------------------------------------- +// On-disk header structures (naturally aligned, no packing) +// +// Fields are ordered so natural alignment matches across compilers without +// needing pragma pack. static_asserts at the bottom of this block pin the +// layout so a reordering or added field cannot silently break cached files. +// --------------------------------------------------------------------------- + +struct CacheFileHeader +{ + uint32_t Magic; + uint32_t Version; + uint64_t SourceFileSize; + int64_t SourceModTimeNs; // last_write_time as nanoseconds since epoch + uint64_t Reserved; +}; + +struct SectionDirectoryEntry +{ + uint32_t SectionId; + uint32_t Reserved; + uint64_t FileOffset; // byte offset from start of file + uint64_t CompressedSize; // size of the CompressedBuffer blob on disk +}; + +// --------------------------------------------------------------------------- +// POD types for memory-mappable section content +// --------------------------------------------------------------------------- + +struct MetadataPod +{ + uint64_t FileSize; + uint64_t TotalEvents; + uint64_t ParseTimeMs; + uint32_t TraceStartUs; + uint32_t TraceEndUs; + // SessionInfo string indices + uint32_t SessionPlatform; + uint32_t SessionAppName; + uint32_t SessionProjectName; + uint32_t SessionCommandLine; + uint32_t SessionBranch; + uint32_t SessionBuildVersion; + uint32_t SessionChangelist; + uint8_t SessionConfigType; + uint8_t SessionHasSession; + uint8_t Padding[2]; +}; + +struct ThreadInfoPod +{ + uint32_t ThreadId; + uint32_t Name; // string index + uint32_t GroupName; // string index + uint32_t SystemId; + int32_t SortHint; + uint32_t Pad; +}; + +struct ChannelInfoPod +{ + uint32_t Name; // string index + uint8_t Enabled; + uint8_t ReadOnly; + uint8_t Pad[2]; +}; + +struct ModuleInfoPod +{ + uint32_t Name; // string index + uint32_t FullPath; // string index + uint64_t Base; + uint32_t Size; + uint32_t ImageIdSize; // byte count in the ImageId blob area + uint32_t ImageIdOffset; // byte offset into the ImageId blob area + uint32_t Pad; +}; + +struct EventTypeCountPod +{ + uint32_t Name; // string index + uint32_t Pad; + uint64_t Count; +}; + +struct CpuScopeStatPod +{ + uint32_t Name; // string index + uint32_t MinUs; + uint32_t MaxUs; + uint32_t Pad; + uint64_t Count; + double MeanUs; + double StdDevUs; +}; + +struct AllocSummaryPod +{ + uint8_t HasMemoryData; + uint8_t Pad0[3]; + uint32_t PeakTimeUs; + uint32_t LiveAllocations; + uint32_t Pad1; + uint64_t TotalAllocs; + uint64_t TotalFrees; + uint64_t TotalReallocAllocs; + uint64_t TotalReallocFrees; + int64_t PeakBytes; + int64_t EndBytes; +}; + +struct HeapInfoPod +{ + uint32_t Id; + uint32_t ParentId; + uint16_t Flags; + uint16_t Pad0; + uint32_t Name; // string index +}; + +struct HeapStatPod +{ + uint32_t HeapId; + uint32_t Pad; + int64_t CurrentBytes; + int64_t PeakBytes; + uint64_t AllocCount; + uint64_t FreeCount; +}; + +struct CallstackAllocStatPod +{ + uint32_t CallstackId; + uint32_t LiveCount; + int64_t LiveBytes; + uint32_t ThreadIdCount; + uint32_t ThreadIds[4]; + uint32_t Pad; + uint32_t Pad2; +}; + +struct CallstackChurnStatPod +{ + uint32_t CallstackId; + uint32_t Pad; + uint64_t ChurnAllocs; + uint64_t ChurnBytes; + uint64_t TotalAllocs; + uint64_t TotalBytes; + double MeanDistance; +}; + +struct CallstackHeaderPod +{ + uint32_t Id; + uint32_t FrameCount; + uint32_t FrameOffset; // index into the frames array + uint32_t Pad; +}; + +struct ResolvedFramePod +{ + uint64_t Address; + uint32_t ModuleIndex; + uint32_t Pad; + uint64_t Offset; +}; + +struct SymbolEntryPod +{ + uint64_t Address; + uint32_t StringIdx; // index into the string table + uint32_t Pad; +}; + +// Pin the on-disk layout. Any change here is a cache format change and must +// bump kCacheVersion. +static_assert(sizeof(CacheFileHeader) == 32); +static_assert(sizeof(SectionDirectoryEntry) == 24); +static_assert(sizeof(MetadataPod) == 64); +static_assert(sizeof(ThreadInfoPod) == 24); +static_assert(sizeof(ChannelInfoPod) == 8); +static_assert(sizeof(ModuleInfoPod) == 32); +static_assert(sizeof(EventTypeCountPod) == 16); +static_assert(sizeof(CpuScopeStatPod) == 40); +static_assert(sizeof(AllocSummaryPod) == 64); +static_assert(sizeof(HeapInfoPod) == 16); +static_assert(sizeof(HeapStatPod) == 40); +static_assert(sizeof(CallstackAllocStatPod) == 48); +static_assert(sizeof(CallstackChurnStatPod) == 48); +static_assert(sizeof(CallstackHeaderPod) == 16); +static_assert(sizeof(ResolvedFramePod) == 24); +static_assert(sizeof(SymbolEntryPod) == 16); + +// --------------------------------------------------------------------------- +// Cache read / write API +// --------------------------------------------------------------------------- + +struct CachedAnalysis +{ + TraceModel Model; + std::unique_ptr<SymbolResolver> Symbols; +}; + +// Try to load a cached analysis from the .ucache_z file next to a .utrace. +// Returns nullopt on any failure (missing, stale, corrupt, version mismatch). +std::optional<CachedAnalysis> TryLoadAnalyzeCache(const std::filesystem::path& CachePath, const std::filesystem::path& SourcePath); + +// Write the analysis cache for future reuse. +void WriteAnalyzeCache(const std::filesystem::path& CachePath, + const std::filesystem::path& SourcePath, + const TraceModel& Model, + const eastl::hash_map<uint64_t, std::string>& ResolvedSymbols); + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_cmd.cpp b/src/zen/trace/trace_cmd.cpp new file mode 100644 index 000000000..35316721e --- /dev/null +++ b/src/zen/trace/trace_cmd.cpp @@ -0,0 +1,402 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_cmd.h" + +#include "browser_launcher.h" +#include "consoleprogress.h" +#include "symbol_resolver.h" +#include "trace_analyze.h" +#include "trace_model.h" +#include "trace_viewer_service.h" +#include "zenserviceclient.h" + +#include <zencore/except_fmt.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <zencore/workthreadpool.h> +#include <zenhttp/httpclient.h> +#include <zenhttp/httpcommon.h> +#include <zenhttp/httpserver.h> + +#include <filesystem> +#include <numeric> + +using namespace std::literals; + +namespace zen { + +namespace { + +#if ZEN_PLATFORM_WINDOWS + constexpr const char* kSymbolBackendHelp = "Symbol backend: auto (default), pdb, dbghelp, llvm, off"; +#elif ZEN_PLATFORM_MAC + constexpr const char* kSymbolBackendHelp = "Symbol backend: auto (default - prefers llvm, falls back to atos), llvm, atos, off"; +#else + constexpr const char* kSymbolBackendHelp = "Symbol backend: auto (default - uses llvm), llvm, off"; +#endif + +} // namespace + +////////////////////////////////////////////////////////////////////////// +// TraceAnalyzeSubCmd + +TraceAnalyzeSubCmd::TraceAnalyzeSubCmd() : ZenSubCmdBase("analyze", "Analyze a .utrace file") +{ + SubOptions().add_option("", "", "file", "Path to .utrace file", cxxopts::value(m_TraceFilePath), "<filepath>"); + SubOptions().add_option("", + "", + "live-allocs", + "Dump top N live-allocation callstacks (0 = off, default 50)", + cxxopts::value(m_LiveAllocs)->default_value("50"), + "<count>"); + SubOptions().add_option("", + "", + "churn", + "Dump top N allocation-churn callstacks (0 = off, default 0)", + cxxopts::value(m_Churn)->default_value("0"), + "<count>"); + SubOptions().add_option("", + "", + "churn-distance", + "Max event distance between alloc and free to count as churn (default 1000)", + cxxopts::value(m_ChurnMin)->default_value("1000"), + "<events>"); + SubOptions().add_option("", "", "symbols", kSymbolBackendHelp, cxxopts::value(m_Symbols)->default_value("auto"), "<backend>"); + SubOptions().add_option("", + "", + "html-report", + "Write a standalone HTML memory report (all live leaks + top 100 churn sites)", + cxxopts::value(m_HtmlReportPath), + "<filepath>"); + SubOptions().add_option("", + "", + "callstack-skip", + "Semicolon-separated wildcard patterns for frames to hide from analyzed callstacks", + cxxopts::value(m_CallstackSkip), + "<pattern;...>"); + SubOptions().add_option("", + "", + "no-callstack-heuristic", + "Disable leading third-party frame trimming in analyzed callstacks", + cxxopts::value(m_NoCallstackHeuristic)->default_value("false"), + "<no-callstack-heuristic>"); + SubOptions().add_option("", + "", + "no-cache", + "Skip reading/writing the .ucache_z analysis cache", + cxxopts::value(m_NoCache)->default_value("false"), + "<no-cache>"); + SubOptions().parse_positional({"file"}); + SubOptions().positional_help("<file.utrace>"); +} + +void +TraceAnalyzeSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + std::filesystem::path FilePath = trace_detail::ResolveTraceFile(m_TraceFilePath, SubOptions()); + + trace_detail::AnalyzeOptions Options; + Options.LiveAllocsLimit = m_LiveAllocs; + Options.ChurnLimit = m_Churn; + Options.ChurnDistanceThreshold = uint64_t(m_ChurnMin); + Options.Symbols = trace_detail::ParseSymbolBackend(m_Symbols); + Options.NoCache = m_NoCache; + Options.EnableCallstackHeuristic = !m_NoCallstackHeuristic; + Options.HtmlReportPath = m_HtmlReportPath; + ForEachStrTok(m_CallstackSkip, ';', [&Options](std::string_view Pattern) { + if (!Pattern.empty()) + { + Options.CallstackSkipPatterns.emplace_back(Pattern); + } + return true; + }); + trace_detail::RunAnalyze(FilePath, Options); +} + +////////////////////////////////////////////////////////////////////////// +// TraceInspectSubCmd + +TraceInspectSubCmd::TraceInspectSubCmd() : ZenSubCmdBase("inspect", "Inspect event schemas in a .utrace file") +{ + SubOptions().add_option("", "", "file", "Path to .utrace file", cxxopts::value(m_TraceFilePath), "<filepath>"); + SubOptions().parse_positional({"file"}); + SubOptions().positional_help("<file.utrace>"); +} + +void +TraceInspectSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + std::filesystem::path FilePath = trace_detail::ResolveTraceFile(m_TraceFilePath, SubOptions()); + trace_detail::RunInspect(FilePath); +} + +////////////////////////////////////////////////////////////////////////// +// TraceServeSubCmd + +TraceServeSubCmd::TraceServeSubCmd() : ZenSubCmdBase("serve", "Serve an interactive viewer for a .utrace file") +{ + AddAlias("view"); + SubOptions().add_option("", "", "file", "Path to .utrace file", cxxopts::value(m_TraceFilePath), "<filepath>"); + SubOptions().add_option("", "p", "port", "Port to listen on", cxxopts::value(m_Port)->default_value("1480"), "<port>"); + SubOptions().add_option("", "", "bind", "Address to bind to", cxxopts::value(m_Bind)->default_value("127.0.0.1"), "<host>"); + SubOptions().add_option("", "", "symbols", kSymbolBackendHelp, cxxopts::value(m_Symbols)->default_value("auto"), "<backend>"); + SubOptions().add_option("", + "", + "no-browser", + "Do not launch a web browser after starting the server", + cxxopts::value(m_NoBrowser)->default_value("false"), + "<no-browser>"); + SubOptions().parse_positional({"file"}); + SubOptions().positional_help("<file.utrace>"); +} + +void +TraceServeSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + std::filesystem::path FilePath = trace_detail::ResolveTraceFile(m_TraceFilePath, SubOptions()); + + WorkerThreadPool ThreadPool(gsl::narrow<int>(GetHardwareConcurrency())); + + uint64_t FileSize = uint64_t(std::filesystem::file_size(FilePath)); + ZEN_CONSOLE("Parsing {} ({})", FilePath.filename().string(), zen::NiceBytes(FileSize)); + + std::unique_ptr<ProgressBase> ProgressOwner(CreateConsoleProgress(ConsoleProgressMode::Pretty)); + std::unique_ptr<ProgressBase::ProgressBar> Progress = ProgressOwner->CreateProgressBar("Parse"); + trace_detail::TraceModel Model = + trace_detail::BuildTraceModel(FilePath, ThreadPool, [&](uint64_t BytesProcessed, uint64_t TotalBytes, uint64_t EventsSoFar) { + Progress->UpdateState( + { + .Task = "Parsing trace", + .Details = fmt::format("{} events", zen::ThousandsNum(EventsSoFar)), + .TotalCount = TotalBytes, + .RemainingCount = TotalBytes - std::min(BytesProcessed, TotalBytes), + }, + false); + }); + Progress->Finish(); + + ZEN_CONSOLE(" Events: {}", zen::ThousandsNum(Model.TotalEvents)); + ZEN_CONSOLE(" Threads: {}", Model.Threads.size()); + ZEN_CONSOLE( + " Scopes: {}", + zen::ThousandsNum(std::accumulate(Model.Timelines.begin(), + Model.Timelines.end(), + size_t(0), + [](size_t Acc, const trace_detail::ThreadTimeline& T) { return Acc + T.Scopes.size(); }))); + ZEN_CONSOLE(" Time: {}", zen::NiceTimeSpanMs(Model.ParseTimeMs)); + + std::unique_ptr<trace_detail::SymbolResolver> Symbols = trace_detail::CreateSymbolResolver(trace_detail::ParseSymbolBackend(m_Symbols)); + for (const trace_detail::ModuleInfo& Mod : Model.Modules) + { + Symbols->LoadModule(Mod); + } + ZEN_CONSOLE(" Symbols: {} modules loaded", Model.Modules.size()); + ZEN_CONSOLE(""); + + HttpServerConfig Config; + Config.ServerClass = "asio"; + Config.IsDedicatedServer = false; + Config.AllowPortProbing = true; + Config.ForceLoopback = (m_Bind == "127.0.0.1" || m_Bind == "localhost" || m_Bind == "::1"); + + Ref<HttpServer> Server = CreateHttpServer(Config); + + std::filesystem::path TempDir = std::filesystem::temp_directory_path() / "zen-trace-viewer"; + std::error_code Ec; + std::filesystem::create_directories(TempDir, Ec); + + int EffectivePort = Server->Initialize(m_Port, TempDir); + if (EffectivePort <= 0) + { + throw zen::runtime_error("Failed to initialize HTTP server"); + } + + TraceViewerService ViewerService(Model, std::move(Symbols)); + Server->RegisterService(ViewerService); + + std::string Url = fmt::format("http://{}:{}{}", m_Bind, EffectivePort, ViewerService.BaseUri()); + ZEN_CONSOLE("Serving trace viewer at {}", Url); + ZEN_CONSOLE("Press Ctrl+C to stop"); + + if (!m_NoBrowser) + { + try + { + LaunchBrowser(Url); + } + catch (const std::exception& E) + { + ZEN_WARN("Failed to launch browser: {}", E.what()); + } + } + + Server->Run(/*IsInteractiveSession=*/true); + Server->Close(); +} + +////////////////////////////////////////////////////////////////////////// +// TraceTrimSubCmd + +TraceTrimSubCmd::TraceTrimSubCmd() : ZenSubCmdBase("trim", "Trim a .utrace file to a time range while preserving important events") +{ + SubOptions().add_option("", "", "file", "Path to input .utrace file", cxxopts::value(m_TraceFilePath), "<filepath>"); + SubOptions().add_option("", "o", "output", "Path to output .utrace file", cxxopts::value(m_OutputPath), "<filepath>"); + SubOptions().add_option("", + "", + "start", + "Start of the time window in seconds from the beginning of the trace", + cxxopts::value(m_StartSec)->default_value("0"), + "<seconds>"); + SubOptions().add_option("", + "", + "end", + "End of the time window in seconds from the beginning of the trace", + cxxopts::value(m_EndSec)->default_value("0"), + "<seconds>"); + SubOptions().parse_positional({"file"}); + SubOptions().positional_help("<file.utrace>"); +} + +void +TraceTrimSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + std::filesystem::path InputPath = trace_detail::ResolveTraceFile(m_TraceFilePath, SubOptions()); + + if (m_OutputPath.empty()) + { + throw zen::OptionParseException("--output is required", SubOptions().help()); + } + if (m_EndSec <= m_StartSec) + { + throw zen::OptionParseException("--end must be greater than --start", SubOptions().help()); + } + + trace_detail::TraceTrimArgs Args; + Args.InputPath = InputPath; + Args.OutputPath = std::filesystem::absolute(m_OutputPath); + Args.StartSec = m_StartSec; + Args.EndSec = m_EndSec; + + trace_detail::RunTraceTrim(Args); +} + +////////////////////////////////////////////////////////////////////////// +// TraceStartSubCmd + +TraceStartSubCmd::TraceStartSubCmd() : ZenSubCmdBase("start", "Start zen server realtime tracing") +{ + SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + SubOptions().add_option("", "", "host", "Stream trace data to a remote host", cxxopts::value(m_TraceHost), "<hostip>"); + SubOptions().add_option("", "", "file", "Write trace data to a file", cxxopts::value(m_TraceFile), "<filepath>"); +} + +void +TraceStartSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + if (m_TraceHost.empty() && m_TraceFile.empty()) + { + throw OptionParseException("Either --host or --file is required", SubOptions().help()); + } + if (!m_TraceHost.empty() && !m_TraceFile.empty()) + { + throw OptionParseException("--host and --file are mutually exclusive", SubOptions().help()); + } + + std::string StartArg = m_TraceHost.empty() ? fmt::format("file={}", m_TraceFile) : fmt::format("host={}", m_TraceHost); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "start"}); + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg))) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + } + else + { + Response.ThrowError("Trace start failed"); + } +} + +////////////////////////////////////////////////////////////////////////// +// TraceStopSubCmd + +TraceStopSubCmd::TraceStopSubCmd() : ZenSubCmdBase("stop", "Stop zen server realtime tracing") +{ + SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} + +void +TraceStopSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "stop"}); + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = Http.Post("/admin/trace/stop"sv)) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + } + else + { + Response.ThrowError("Trace stop failed"); + } +} + +////////////////////////////////////////////////////////////////////////// +// TraceStatusSubCmd + +TraceStatusSubCmd::TraceStatusSubCmd() : ZenSubCmdBase("status", "Report zen server realtime tracing status") +{ + SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} + +void +TraceStatusSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "status"}); + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = Http.Get("/admin/trace"sv)) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + } + else + { + Response.ThrowError("Trace status failed"); + } +} + +////////////////////////////////////////////////////////////////////////// +// TraceCommand + +TraceCommand::TraceCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); + m_Options.parse_positional({"subcommand"}); + + AddSubCommand(m_AnalyzeSubCmd); + AddSubCommand(m_InspectSubCmd); + AddSubCommand(m_ServeSubCmd); + AddSubCommand(m_TrimSubCmd); + AddSubCommand(m_StartSubCmd); + AddSubCommand(m_StopSubCmd); + AddSubCommand(m_StatusSubCmd); +} + +TraceCommand::~TraceCommand() = default; + +} // namespace zen diff --git a/src/zen/trace/trace_cmd.h b/src/zen/trace/trace_cmd.h new file mode 100644 index 000000000..bb2759241 --- /dev/null +++ b/src/zen/trace/trace_cmd.h @@ -0,0 +1,123 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zen.h" + +#include <filesystem> +#include <string> + +namespace zen { + +class TraceAnalyzeSubCmd : public ZenSubCmdBase +{ +public: + TraceAnalyzeSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::filesystem::path m_TraceFilePath; + int m_LiveAllocs = 0; + int m_Churn = 0; + int m_ChurnMin = 1000; + std::string m_Symbols; + std::string m_CallstackSkip; + bool m_NoCache = false; + bool m_NoCallstackHeuristic = false; + std::filesystem::path m_HtmlReportPath; +}; + +class TraceInspectSubCmd : public ZenSubCmdBase +{ +public: + TraceInspectSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::filesystem::path m_TraceFilePath; +}; + +class TraceServeSubCmd : public ZenSubCmdBase +{ +public: + TraceServeSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::filesystem::path m_TraceFilePath; + std::string m_Symbols; + int m_Port = 0; + std::string m_Bind = "127.0.0.1"; + bool m_NoBrowser = false; +}; + +class TraceTrimSubCmd : public ZenSubCmdBase +{ +public: + TraceTrimSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::filesystem::path m_TraceFilePath; + std::filesystem::path m_OutputPath; + double m_StartSec = 0.0; + double m_EndSec = 0.0; +}; + +class TraceStartSubCmd : public ZenSubCmdBase +{ +public: + TraceStartSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_HostName; + std::string m_TraceHost; + std::string m_TraceFile; +}; + +class TraceStopSubCmd : public ZenSubCmdBase +{ +public: + TraceStopSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_HostName; +}; + +class TraceStatusSubCmd : public ZenSubCmdBase +{ +public: + TraceStatusSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_HostName; +}; + +class TraceCommand : public ZenCmdWithSubCommands +{ +public: + static constexpr char Name[] = "trace"; + static constexpr char Description[] = "Control zen realtime tracing and work with .utrace files"; + + TraceCommand(); + ~TraceCommand(); + + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + + TraceAnalyzeSubCmd m_AnalyzeSubCmd; + TraceInspectSubCmd m_InspectSubCmd; + TraceServeSubCmd m_ServeSubCmd; + TraceTrimSubCmd m_TrimSubCmd; + TraceStartSubCmd m_StartSubCmd; + TraceStopSubCmd m_StopSubCmd; + TraceStatusSubCmd m_StatusSubCmd; +}; + +} // namespace zen diff --git a/src/zen/trace/trace_memory.cpp b/src/zen/trace/trace_memory.cpp new file mode 100644 index 000000000..704b8bcde --- /dev/null +++ b/src/zen/trace/trace_memory.cpp @@ -0,0 +1,901 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_memory.h" + +#include "trace_model.h" + +#include <zencore/fmtutils.h> +#include <zencore/logging.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/sort.h> +#include <analysis/analyzer.h> +ZEN_THIRD_PARTY_INCLUDES_END + +using namespace zen::trace_detail; + +////////////////////////////////////////////////////////////////////////////// +// Event outlines for Memory.* trace events +// +// Field names and types match the UE wire format exactly. +// See Engine/Source/Runtime/Core/Private/ProfilingDebugging/MemoryTrace.cpp. + +// clang-format off +begin_outline(Memory, Init) + field(uint8, Version) + field(uint32, MarkerPeriod) + field(uint8, MinAlignment) + field(uint8, SizeShift) + field(uint64, PageSize) +end_outline() + +begin_outline(Memory, Marker) + field(uint64, Cycle) +end_outline() + +begin_outline(Memory, Alloc) + field(uint64, Address) + field(uint32, CallstackId) + field(uint32, Size) + field(uint8, AlignmentPow2_SizeLower) + field(uint8, RootHeap) +end_outline() + +begin_outline(Memory, AllocSystem) + field(uint64, Address) + field(uint32, CallstackId) + field(uint32, Size) + field(uint8, AlignmentPow2_SizeLower) +end_outline() + +begin_outline(Memory, AllocVideo) + field(uint64, Address) + field(uint32, CallstackId) + field(uint32, Size) + field(uint8, AlignmentPow2_SizeLower) +end_outline() + +begin_outline(Memory, Free) + field(uint64, Address) + field(uint32, CallstackId) + field(uint8, RootHeap) +end_outline() + +begin_outline(Memory, FreeSystem) + field(uint64, Address) + field(uint32, CallstackId) +end_outline() + +begin_outline(Memory, FreeVideo) + field(uint64, Address) + field(uint32, CallstackId) +end_outline() + +begin_outline(Memory, ReallocAlloc) + field(uint64, Address) + field(uint32, CallstackId) + field(uint32, Size) + field(uint8, AlignmentPow2_SizeLower) + field(uint8, RootHeap) +end_outline() + +begin_outline(Memory, ReallocAllocSystem) + field(uint64, Address) + field(uint32, CallstackId) + field(uint32, Size) + field(uint8, AlignmentPow2_SizeLower) +end_outline() + +begin_outline(Memory, ReallocFree) + field(uint64, Address) + field(uint32, CallstackId) + field(uint8, RootHeap) +end_outline() + +begin_outline(Memory, ReallocFreeSystem) + field(uint64, Address) + field(uint32, CallstackId) +end_outline() + +begin_outline(Memory, HeapSpec) + field(uint32, Id) + field(uint32, ParentId) + field(uint16, Flags) + field(FieldStr, Name) +end_outline() + +begin_outline(Memory, HeapMarkAlloc) + field(uint64, Address) + field(uint32, CallstackId) + field(uint16, Flags) + field(uint32, Heap) +end_outline() + +begin_outline(Memory, HeapUnmarkAlloc) + field(uint64, Address) + field(uint32, CallstackId) + field(uint32, Heap) +end_outline() + +begin_outline(Memory, TagSpec) + field(int32, Tag) + field(int32, Parent) + field(FieldStr, Display) +end_outline() + +begin_outline(Memory, CallstackSpec) + field(uint32, CallstackId) + field(uint64[], Frames) +end_outline() + +begin_outline(Memory, CallstackSpecDeltaVarInt) + field(uint32, CallstackId) + field(uint8[], CompressedFrames) +end_outline() + +begin_outline(Memory, CallstackSpecDelta7bit) + field(uint32, CallstackId) + field(uint8[], CompressedFrames) +end_outline() + +begin_outline(Memory, CallstackSpecXORAndRLE) + field(uint32, CallstackId) + field(uint8[], CompressedFrames) +end_outline() + // clang-format on + + ////////////////////////////////////////////////////////////////////////////// + // Callstack decompression helpers + + namespace +{ + inline int64_t ZigZagDecode(uint64_t Encoded) { return int64_t(Encoded >> 1) ^ -int64_t(Encoded & 1); } + + // UE VarInt: leading 1-bits in the first byte indicate total byte count. + // 0xxxxxxx = 1 byte (7 value bits) + // 10xxxxxx = 2 bytes (14 value bits) + // 110xxxxx = 3 bytes (21 value bits) ...up to 9 bytes. + // Remaining bytes are big-endian value continuation. + eastl::vector<uint64_t> DecodeDeltaVarInt(const uint8_t* Data, uint32_t Size) + { + eastl::vector<uint64_t> Frames; + uint64_t Prev = 0; + const uint8_t* Cur = Data; + const uint8_t* End = Data + Size; + + while (Cur < End) + { + uint8_t First = *Cur; + uint32_t ByteCount = 1; + uint8_t Mask = 0x80; + while ((First & Mask) && ByteCount < 9) + { + ByteCount++; + Mask >>= 1; + } + + if (Cur + ByteCount > End) + { + break; + } + + uint64_t Raw = 0; + if (ByteCount == 9) + { + // First byte is 0xFF; next 8 bytes are the raw value. + for (uint32_t I = 1; I <= 8; I++) + { + Raw = (Raw << 8) | Cur[I]; + } + } + else + { + // First byte contributes value bits after stripping the length prefix. + uint8_t ValueMask = uint8_t((1u << (8 - ByteCount)) - 1); + Raw = First & ValueMask; + for (uint32_t I = 1; I < ByteCount; I++) + { + Raw = (Raw << 8) | Cur[I]; + } + } + Cur += ByteCount; + + int64_t Delta = ZigZagDecode(Raw); + Prev = uint64_t(int64_t(Prev) + Delta); + Frames.push_back(Prev); + } + + return Frames; + } + + // 7-bit continuation encoding: bit 7 = more bytes, bits 0-6 = value (little-endian). + eastl::vector<uint64_t> DecodeDelta7bit(const uint8_t* Data, uint32_t Size) + { + eastl::vector<uint64_t> Frames; + uint64_t Prev = 0; + const uint8_t* Cur = Data; + const uint8_t* End = Data + Size; + + while (Cur < End) + { + uint64_t Raw = 0; + uint32_t Shift = 0; + for (;;) + { + if (Cur >= End) + { + break; + } + uint8_t Byte = *Cur++; + Raw |= uint64_t(Byte & 0x7F) << Shift; + Shift += 7; + if ((Byte & 0x80) == 0) + { + break; + } + } + + int64_t Delta = ZigZagDecode(Raw); + Prev = uint64_t(int64_t(Prev) + Delta); + Frames.push_back(Prev); + } + + return Frames; + } + + // XOR + RLE: first byte = leading zero bit count in (frame XOR prev). + // Remaining ceil((64 - zeros) / 8) bytes are the non-zero suffix, little-endian. + eastl::vector<uint64_t> DecodeXORAndRLE(const uint8_t* Data, uint32_t Size) + { + eastl::vector<uint64_t> Frames; + uint64_t Prev = 0; + const uint8_t* Cur = Data; + const uint8_t* End = Data + Size; + + while (Cur < End) + { + uint8_t LeadingZeros = *Cur++; + if (LeadingZeros >= 64) + { + Frames.push_back(Prev); + continue; + } + + uint32_t ValueBits = 64 - LeadingZeros; + uint32_t ValueBytes = (ValueBits + 7) / 8; + + if (Cur + ValueBytes > End) + { + break; + } + + uint64_t XorVal = 0; + for (uint32_t I = 0; I < ValueBytes; I++) + { + XorVal |= uint64_t(Cur[I]) << (I * 8); + } + Cur += ValueBytes; + + Prev ^= XorVal; + Frames.push_back(Prev); + } + + return Frames; + } + +} // anonymous namespace + +////////////////////////////////////////////////////////////////////////////// +// AllocationAnalyzer implementation + +AllocationAnalyzer::AllocationAnalyzer(const TraceTiming* Timing) : m_Timing(Timing) +{ +} + +void +AllocationAnalyzer::subscribe(Vector<Subscription>& Subs) +{ + Subs.emplace_back(this, &AllocationAnalyzer::OnInit); + Subs.emplace_back(this, &AllocationAnalyzer::OnMarker); + Subs.emplace_back(this, &AllocationAnalyzer::OnAlloc); + Subs.emplace_back(this, &AllocationAnalyzer::OnAllocSystem); + Subs.emplace_back(this, &AllocationAnalyzer::OnAllocVideo); + Subs.emplace_back(this, &AllocationAnalyzer::OnFree); + Subs.emplace_back(this, &AllocationAnalyzer::OnFreeSystem); + Subs.emplace_back(this, &AllocationAnalyzer::OnFreeVideo); + Subs.emplace_back(this, &AllocationAnalyzer::OnReallocAlloc); + Subs.emplace_back(this, &AllocationAnalyzer::OnReallocAllocSystem); + Subs.emplace_back(this, &AllocationAnalyzer::OnReallocFree); + Subs.emplace_back(this, &AllocationAnalyzer::OnReallocFreeSystem); + Subs.emplace_back(this, &AllocationAnalyzer::OnHeapSpec); + Subs.emplace_back(this, &AllocationAnalyzer::OnHeapMarkAlloc); + Subs.emplace_back(this, &AllocationAnalyzer::OnHeapUnmarkAlloc); + Subs.emplace_back(this, &AllocationAnalyzer::OnTagSpec); +} + +////////////////////////////////////////////////////////////////////////////// +// Internal helpers + +uint64_t +AllocationAnalyzer::DecodeAllocSize(uint32_t RawSize, uint8_t AlignSizeLower) const +{ + uint32_t Shift = m_SizeShift; + uint32_t LowMask = (1u << Shift) - 1; + return (uint64_t(RawSize) << Shift) | (AlignSizeLower & LowMask); +} + +void +AllocationAnalyzer::HandleAlloc(uint64_t Address, uint64_t Size, uint8_t RootHeap, uint32_t CallstackId, uint32_t ThreadId, bool IsRealloc) +{ + // If address is already tracked (shouldn't normally happen), treat as + // implicit free of the old allocation so the counters stay consistent. + auto It = m_LiveAllocs.find(Address); + if (It != m_LiveAllocs.end()) + { + // Heap-marked allocs were already subtracted from totals in OnHeapMarkAlloc. + if (!It->second.IsHeap) + { + int64_t OldSize = int64_t(It->second.Size); + uint8_t OldHeap = It->second.RootHeap; + m_CurrentBytes -= OldSize; + if (OldHeap == 0) + { + m_SystemBytes -= OldSize; + } + else if (OldHeap == 1) + { + m_VideoBytes -= OldSize; + } + auto HIt = m_RootHeapStats.find(OldHeap); + if (HIt != m_RootHeapStats.end()) + { + HIt->second.CurrentBytes -= OldSize; + HIt->second.FreeCount++; + } + } + It->second = LiveAlloc{Size, CallstackId, ThreadId, m_AllocEventSeq, RootHeap, false}; + } + else + { + m_LiveAllocs.insert({Address, LiveAlloc{Size, CallstackId, ThreadId, m_AllocEventSeq, RootHeap, false}}); + } + + int64_t SignedSize = int64_t(Size); + m_CurrentBytes += SignedSize; + if (RootHeap == 0) + { + m_SystemBytes += SignedSize; + } + else if (RootHeap == 1) + { + m_VideoBytes += SignedSize; + } + + // Update per-root-heap stats + HeapStat& HStat = m_RootHeapStats[RootHeap]; + HStat.HeapId = RootHeap; + HStat.CurrentBytes += SignedSize; + HStat.AllocCount++; + if (HStat.CurrentBytes > HStat.PeakBytes) + { + HStat.PeakBytes = HStat.CurrentBytes; + } + + // Track global peak + if (m_CurrentBytes > m_PeakBytes) + { + m_PeakBytes = m_CurrentBytes; + m_PeakTimeUs = m_LastMarkerTimeUs; + } + + if (IsRealloc) + { + m_TotalReallocAllocs++; + } + else + { + m_TotalAllocs++; + } + + // Churn tracking + m_AllocEventSeq++; + if (CallstackId != 0) + { + ChurnAccum& Churn = m_ChurnByCallstack[CallstackId]; + Churn.TotalAllocs++; + Churn.TotalBytes += Size; + } + + // Size histogram: bucket 0 captures zero-size allocs, bucket i (i>=1) + // captures sizes in [2^(i-1)+1, 2^i]. Use ceil(log2) so power-of-two + // sizes land on their own bucket (e.g. 16 -> bucket 4 = (8, 16]). + size_t BucketIndex = 0; + if (Size > 0) + { + uint64_t Shifted = Size - 1; + while (Shifted > 0 && BucketIndex < kSizeHistogramBuckets - 1) + { + Shifted >>= 1; + ++BucketIndex; + } + } + m_SizeHistogram[BucketIndex].Count++; + m_SizeHistogram[BucketIndex].Bytes += Size; +} + +void +AllocationAnalyzer::HandleFree(uint64_t Address, uint8_t /*RootHeap*/, uint32_t /*CallstackId*/, bool IsRealloc) +{ + auto It = m_LiveAllocs.find(Address); + if (It == m_LiveAllocs.end()) + { + // Allocation happened before the trace started -- nothing to subtract. + if (IsRealloc) + { + m_TotalReallocFrees++; + } + else + { + m_TotalFrees++; + } + return; + } + + int64_t Size = int64_t(It->second.Size); + uint8_t AllocHeap = It->second.RootHeap; + bool WasHeap = It->second.IsHeap; + uint32_t AllocCsId = It->second.CallstackId; + uint64_t AllocEventSeq = It->second.EventSeq; + + // Heap-marked allocs were already subtracted from totals in OnHeapMarkAlloc. + if (!WasHeap) + { + m_CurrentBytes -= Size; + if (AllocHeap == 0) + { + m_SystemBytes -= Size; + } + else if (AllocHeap == 1) + { + m_VideoBytes -= Size; + } + + auto HIt = m_RootHeapStats.find(AllocHeap); + if (HIt != m_RootHeapStats.end()) + { + HIt->second.CurrentBytes -= Size; + HIt->second.FreeCount++; + } + } + + m_LiveAllocs.erase(It); + + // Churn tracking: record event distance for this alloc→free pair. + // Short distances indicate short-lived (churny) allocations. + if (AllocCsId != 0) + { + uint64_t Distance = m_AllocEventSeq - AllocEventSeq; + auto ChurnIt = m_ChurnByCallstack.find(AllocCsId); + if (ChurnIt != m_ChurnByCallstack.end()) + { + ChurnIt->second.ChurnDistanceSum += Distance; + ChurnIt->second.ChurnAllocs++; + ChurnIt->second.ChurnBytes += uint64_t(Size); + } + } + + if (IsRealloc) + { + m_TotalReallocFrees++; + } + else + { + m_TotalFrees++; + } +} + +void +AllocationAnalyzer::MaybeEmitSample(uint32_t TimeUs) +{ + if (TimeUs < m_LastSampleTimeUs + kTimelineSampleIntervalUs) + { + return; + } + m_LastSampleTimeUs = TimeUs; + m_Timeline.push_back(MemoryTimelineSample{ + .TimeUs = TimeUs, + .TotalAllocatedBytes = m_CurrentBytes, + .SystemBytes = m_SystemBytes, + .VideoBytes = m_VideoBytes, + }); +} + +////////////////////////////////////////////////////////////////////////////// +// Event handlers + +void +AllocationAnalyzer::OnInit(const ::Memory_Init& Ev) +{ + m_SizeShift = Ev.SizeShift(); + m_Initialized = true; + ZEN_DEBUG("Memory trace init: version={}, sizeShift={}, minAlignment={}, markerPeriod={}, pageSize={}", + Ev.Version(), + m_SizeShift, + Ev.MinAlignment(), + Ev.MarkerPeriod(), + Ev.PageSize()); +} + +void +AllocationAnalyzer::OnMarker(const ::Memory_Marker& Ev) +{ + if (!m_Timing || m_Timing->Freq == 0) + { + return; + } + uint32_t TimeUs = m_Timing->CycleToTimeUs(Ev.Cycle()); + m_LastMarkerTimeUs = TimeUs; + m_HasReceivedMarker = true; + MaybeEmitSample(TimeUs); +} + +void +AllocationAnalyzer::OnAlloc(const ::Memory_Alloc& Ev) +{ + uint64_t Size = DecodeAllocSize(Ev.Size(), Ev.AlignmentPow2_SizeLower()); + HandleAlloc(Ev.Address(), Size, Ev.RootHeap(), Ev.CallstackId(), Ev.get_thread_id(), /*IsRealloc=*/false); +} + +void +AllocationAnalyzer::OnAllocSystem(const ::Memory_AllocSystem& Ev) +{ + uint64_t Size = DecodeAllocSize(Ev.Size(), Ev.AlignmentPow2_SizeLower()); + HandleAlloc(Ev.Address(), Size, /*RootHeap=*/0, Ev.CallstackId(), Ev.get_thread_id(), /*IsRealloc=*/false); +} + +void +AllocationAnalyzer::OnAllocVideo(const ::Memory_AllocVideo& Ev) +{ + uint64_t Size = DecodeAllocSize(Ev.Size(), Ev.AlignmentPow2_SizeLower()); + HandleAlloc(Ev.Address(), Size, /*RootHeap=*/1, Ev.CallstackId(), Ev.get_thread_id(), /*IsRealloc=*/false); +} + +void +AllocationAnalyzer::OnFree(const ::Memory_Free& Ev) +{ + HandleFree(Ev.Address(), Ev.RootHeap(), Ev.CallstackId(), /*IsRealloc=*/false); +} + +void +AllocationAnalyzer::OnFreeSystem(const ::Memory_FreeSystem& Ev) +{ + HandleFree(Ev.Address(), /*RootHeap=*/0, Ev.CallstackId(), /*IsRealloc=*/false); +} + +void +AllocationAnalyzer::OnFreeVideo(const ::Memory_FreeVideo& Ev) +{ + HandleFree(Ev.Address(), /*RootHeap=*/1, Ev.CallstackId(), /*IsRealloc=*/false); +} + +void +AllocationAnalyzer::OnReallocAlloc(const ::Memory_ReallocAlloc& Ev) +{ + uint64_t Size = DecodeAllocSize(Ev.Size(), Ev.AlignmentPow2_SizeLower()); + HandleAlloc(Ev.Address(), Size, Ev.RootHeap(), Ev.CallstackId(), Ev.get_thread_id(), /*IsRealloc=*/true); +} + +void +AllocationAnalyzer::OnReallocAllocSystem(const ::Memory_ReallocAllocSystem& Ev) +{ + uint64_t Size = DecodeAllocSize(Ev.Size(), Ev.AlignmentPow2_SizeLower()); + HandleAlloc(Ev.Address(), Size, /*RootHeap=*/0, Ev.CallstackId(), Ev.get_thread_id(), /*IsRealloc=*/true); +} + +void +AllocationAnalyzer::OnReallocFree(const ::Memory_ReallocFree& Ev) +{ + HandleFree(Ev.Address(), Ev.RootHeap(), Ev.CallstackId(), /*IsRealloc=*/true); +} + +void +AllocationAnalyzer::OnReallocFreeSystem(const ::Memory_ReallocFreeSystem& Ev) +{ + HandleFree(Ev.Address(), /*RootHeap=*/0, Ev.CallstackId(), /*IsRealloc=*/true); +} + +void +AllocationAnalyzer::OnHeapSpec(const ::Memory_HeapSpec& Ev) +{ + uint32_t Id = Ev.Id(); + HeapInfo& Info = m_Heaps[Id]; + Info.Id = Id; + Info.ParentId = Ev.ParentId(); + Info.Flags = Ev.Flags(); + Info.Name = SafeFieldStr(Ev.Name()); +} + +void +AllocationAnalyzer::OnHeapMarkAlloc(const ::Memory_HeapMarkAlloc& Ev) +{ + uint64_t Address = Ev.Address(); + auto It = m_LiveAllocs.find(Address); + if (It == m_LiveAllocs.end()) + { + return; + } + + LiveAlloc& Alloc = It->second; + if (Alloc.IsHeap) + { + return; // already marked + } + + Alloc.IsHeap = true; + + // Remove this allocation from the running totals — heap-marked + // allocations represent address-space reservations (e.g. module images) + // and should not count towards the regular memory budget. + int64_t SignedSize = int64_t(Alloc.Size); + m_CurrentBytes -= SignedSize; + if (Alloc.RootHeap == 0) + { + m_SystemBytes -= SignedSize; + } + else if (Alloc.RootHeap == 1) + { + m_VideoBytes -= SignedSize; + } + auto HIt = m_RootHeapStats.find(Alloc.RootHeap); + if (HIt != m_RootHeapStats.end()) + { + HIt->second.CurrentBytes -= SignedSize; + } +} + +void +AllocationAnalyzer::OnHeapUnmarkAlloc(const ::Memory_HeapUnmarkAlloc& Ev) +{ + uint64_t Address = Ev.Address(); + auto It = m_LiveAllocs.find(Address); + if (It == m_LiveAllocs.end()) + { + return; + } + + LiveAlloc& Alloc = It->second; + if (!Alloc.IsHeap) + { + return; // not marked + } + + Alloc.IsHeap = false; + + // Add back to running totals. + int64_t SignedSize = int64_t(Alloc.Size); + m_CurrentBytes += SignedSize; + if (Alloc.RootHeap == 0) + { + m_SystemBytes += SignedSize; + } + else if (Alloc.RootHeap == 1) + { + m_VideoBytes += SignedSize; + } + auto HIt = m_RootHeapStats.find(Alloc.RootHeap); + if (HIt != m_RootHeapStats.end()) + { + HIt->second.CurrentBytes += SignedSize; + } +} + +void +AllocationAnalyzer::OnTagSpec(const ::Memory_TagSpec& Ev) +{ + int32_t Tag = Ev.Tag(); + TagInfo& Info = m_Tags[Tag]; + Info.Tag = Tag; + Info.Parent = Ev.Parent(); + Info.Display = SafeFieldStr(Ev.Display()); +} + +////////////////////////////////////////////////////////////////////////////// +// Public accessors + +AllocationSummary +AllocationAnalyzer::Summary() const +{ + AllocationSummary S; + S.HasMemoryData = m_Initialized || m_TotalAllocs > 0; + S.TotalAllocs = m_TotalAllocs; + S.TotalFrees = m_TotalFrees; + S.TotalReallocAllocs = m_TotalReallocAllocs; + S.TotalReallocFrees = m_TotalReallocFrees; + S.PeakBytes = m_PeakBytes; + S.PeakTimeUs = m_PeakTimeUs; + S.EndBytes = m_CurrentBytes; + + uint32_t LiveCount = 0; + for (const auto& [Addr, Alloc] : m_LiveAllocs) + { + if (!Alloc.IsHeap) + { + ++LiveCount; + } + } + S.LiveAllocations = LiveCount; + return S; +} + +void +AllocationAnalyzer::EmitFinalSample(uint32_t TraceEndUs) +{ + if (!m_Initialized) + { + return; + } + // Force-emit a final sample at the trace end so the timeline captures + // the terminal memory state even if no Marker arrived recently. + uint32_t FinalTimeUs = m_HasReceivedMarker ? std::max(m_LastMarkerTimeUs, TraceEndUs) : TraceEndUs; + m_Timeline.push_back(MemoryTimelineSample{ + .TimeUs = FinalTimeUs, + .TotalAllocatedBytes = m_CurrentBytes, + .SystemBytes = m_SystemBytes, + .VideoBytes = m_VideoBytes, + }); +} + +eastl::vector<CallstackAllocStat> +AllocationAnalyzer::BuildCallstackStats() const +{ + eastl::hash_map<uint32_t, CallstackAllocStat> Map; + for (const auto& [Addr, Alloc] : m_LiveAllocs) + { + if (Alloc.CallstackId == 0 || Alloc.IsHeap) + { + continue; + } + CallstackAllocStat& S = Map[Alloc.CallstackId]; + S.CallstackId = Alloc.CallstackId; + S.LiveBytes += int64_t(Alloc.Size); + S.LiveCount++; + if (eastl::find(S.ThreadIds.begin(), S.ThreadIds.end(), Alloc.ThreadId) == S.ThreadIds.end()) + { + S.ThreadIds.push_back(Alloc.ThreadId); + } + } + + eastl::vector<CallstackAllocStat> Result; + Result.reserve(Map.size()); + for (auto& [Id, Stat] : Map) + { + Result.push_back(Stat); + } + eastl::sort(Result.begin(), Result.end(), [](const CallstackAllocStat& A, const CallstackAllocStat& B) { + return A.LiveBytes > B.LiveBytes; + }); + return Result; +} + +eastl::vector<CallstackChurnStat> +AllocationAnalyzer::BuildChurnStats(uint64_t ChurnDistanceThreshold) const +{ + // The ChurnAccum already separates total allocs from churny allocs. + // ChurnAllocs/ChurnBytes count every freed allocation (regardless of + // distance). We now need to re-bucket using the threshold. But since + // we only stored the sum of distances (not per-alloc distances), we + // use the average: if MeanDistance <= threshold, all freed allocs from + // that callstack are considered churny. This is an approximation — + // a per-alloc histogram would be more precise but much more expensive. + eastl::vector<CallstackChurnStat> Result; + Result.reserve(m_ChurnByCallstack.size()); + for (const auto& [Id, Churn] : m_ChurnByCallstack) + { + if (Churn.ChurnAllocs == 0) + { + continue; + } + double MeanDist = double(Churn.ChurnDistanceSum) / double(Churn.ChurnAllocs); + if (MeanDist > double(ChurnDistanceThreshold)) + { + continue; + } + CallstackChurnStat S; + S.CallstackId = Id; + S.ChurnAllocs = Churn.ChurnAllocs; + S.ChurnBytes = Churn.ChurnBytes; + S.TotalAllocs = Churn.TotalAllocs; + S.TotalBytes = Churn.TotalBytes; + S.MeanDistance = MeanDist; + Result.push_back(S); + } + eastl::sort(Result.begin(), Result.end(), [](const CallstackChurnStat& A, const CallstackChurnStat& B) { + return A.ChurnAllocs > B.ChurnAllocs; + }); + return Result; +} + +eastl::vector<AllocSizeBucket> +AllocationAnalyzer::BuildSizeHistogram() const +{ + eastl::vector<AllocSizeBucket> Result; + Result.reserve(kSizeHistogramBuckets); + for (size_t I = 0; I < kSizeHistogramBuckets; ++I) + { + const SizeBucketAccum& Accum = m_SizeHistogram[I]; + if (Accum.Count == 0) + { + continue; + } + AllocSizeBucket Bucket; + if (I == 0) + { + Bucket.MinSize = 0; + Bucket.MaxSize = 0; + } + else + { + // Bucket i covers (2^(i-1), 2^i]; bucket 1 is just size 1. + Bucket.MinSize = (I == 1) ? 1 : ((uint64_t(1) << (I - 1)) + 1); + Bucket.MaxSize = (I >= 64) ? ~uint64_t(0) : (uint64_t(1) << I); + } + Bucket.Count = Accum.Count; + Bucket.Bytes = Accum.Bytes; + Result.push_back(Bucket); + } + return Result; +} + +////////////////////////////////////////////////////////////////////////////// +// CallstackAnalyzer implementation + +void +CallstackAnalyzer::subscribe(Vector<Subscription>& Subs) +{ + Subs.emplace_back(this, &CallstackAnalyzer::OnCallstackSpec); + Subs.emplace_back(this, &CallstackAnalyzer::OnCallstackSpecDeltaVarInt); + Subs.emplace_back(this, &CallstackAnalyzer::OnCallstackSpecDelta7bit); + Subs.emplace_back(this, &CallstackAnalyzer::OnCallstackSpecXORAndRLE); +} + +void +CallstackAnalyzer::StoreCallstack(uint32_t Id, const uint64_t* Frames, size_t Count) +{ + if (Id == 0 || Count == 0) + { + return; + } + auto& Entry = m_Callstacks[Id]; + Entry.assign(Frames, Frames + Count); +} + +void +CallstackAnalyzer::OnCallstackSpec(const ::Memory_CallstackSpec& Ev) +{ + Array<uint64[]> Frames = Ev.Frames(); + StoreCallstack(Ev.CallstackId(), Frames.get(), Frames.get_count()); +} + +void +CallstackAnalyzer::OnCallstackSpecDeltaVarInt(const ::Memory_CallstackSpecDeltaVarInt& Ev) +{ + Array<uint8[]> Compressed = Ev.CompressedFrames(); + eastl::vector<uint64_t> Frames = DecodeDeltaVarInt(Compressed.get(), Compressed.get_size()); + StoreCallstack(Ev.CallstackId(), Frames.data(), Frames.size()); +} + +void +CallstackAnalyzer::OnCallstackSpecDelta7bit(const ::Memory_CallstackSpecDelta7bit& Ev) +{ + Array<uint8[]> Compressed = Ev.CompressedFrames(); + eastl::vector<uint64_t> Frames = DecodeDelta7bit(Compressed.get(), Compressed.get_size()); + StoreCallstack(Ev.CallstackId(), Frames.data(), Frames.size()); +} + +void +CallstackAnalyzer::OnCallstackSpecXORAndRLE(const ::Memory_CallstackSpecXORAndRLE& Ev) +{ + Array<uint8[]> Compressed = Ev.CompressedFrames(); + eastl::vector<uint64_t> Frames = DecodeXORAndRLE(Compressed.get(), Compressed.get_size()); + StoreCallstack(Ev.CallstackId(), Frames.data(), Frames.size()); +} diff --git a/src/zen/trace/trace_memory.h b/src/zen/trace/trace_memory.h new file mode 100644 index 000000000..da33d8218 --- /dev/null +++ b/src/zen/trace/trace_memory.h @@ -0,0 +1,301 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/fixed_vector.h> +#include <EASTL/hash_map.h> +#include <EASTL/vector.h> +#include <analysis/analyzer.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <cstdint> +#include <string> + +// Forward declarations of outline types (defined in trace_memory.cpp). +// These are global-scope structs created by the begin_outline() macro. +struct Memory_Init; +struct Memory_Marker; +struct Memory_Alloc; +struct Memory_AllocSystem; +struct Memory_AllocVideo; +struct Memory_Free; +struct Memory_FreeSystem; +struct Memory_FreeVideo; +struct Memory_ReallocAlloc; +struct Memory_ReallocAllocSystem; +struct Memory_ReallocFree; +struct Memory_ReallocFreeSystem; +struct Memory_HeapSpec; +struct Memory_HeapMarkAlloc; +struct Memory_HeapUnmarkAlloc; +struct Memory_TagSpec; +struct Memory_CallstackSpec; +struct Memory_CallstackSpecDeltaVarInt; +struct Memory_CallstackSpecDelta7bit; +struct Memory_CallstackSpecXORAndRLE; + +namespace zen::trace_detail { + +struct TraceTiming; + +// -- Allocation data structures -------------------------------------------- + +struct HeapInfo +{ + uint32_t Id = 0; + uint32_t ParentId = ~0u; + uint16_t Flags = 0; // EMemoryTraceHeapFlags bits + std::string Name; +}; + +struct TagInfo +{ + int32_t Tag = 0; + int32_t Parent = 0; + std::string Display; +}; + +struct MemoryTimelineSample +{ + uint32_t TimeUs; + int64_t TotalAllocatedBytes; + int64_t SystemBytes; + int64_t VideoBytes; +}; + +struct HeapStat +{ + uint32_t HeapId = 0; + int64_t CurrentBytes = 0; + int64_t PeakBytes = 0; + uint64_t AllocCount = 0; + uint64_t FreeCount = 0; +}; + +struct AllocationSummary +{ + bool HasMemoryData = false; + uint64_t TotalAllocs = 0; + uint64_t TotalFrees = 0; + uint64_t TotalReallocAllocs = 0; + uint64_t TotalReallocFrees = 0; + int64_t PeakBytes = 0; + uint32_t PeakTimeUs = 0; + int64_t EndBytes = 0; + uint32_t LiveAllocations = 0; +}; + +// One power-of-two bucket of the allocation size histogram. The bucket covers +// sizes in [MinSize, MaxSize] inclusive (MaxSize = MinSize*2 - 1, or 0 for the +// zero-size bucket). Count and Bytes aggregate every alloc/realloc-alloc seen +// during the trace (not just currently-live allocations). +struct AllocSizeBucket +{ + uint64_t MinSize = 0; + uint64_t MaxSize = 0; + uint64_t Count = 0; + uint64_t Bytes = 0; +}; + +// -- Callstack data structures --------------------------------------------- + +// A single resolved stack frame. ModuleIndex references TraceModel::Modules; +// ~0u means the frame did not map to any loaded module. +struct ResolvedFrame +{ + uint64_t Address = 0; + uint32_t ModuleIndex = ~0u; + uint64_t Offset = 0; +}; + +// A decoded callstack: the ordered list of instruction-pointer frames +// captured at the point of an allocation (or free). +struct CallstackEntry +{ + uint32_t Id = 0; + eastl::vector<ResolvedFrame> Frames; // outermost (caller) first +}; + +// Per-callstack allocation churn statistics. "Churn" is measured by how +// quickly an allocation is freed — specifically, the number of alloc events +// that occur between the alloc and its matching free (event distance). +struct CallstackChurnStat +{ + uint32_t CallstackId = 0; + uint64_t ChurnAllocs = 0; // allocations freed within the distance threshold + uint64_t ChurnBytes = 0; // cumulative bytes of those short-lived allocations + uint64_t TotalAllocs = 0; // all allocations from this callstack (for context) + uint64_t TotalBytes = 0; + double MeanDistance = 0.0; // average event distance for the churny allocs +}; + +// Per-callstack live allocation statistics. +struct CallstackAllocStat +{ + uint32_t CallstackId = 0; + int64_t LiveBytes = 0; + uint32_t LiveCount = 0; + eastl::fixed_vector<uint32_t, 4, true> ThreadIds; // unique thread IDs that contributed allocations +}; + +// -- AllocationAnalyzer ---------------------------------------------------- + +// Subscribes to Memory.* trace events and tracks aggregate allocation +// statistics, a memory-over-time timeline, heap specs, and tag specs. +// Intended to be instantiated by BuildTraceModel alongside the other +// analyzers and registered with the Dispatcher. +class AllocationAnalyzer : public Analyzer +{ +public: + explicit AllocationAnalyzer(const TraceTiming* Timing); + + void subscribe(Vector<Subscription>& Subs) override; + + // -- Accessors (call after IterateTrace completes) -- + + bool Initialized() const { return m_Initialized; } + AllocationSummary Summary() const; + void EmitFinalSample(uint32_t TraceEndUs); + + eastl::vector<MemoryTimelineSample>& MutableTimeline() { return m_Timeline; } + const eastl::hash_map<uint32_t, HeapInfo>& Heaps() const { return m_Heaps; } + const eastl::hash_map<int32_t, TagInfo>& Tags() const { return m_Tags; } + const eastl::hash_map<uint8_t, HeapStat>& RootHeapStats() const { return m_RootHeapStats; } + + // Build per-callstack statistics from the current live allocation set. + eastl::vector<CallstackAllocStat> BuildCallstackStats() const; + + // Build per-callstack churn statistics sorted by churn alloc count descending. + // ChurnDistanceThreshold: allocations freed within this many alloc-events are + // considered "short-lived" / churny. + eastl::vector<CallstackChurnStat> BuildChurnStats(uint64_t ChurnDistanceThreshold = 1000) const; + + // Build a size-bucketed histogram of all observed allocations. Returns + // only populated buckets, ordered by MinSize ascending. + eastl::vector<AllocSizeBucket> BuildSizeHistogram() const; + +private: + // -- Event handlers -- + + void OnInit(const ::Memory_Init& Ev); + void OnMarker(const ::Memory_Marker& Ev); + void OnAlloc(const ::Memory_Alloc& Ev); + void OnAllocSystem(const ::Memory_AllocSystem& Ev); + void OnAllocVideo(const ::Memory_AllocVideo& Ev); + void OnFree(const ::Memory_Free& Ev); + void OnFreeSystem(const ::Memory_FreeSystem& Ev); + void OnFreeVideo(const ::Memory_FreeVideo& Ev); + void OnReallocAlloc(const ::Memory_ReallocAlloc& Ev); + void OnReallocAllocSystem(const ::Memory_ReallocAllocSystem& Ev); + void OnReallocFree(const ::Memory_ReallocFree& Ev); + void OnReallocFreeSystem(const ::Memory_ReallocFreeSystem& Ev); + void OnHeapSpec(const ::Memory_HeapSpec& Ev); + void OnHeapMarkAlloc(const ::Memory_HeapMarkAlloc& Ev); + void OnHeapUnmarkAlloc(const ::Memory_HeapUnmarkAlloc& Ev); + void OnTagSpec(const ::Memory_TagSpec& Ev); + + // -- Internal helpers -- + + struct LiveAlloc + { + uint64_t Size; + uint32_t CallstackId; + uint32_t ThreadId; + uint64_t EventSeq; // alloc event sequence number for churn distance + uint8_t RootHeap; + bool IsHeap = false; // true after HeapMarkAlloc; excluded from totals + }; + + uint64_t DecodeAllocSize(uint32_t RawSize, uint8_t AlignSizeLower) const; + void HandleAlloc(uint64_t Address, uint64_t Size, uint8_t RootHeap, uint32_t CallstackId, uint32_t ThreadId, bool IsRealloc); + void HandleFree(uint64_t Address, uint8_t RootHeap, uint32_t CallstackId, bool IsRealloc); + void MaybeEmitSample(uint32_t TimeUs); + + // -- State -- + + static constexpr uint32_t kTimelineSampleIntervalUs = 10'000; // 10ms + + const TraceTiming* m_Timing = nullptr; + + // Init params + uint8_t m_SizeShift = 3; // overridden by Memory.Init if present; 3 matches zencore's default + bool m_Initialized = false; + + // Live allocation map (address -> size + root heap) + eastl::hash_map<uint64_t, LiveAlloc> m_LiveAllocs; + + // Running byte counters + int64_t m_CurrentBytes = 0; + int64_t m_SystemBytes = 0; + int64_t m_VideoBytes = 0; + int64_t m_PeakBytes = 0; + uint32_t m_PeakTimeUs = 0; + + // Event counters + uint64_t m_TotalAllocs = 0; + uint64_t m_TotalFrees = 0; + uint64_t m_TotalReallocAllocs = 0; + uint64_t m_TotalReallocFrees = 0; + + // Timeline sampling + eastl::vector<MemoryTimelineSample> m_Timeline; + uint32_t m_LastSampleTimeUs = 0; + uint32_t m_LastMarkerTimeUs = 0; + bool m_HasReceivedMarker = false; + + // Per-callstack churn counters: total allocs + short-lived alloc stats + struct ChurnAccum + { + uint64_t TotalAllocs = 0; + uint64_t TotalBytes = 0; + uint64_t ChurnAllocs = 0; // freed within the distance threshold + uint64_t ChurnBytes = 0; + uint64_t ChurnDistanceSum = 0; // sum of event distances for churny allocs + }; + eastl::hash_map<uint32_t, ChurnAccum> m_ChurnByCallstack; + uint64_t m_AllocEventSeq = 0; // monotonic alloc event counter + + // Allocation size histogram: bucket i covers sizes [2^(i-1)+1, 2^i], with + // bucket 0 reserved for zero-size allocations. 65 buckets covers up to 2^64. + static constexpr size_t kSizeHistogramBuckets = 65; + struct SizeBucketAccum + { + uint64_t Count = 0; + uint64_t Bytes = 0; + }; + SizeBucketAccum m_SizeHistogram[kSizeHistogramBuckets] = {}; + + // Metadata + eastl::hash_map<uint32_t, HeapInfo> m_Heaps; + eastl::hash_map<int32_t, TagInfo> m_Tags; + eastl::hash_map<uint8_t, HeapStat> m_RootHeapStats; +}; + +// -- CallstackAnalyzer ----------------------------------------------------- + +// Subscribes to Memory.CallstackSpec* trace events, decodes compressed +// frames, and stores a callstack ID -> frame addresses mapping. Frame +// addresses are raw instruction pointers; resolution to module+offset +// happens in BuildTraceModel post-processing. +class CallstackAnalyzer : public Analyzer +{ +public: + void subscribe(Vector<Subscription>& Subs) override; + + const eastl::hash_map<uint32_t, eastl::vector<uint64_t>>& RawCallstacks() const { return m_Callstacks; } + +private: + void OnCallstackSpec(const ::Memory_CallstackSpec& Ev); + void OnCallstackSpecDeltaVarInt(const ::Memory_CallstackSpecDeltaVarInt& Ev); + void OnCallstackSpecDelta7bit(const ::Memory_CallstackSpecDelta7bit& Ev); + void OnCallstackSpecXORAndRLE(const ::Memory_CallstackSpecXORAndRLE& Ev); + + void StoreCallstack(uint32_t Id, const uint64_t* Frames, size_t Count); + + eastl::hash_map<uint32_t, eastl::vector<uint64_t>> m_Callstacks; +}; + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_model.cpp b/src/zen/trace/trace_model.cpp new file mode 100644 index 000000000..ac81161a1 --- /dev/null +++ b/src/zen/trace/trace_model.cpp @@ -0,0 +1,3898 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_model.h" + +#include <zencore/basicfile.h> +#include <zencore/except_fmt.h> +#include <zencore/fmtutils.h> +#include <zencore/intmath.h> +#include <zencore/logging.h> +#include <zencore/logging/tracelog.h> +#include <zencore/scopeguard.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <zencore/timer.h> +#include <zenutil/parallelsort.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/hash_map.h> +#include <EASTL/map.h> +#include <EASTL/set.h> +#include <EASTL/sort.h> +#include <EASTL/vector.h> +#include <analysis/analyzer.h> +#include <analysis/dispatcher.h> +#include <fmt/format.h> +#include <trace/trace.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <algorithm> +#include <cmath> +#include <cstring> + +using namespace std::literals; + +// Toggle to A/B test cross-platform parallel sort vs sequential eastl::sort. +constexpr bool kUseParallelSort = true; + +namespace eastl { + +template<> +struct hash<std::string> +{ + size_t operator()(const std::string& S) const { return eastl::hash<const char*>()(S.c_str()); } +}; + +} // namespace eastl + +////////////////////////////////////////////////////////////////////////////// +// Trace analysis types (global namespace alongside tourist types) + +namespace { + +using zen::ReciprocalU64; + +// Welford's online algorithm for computing mean and standard deviation +class Distribution +{ +public: + void add(double X) + { + m_Count++; + if (m_Count == 1) + { + m_OldM = m_NewM = X; + m_OldS = 0.0; + } + else + { + m_NewM = m_OldM + (X - m_OldM) / double(m_Count); + m_NewS = m_OldS + (X - m_OldM) * (X - m_NewM); + m_OldM = m_NewM; + m_OldS = m_NewS; + } + } + + uint32_t Count() const { return m_Count; } + double Mean() const { return (m_Count > 0) ? m_NewM : 0.0; } + double Variance() const { return (m_Count > 1) ? m_NewS / double(m_Count - 1) : 0.0; } + double StdDev() const { return std::sqrt(Variance()); } + +private: + double m_OldM = 0.0; + double m_NewM = 0.0; + double m_OldS = 0.0; + double m_NewS = 0.0; + uint32_t m_Count = 0; +}; + +class NameDepot +{ +public: + uint64 Add(StringView Name) + { + uint64 NameHash = Hash(Name); + Add(NameHash, Name); + return NameHash; + } + + void Add(uint64 NameHash, StringView Name) + { + if (auto It = m_Names.insert({NameHash, String()}); It.second) + { + It.first->second = Name; + } + } + + StringView Get(uint64 NameHash) const + { + auto Iter = m_Names.find(NameHash); + if (Iter == m_Names.end()) + { + return "???"; + } + return Iter->second; + } + +private: + eastl::hash_map<uint64, String> m_Names; +}; + +struct CpuEventStat +{ + Distribution Dist; + uint32_t Min = ~0u; + uint32_t Max = 0; +}; + +class EventStats +{ +public: + void Record(uint64 NameHash, uint32 DurationUs) + { + CpuEventStat& Stat = m_Stats[NameHash]; + Stat.Min = std::min(Stat.Min, DurationUs); + Stat.Max = std::max(Stat.Max, DurationUs); + Stat.Dist.add(double(DurationUs)); + } + + auto begin() const { return m_Stats.begin(); } + auto end() const { return m_Stats.end(); } + bool empty() const { return m_Stats.empty(); } + +private: + eastl::hash_map<uint64, CpuEventStat> m_Stats; +}; + +////////////////////////////////////////////////////////////////////////////// +// Event outlines + +// clang-format off +begin_outline($Trace, NewTrace) + field(uint64, CycleFrequency) + field(uint64, StartCycle) +end_outline() + +begin_outline(CpuProfiler, EventSpec) + field(uint32, Id) + field(FieldStr, Name) +end_outline() + +begin_outline(CpuProfiler, EventBatch) + field(uint32, ThreadId) + field(uint8[], Data) +end_outline() + +begin_outline(CpuProfiler, EventBatchV2) + field(uint8[], Data) +end_outline() + +begin_outline(CpuProfiler, EventBatchV3) + field(uint8[], Data) +end_outline() + +begin_outline(CpuProfiler, Metadata) + field(uint32, Id) + field(uint32, SpecId) + field(uint8[], Metadata) +end_outline() + +begin_outline($Trace, ThreadInfo) + field(FieldStr, Name) + field(int32, SortHint) + field(uint32, ThreadId) + field(uint32, SystemId) +end_outline() + +begin_outline($Trace, ThreadGroupBegin) + field(FieldStr, Name) +end_outline() + +begin_outline($Trace, ThreadGroupEnd) +end_outline() + +begin_outline(Diagnostics, Session2) + field(FieldStr, Platform) + field(FieldStr, AppName) + field(FieldStr, ProjectName) + field(FieldStr, CommandLine) + field(FieldStr, Branch) + field(FieldStr, BuildVersion) + field(uint32, Changelist) + field(uint8, ConfigurationType) + field(uint8, TargetType) +end_outline() + +begin_outline(Diagnostics, ModuleInit) + field(FieldStr, SymbolFormat) + field(uint8, ModuleBaseShift) +end_outline() + +begin_outline(Diagnostics, ModuleLoad) + field(FieldStr, Name) + field(uint64, Base) + field(uint32, Size) + field(uint8[], ImageId) +end_outline() + +begin_outline(Diagnostics, ModuleUnload) + field(uint64, Base) +end_outline() + +begin_outline(Trace, ChannelAnnounce) + field(uint32, Id) + field(uint8, IsEnabled) + field(uint8, ReadOnly) + field(FieldStr, Name) +end_outline() + +begin_outline(Trace, ChannelToggle) + field(uint32, Id) + field(uint8, IsEnabled) +end_outline() + +begin_outline(Logging, LogCategory) + field(uint64, CategoryPointer) + field(uint8, DefaultVerbosity) + field(FieldStr, Name) +end_outline() + +begin_outline(Logging, LogMessageSpec) + field(uint64, LogPoint) + field(uint64, CategoryPointer) + field(int32, Line) + field(uint8, Verbosity) + field(FieldStr, FileName) + field(FieldStr, FormatString) +end_outline() + +begin_outline(Logging, LogMessage) + field(uint64, LogPoint) + field(uint64, Cycle) + field(uint8[], FormatArgs) +end_outline() + +// Analyzer-level schema for the zencore log events defined in +// src/zencore/logging/tracelog.cpp. The field layout mirrors the Logging.* +// outline above so the analyzer plumbing is symmetric, but the events carry +// fmt-style `{}` format strings instead of printf-style `%` specifiers and +// the FormatArgs blob uses a zen-specific descriptor encoding. +begin_outline(ZenLog, Category) + field(uint64, CategoryPointer) + field(uint8, DefaultVerbosity) + field(FieldStr, Name) +end_outline() + +begin_outline(ZenLog, MessageSpec) + field(uint64, LogPoint) + field(uint64, CategoryPointer) + field(int32, Line) + field(uint8, Verbosity) + field(FieldStr, FileName) + field(FieldStr, FormatString) +end_outline() + +begin_outline(ZenLog, Message) + field(uint64, LogPoint) + field(uint64, Cycle) + field(uint8[], FormatArgs) +end_outline() + +begin_outline(Misc, BookmarkSpec) + field(uint64, BookmarkPoint) + field(int32, Line) + field(FieldStr, FormatString) + field(FieldStr, FileName) +end_outline() + +begin_outline(Misc, Bookmark) + field(uint64, Cycle) + field(uint64, BookmarkPoint) + field(uint8[], FormatArgs) +end_outline() + +begin_outline(Misc, RegionBegin) + field(uint64, Cycle) + field(uint8[], RegionName) + field(uint8[], Category) +end_outline() + +begin_outline(Misc, RegionBeginWithId) + field(uint64, CycleAndId) + field(uint8[], RegionName) + field(uint8[], Category) +end_outline() + +begin_outline(Misc, RegionEnd) + field(uint64, Cycle) + field(uint8[], RegionName) +end_outline() + +begin_outline(Misc, RegionEndWithId) + field(uint64, Cycle) + field(uint64, RegionId) +end_outline() + +// CsvProfiler events +begin_outline(CsvProfiler, RegisterCategory) + field(int32, Index) + field(uint8[], Name) +end_outline() + +begin_outline(CsvProfiler, DefineInlineStat) + field(uint64, StatId) + field(int32, CategoryIndex) + field(uint8[], Name) +end_outline() + +begin_outline(CsvProfiler, DefineDeclaredStat) + field(uint64, StatId) + field(int32, CategoryIndex) + field(uint8[], Name) +end_outline() + +begin_outline(CsvProfiler, BeginStat) + field(uint64, StatId) + field(uint64, Cycle) +end_outline() + +begin_outline(CsvProfiler, EndStat) + field(uint64, StatId) + field(uint64, Cycle) +end_outline() + +begin_outline(CsvProfiler, BeginExclusiveStat) + field(uint64, StatId) + field(uint64, Cycle) +end_outline() + +begin_outline(CsvProfiler, EndExclusiveStat) + field(uint64, StatId) + field(uint64, Cycle) +end_outline() + +begin_outline(CsvProfiler, CustomStatInt) + field(uint64, StatId) + field(uint64, Cycle) + field(int32, Value) + field(uint8, OpType) +end_outline() + +begin_outline(CsvProfiler, CustomStatFloat) + field(uint64, StatId) + field(uint64, Cycle) + field(float, Value) + field(uint8, OpType) +end_outline() + +begin_outline(CsvProfiler, Event) + field(uint64, Cycle) + field(int32, CategoryIndex) + field(uint8[], Text) +end_outline() + +begin_outline(CsvProfiler, BeginCapture) + field(uint64, Cycle) + field(uint32, RenderThreadId) + field(uint32, RHIThreadId) + field(uint8, EnableCounts) + field(uint8[], FileName) +end_outline() + +begin_outline(CsvProfiler, EndCapture) + field(uint64, Cycle) +end_outline() + +begin_outline(CsvProfiler, Metadata) + field(uint8[], Key) + field(uint8[], Value) +end_outline() + // clang-format on + + ////////////////////////////////////////////////////////////////////////////// + // Forward declarations needed by the helper analyzers below. + + using zen::trace_detail::SafeFieldStr; + +////////////////////////////////////////////////////////////////////////////// +// Minimal CBOR formatter +// +// CpuProfiler.Metadata payloads are CBOR-encoded (RFC 7049) blobs produced +// by UE's FCborWriter. We don't need a full decoder -- we just walk the +// bytes and append human-readable values to an output string. Handles the +// subset actually emitted by UE's metadata scopes: unsigned / negative +// integers, byte / text strings, arrays, maps, floats, and the boolean / +// null simple values. + +static bool CborAppendValue(const uint8*& p, const uint8* end, std::string& out, int depth); + +static bool +CborReadArg(const uint8*& p, const uint8* end, uint8 info, uint64& value) +{ + if (info < 24) + { + value = info; + return true; + } + if (info == 24) + { + if (end - p < 1) + return false; + value = *p++; + return true; + } + if (info == 25) + { + if (end - p < 2) + return false; + value = (uint64(p[0]) << 8) | p[1]; + p += 2; + return true; + } + if (info == 26) + { + if (end - p < 4) + return false; + value = (uint64(p[0]) << 24) | (uint64(p[1]) << 16) | (uint64(p[2]) << 8) | p[3]; + p += 4; + return true; + } + if (info == 27) + { + if (end - p < 8) + return false; + value = 0; + for (int i = 0; i < 8; ++i) + { + value = (value << 8) | p[i]; + } + p += 8; + return true; + } + return false; +} + +static bool +CborAppendValue(const uint8*& p, const uint8* end, std::string& out, int depth) +{ + if (depth > 4 || p >= end) + { + return false; + } + + const uint8 ib = *p++; + const uint8 major = ib >> 5; + const uint8 info = ib & 0x1f; + + switch (major) + { + case 0: // unsigned integer + { + uint64 v; + if (!CborReadArg(p, end, info, v)) + return false; + out += fmt::format("{}", v); + return true; + } + case 1: // negative integer: -1 - value + { + uint64 v; + if (!CborReadArg(p, end, info, v)) + return false; + out += fmt::format("{}", -1 - int64_t(v)); + return true; + } + case 2: // byte string + case 3: // text string + { + uint64 len; + if (!CborReadArg(p, end, info, len)) + return false; + if (len > uint64(end - p)) + return false; + out.append(reinterpret_cast<const char*>(p), size_t(len)); + p += len; + return true; + } + case 4: // array + { + uint64 count; + if (!CborReadArg(p, end, info, count)) + return false; + for (uint64 i = 0; i < count; ++i) + { + if (i > 0) + out += ", "; + if (!CborAppendValue(p, end, out, depth + 1)) + return false; + } + return true; + } + case 5: // map + { + uint64 count; + if (!CborReadArg(p, end, info, count)) + return false; + for (uint64 i = 0; i < count; ++i) + { + if (i > 0) + out += ", "; + if (!CborAppendValue(p, end, out, depth + 1)) + return false; + out += "="; + if (!CborAppendValue(p, end, out, depth + 1)) + return false; + } + return true; + } + case 7: // simple values / floats + { + if (info == 20) + { + out += "false"; + return true; + } + if (info == 21) + { + out += "true"; + return true; + } + if (info == 22) + { + out += "null"; + return true; + } + if (info == 26) + { + if (end - p < 4) + return false; + uint32 bits = (uint32(p[0]) << 24) | (uint32(p[1]) << 16) | (uint32(p[2]) << 8) | p[3]; + float v; + std::memcpy(&v, &bits, 4); + out += fmt::format("{}", v); + p += 4; + return true; + } + if (info == 27) + { + if (end - p < 8) + return false; + uint64 bits = 0; + for (int i = 0; i < 8; ++i) + { + bits = (bits << 8) | p[i]; + } + p += 8; + double v; + std::memcpy(&v, &bits, 8); + out += fmt::format("{}", v); + return true; + } + return false; + } + default: + return false; + } +} + +static std::string +FormatMetadataValues(const uint8_t* Bytes, size_t Size) +{ + std::string out; + if (Size == 0) + { + return out; + } + const uint8* p = Bytes; + CborAppendValue(p, Bytes + Size, out, 0); + return out; +} + +using zen::trace_detail::TraceTiming; + +////////////////////////////////////////////////////////////////////////////// +// Metadata registry +// +// Subscribes to CpuProfiler.Metadata events and stores each payload's +// CBOR-encoded bytes keyed by MetadataId, along with the SpecId they +// reference. Both CpuAnalyzer and TimelineAnalyzer query the registry when +// they encounter a V3 scope with the metadata bit set so the scope can be +// rendered as `{base name} - {formatted values}`. + +struct MetadataEntry +{ + uint32_t SpecId = 0; + eastl::vector<uint8_t> Bytes; +}; + +class MetadataRegistry : public Analyzer +{ +public: + void subscribe(Vector<Subscription>& Subs) override { Subs.emplace_back(this, &MetadataRegistry::OnMetadata); } + + const MetadataEntry* Lookup(uint32_t MetadataId) const + { + auto It = m_Entries.find(MetadataId); + return (It != m_Entries.end()) ? &It->second : nullptr; + } + +private: + void OnMetadata(const CpuProfiler_Metadata& Ev) + { + uint32_t MetadataId = Ev.Id(); + uint32_t SpecId = Ev.SpecId(); + Array<uint8[]> Data = Ev.Metadata(); + + MetadataEntry& Entry = m_Entries[MetadataId]; + Entry.SpecId = SpecId; + Entry.Bytes.assign(Data.get(), Data.get() + Data.get_size()); + } + + eastl::hash_map<uint32_t, MetadataEntry> m_Entries; +}; + +////////////////////////////////////////////////////////////////////////////// +// Log message formatting (upstream UE Logging.* wire) +// +// UE's trace emits log messages as a sequence of typed arguments that need +// to be substituted into a printf-style format string. The wire format is: +// +// [ArgumentCount: uint8] +// [Descriptors: uint8 * ArgumentCount] // each byte = category | size +// [Payload: bytes] +// +// Category bits live in the upper 2 bits (shifted by FormatArgTypeCode_- +// CategoryBitShift == 6): Integer=1, Float=2, String=3. The low 6 bits are +// the argument size in bytes; for strings the size is the per-character +// width (1 == ANSI, 2 == UTF-16). +// +// We walk the format string, extract each specifier, pull the matching arg +// from the stream and hand both to std::snprintf. Width/precision stars +// (e.g. "%*.*f") are not supported; they're rare in log formats. +// +// Note: this is the upstream UE printf-style wire only. The zen-specific +// ZenLog.* events use a different descriptor encoding (3-bit category / +// 5-bit size, plus a dedicated bool and pointer category) and a different +// format-spec grammar. See zen::logging::FormatLogArgs in +// src/zencore/logging/tracelog.cpp for that path. + +struct FormatArgStream +{ + const uint8_t* Descriptors; + const uint8_t* Payload; + uint8_t Remaining; + + bool HasNext() const { return Remaining > 0; } + + uint8_t PeekCategory() const { return (*Descriptors) & 0xC0; } + uint8_t PeekSize() const { return (*Descriptors) & 0x3F; } + + void Advance(size_t PayloadBytes) + { + Payload += PayloadBytes; + ++Descriptors; + --Remaining; + } +}; + +static bool +InitFormatArgStream(FormatArgStream& Ctx, const uint8_t* Data, size_t Size) +{ + if (!Data || Size == 0) + { + Ctx.Remaining = 0; + return false; + } + uint8_t Count = Data[0]; + if (size_t(1) + Count > Size) + { + Ctx.Remaining = 0; + return false; + } + Ctx.Descriptors = Data + 1; + Ctx.Payload = Data + 1 + Count; + Ctx.Remaining = Count; + return true; +} + +static bool +IsPrintfSpecifierChar(char c) +{ + switch (c) + { + case 'd': + case 'i': + case 'u': + case 'o': + case 'x': + case 'X': + case 'c': + case 'p': + case 'f': + case 'F': + case 'e': + case 'E': + case 'g': + case 'G': + case 'a': + case 'A': + case 's': + case 'S': + case 'n': + return true; + default: + return false; + } +} + +static std::string +FormatLogMessage(std::string_view Format, const uint8_t* ArgsData, size_t ArgsSize) +{ + FormatArgStream Stream{}; + InitFormatArgStream(Stream, ArgsData, ArgsSize); + + std::string Out; + Out.reserve(Format.size() + 32); + + size_t i = 0; + while (i < Format.size()) + { + char c = Format[i]; + if (c != '%') + { + Out.push_back(c); + ++i; + continue; + } + + // Handle "%%" -> literal percent. + if (i + 1 < Format.size() && Format[i + 1] == '%') + { + Out.push_back('%'); + i += 2; + continue; + } + + // Walk the specifier until we find a terminating character. + size_t SpecStart = i++; + while (i < Format.size() && !IsPrintfSpecifierChar(Format[i])) + { + ++i; + } + if (i >= Format.size()) + { + // Truncated specifier -- copy the remainder literally. + Out.append(Format.substr(SpecStart)); + break; + } + + char Specifier = Format[i++]; + std::string Spec(Format.substr(SpecStart, i - SpecStart)); + + if (!Stream.HasNext()) + { + // Not enough arguments: emit the raw specifier so the user can + // at least tell something is missing. + Out.append(Spec); + continue; + } + + const uint8_t Category = Stream.PeekCategory(); + const uint8_t Size = Stream.PeekSize(); + + char Buf[512]; + Buf[0] = '\0'; + + if (Category == 0x40) // integer + { + uint64_t Raw = 0; + if (Size <= sizeof(Raw) && Size > 0) + { + std::memcpy(&Raw, Stream.Payload, Size); + } + + // Route through the correct snprintf type based on the + // specifier. Cast to int64_t for signed integer specifiers. + switch (Specifier) + { + case 'd': + case 'i': + { + // Sign-extend based on Size. + int64_t Signed = 0; + switch (Size) + { + case 1: + Signed = int8_t(Raw & 0xff); + break; + case 2: + Signed = int16_t(Raw & 0xffff); + break; + case 4: + Signed = int32_t(Raw & 0xffffffff); + break; + case 8: + Signed = int64_t(Raw); + break; + default: + Signed = int64_t(Raw); + break; + } + // Replace length modifier so snprintf interprets the + // correctly-sized value. Simplest: append "ll". + std::string AdjustedSpec = Spec; + AdjustedSpec.insert(AdjustedSpec.size() - 1, "ll"); + std::snprintf(Buf, sizeof(Buf), AdjustedSpec.c_str(), static_cast<long long>(Signed)); + break; + } + case 'u': + case 'o': + case 'x': + case 'X': + case 'p': + { + std::string AdjustedSpec = Spec; + AdjustedSpec.insert(AdjustedSpec.size() - 1, "ll"); + std::snprintf(Buf, sizeof(Buf), AdjustedSpec.c_str(), static_cast<unsigned long long>(Raw)); + break; + } + case 'c': + { + std::snprintf(Buf, sizeof(Buf), Spec.c_str(), int(Raw & 0xff)); + break; + } + default: + std::snprintf(Buf, sizeof(Buf), "%llu", static_cast<unsigned long long>(Raw)); + break; + } + Stream.Advance(Size); + } + else if (Category == 0x80) // floating point + { + double Value = 0.0; + if (Size == 4) + { + float F; + std::memcpy(&F, Stream.Payload, 4); + Value = double(F); + } + else if (Size == 8) + { + std::memcpy(&Value, Stream.Payload, 8); + } + std::snprintf(Buf, sizeof(Buf), Spec.c_str(), Value); + Stream.Advance(Size); + } + else if (Category == 0xC0) // string + { + std::string Tmp; + if (Size == 1) + { + const char* S = reinterpret_cast<const char*>(Stream.Payload); + size_t Len = std::strlen(S); + Tmp.assign(S, Len); + std::snprintf(Buf, sizeof(Buf), Spec.c_str(), Tmp.c_str()); + Stream.Advance(Len + 1); + } + else if (Size == 2) + { + const char16_t* W = reinterpret_cast<const char16_t*>(Stream.Payload); + size_t Len = 0; + while (W[Len] != 0) + ++Len; + Tmp.reserve(Len); + for (size_t k = 0; k < Len; ++k) + { + char16_t ch = W[k]; + Tmp.push_back(ch < 0x80 ? char(ch) : '?'); + } + std::snprintf(Buf, sizeof(Buf), Spec.c_str(), Tmp.c_str()); + Stream.Advance((Len + 1) * 2); + } + else + { + std::snprintf(Buf, sizeof(Buf), "<unsupported string width %u>", unsigned(Size)); + Stream.Advance(0); + ++Stream.Descriptors; + --Stream.Remaining; + } + } + else + { + std::snprintf(Buf, sizeof(Buf), "<arg>"); + Stream.Advance(Size); + } + + Out.append(Buf); + } + + return Out; +} + +////////////////////////////////////////////////////////////////////////////// +// Log analyzer + +class LogAnalyzer : public Analyzer +{ +public: + explicit LogAnalyzer(const TraceTiming* Timing = nullptr) : m_Timing(Timing) {} + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &LogAnalyzer::OnLogCategory); + Subs.emplace_back(this, &LogAnalyzer::OnLogMessageSpec); + Subs.emplace_back(this, &LogAnalyzer::OnLogMessage); + Subs.emplace_back(this, &LogAnalyzer::OnZenLogCategory); + Subs.emplace_back(this, &LogAnalyzer::OnZenLogMessageSpec); + Subs.emplace_back(this, &LogAnalyzer::OnZenLogMessage); + } + + eastl::vector<zen::trace_detail::LogCategoryInfo> BuildCategories(eastl::hash_map<uint64_t, uint32_t>& OutPointerToIndex) const + { + eastl::vector<zen::trace_detail::LogCategoryInfo> Cats; + Cats.reserve(m_Categories.size()); + OutPointerToIndex.clear(); + for (const auto& [Ptr, Info] : m_Categories) + { + OutPointerToIndex[Ptr] = uint32_t(Cats.size()); + Cats.push_back(Info); + } + return Cats; + } + + const eastl::vector<zen::trace_detail::LogEntry>& Entries() const { return m_Entries; } + eastl::vector<zen::trace_detail::LogEntry>& MutableEntries() { return m_Entries; } + + // The shared TraceTiming pointer lets external callers read the trace's + // cycle base / frequency without each analyzer having to own a copy. + const TraceTiming* Timing() const { return m_Timing; } + + struct MessageSpec + { + uint64_t CategoryPointer = 0; + int32_t Line = 0; + uint8_t Verbosity = 0; + std::string File; + std::string FormatString; + }; + + const eastl::hash_map<uint64_t, MessageSpec>& MessageSpecs() const { return m_Specs; } + +private: + // Both Logging.* and ZenLog.* use identical event fields; the difference is + // only how their respective FormatArgs get rendered (printf vs fmt), which + // is decided at message-emit time. + template<typename CategoryEvent> + void IngestCategory(const CategoryEvent& Ev) + { + uint64_t Ptr = Ev.CategoryPointer(); + zen::trace_detail::LogCategoryInfo& Info = m_Categories[Ptr]; + Info.Name = SafeFieldStr(Ev.Name()); + Info.DefaultVerbosity = Ev.DefaultVerbosity(); + } + + template<typename SpecEvent> + void IngestSpec(const SpecEvent& Ev) + { + MessageSpec& Spec = m_Specs[Ev.LogPoint()]; + Spec.CategoryPointer = Ev.CategoryPointer(); + Spec.Line = Ev.Line(); + Spec.Verbosity = Ev.Verbosity(); + Spec.File = SafeFieldStr(Ev.FileName()); + Spec.FormatString = SafeFieldStr(Ev.FormatString()); + } + + void OnLogCategory(const Logging_LogCategory& Ev) { IngestCategory(Ev); } + void OnLogMessageSpec(const Logging_LogMessageSpec& Ev) { IngestSpec(Ev); } + void OnLogMessage(const Logging_LogMessage& Ev) { EmitEntry(Ev.LogPoint(), Ev.Cycle(), Ev.FormatArgs(), /*IsZenLog*/ false); } + + // ZenLog.* shares the same MessageSpec table. LogPoint pointers are emitted + // by distinct processes (zenserver vs. a hypothetical UE trace we imported) + // so the two keyspaces don't collide in practice. + void OnZenLogCategory(const ZenLog_Category& Ev) { IngestCategory(Ev); } + void OnZenLogMessageSpec(const ZenLog_MessageSpec& Ev) { IngestSpec(Ev); } + void OnZenLogMessage(const ZenLog_Message& Ev) { EmitEntry(Ev.LogPoint(), Ev.Cycle(), Ev.FormatArgs(), /*IsZenLog*/ true); } + + void EmitEntry(uint64_t LogPoint, uint64_t Cycle, Array<uint8[]> Args, bool IsZenLog) + { + auto SpecIt = m_Specs.find(LogPoint); + if (SpecIt == m_Specs.end()) + { + return; + } + const MessageSpec& Spec = SpecIt->second; + + uint32_t TimeUs = m_Timing ? m_Timing->CycleToTimeUs(Cycle) : 0; + + std::string Msg = IsZenLog ? zen::logging::FormatLogArgs(std::string_view(Spec.FormatString), Args.get(), Args.get_size()) + : FormatLogMessage(std::string_view(Spec.FormatString), Args.get(), Args.get_size()); + + zen::trace_detail::LogEntry Entry; + Entry.TimeUs = TimeUs; + Entry.Verbosity = Spec.Verbosity; + Entry.Line = Spec.Line; + Entry.File = Spec.File; + Entry.Message = std::move(Msg); + // Use the category pointer temporarily so BuildTraceModel can resolve + // it against the categories table. + Entry.CategoryIndex = SpecToCategoryIndex(Spec.CategoryPointer); + m_Entries.push_back(std::move(Entry)); + } + + uint32_t SpecToCategoryIndex(uint64_t Ptr) + { + // Encoded pointer stuffed into uint32_t so BuildTraceModel can remap. + // Lossy but deterministic: use a stable sequential index per unique + // pointer so we never need the full 64-bit value beyond build time. + auto It = m_CategoryIndex.find(Ptr); + if (It != m_CategoryIndex.end()) + { + return It->second; + } + uint32_t Idx = uint32_t(m_CategoryIndex.size()); + m_CategoryIndex[Ptr] = Idx; + return Idx; + } + +public: + // Mapping from the intermediate index stored in LogEntry::CategoryIndex + // during capture to the real category pointer; BuildTraceModel uses + // this to remap entries against the flattened LogCategories array. + const eastl::hash_map<uint64_t, uint32_t>& CategoryPointerIndex() const { return m_CategoryIndex; } + +private: + const TraceTiming* m_Timing = nullptr; + eastl::hash_map<uint64_t, zen::trace_detail::LogCategoryInfo> m_Categories; + eastl::hash_map<uint64_t, MessageSpec> m_Specs; + eastl::hash_map<uint64_t, uint32_t> m_CategoryIndex; + eastl::vector<zen::trace_detail::LogEntry> m_Entries; +}; + +////////////////////////////////////////////////////////////////////////////// +// Bookmarks and regions +// +// UE's bookmark wire format mirrors LogMessage: a BookmarkSpec introduces +// a (FileName, Line, FormatString) triple keyed by a BookmarkPoint pointer, +// and each Misc.Bookmark event carries that pointer, a cycle, and the same +// FFormatArgsTrace payload the log pipeline already knows how to decode. +// Region events come in two flavours: the legacy name-paired +// RegionBegin/RegionEnd and the newer *WithId variants that pack a unique +// id into the begin event's cycle. + +class BookmarksAnalyzer : public Analyzer +{ +public: + explicit BookmarksAnalyzer(const TraceTiming* Timing = nullptr) : m_Timing(Timing) {} + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &BookmarksAnalyzer::OnBookmarkSpec); + Subs.emplace_back(this, &BookmarksAnalyzer::OnBookmark); + Subs.emplace_back(this, &BookmarksAnalyzer::OnRegionBegin); + Subs.emplace_back(this, &BookmarksAnalyzer::OnRegionBeginWithId); + Subs.emplace_back(this, &BookmarksAnalyzer::OnRegionEnd); + Subs.emplace_back(this, &BookmarksAnalyzer::OnRegionEndWithId); + } + + eastl::vector<zen::trace_detail::Bookmark>& MutableBookmarks() { return m_Bookmarks; } + eastl::vector<zen::trace_detail::RegionEntry>& MutableRegions() { return m_Regions; } + +private: + struct BookmarkSpec + { + int32_t Line = 0; + std::string File; + std::string FormatString; + }; + + uint32_t CycleToTimeUs(uint64_t Cycle) const { return m_Timing ? m_Timing->CycleToTimeUs(Cycle) : 0; } + + void OnBookmarkSpec(const Misc_BookmarkSpec& Ev) + { + BookmarkSpec& Spec = m_Specs[Ev.BookmarkPoint()]; + Spec.Line = Ev.Line(); + Spec.File = SafeFieldStr(Ev.FileName()); + Spec.FormatString = SafeFieldStr(Ev.FormatString()); + } + + void OnBookmark(const Misc_Bookmark& Ev) + { + auto SpecIt = m_Specs.find(Ev.BookmarkPoint()); + if (SpecIt == m_Specs.end()) + { + return; + } + const BookmarkSpec& Spec = SpecIt->second; + + Array<uint8[]> Args = Ev.FormatArgs(); + std::string Text = FormatLogMessage(std::string_view(Spec.FormatString), Args.get(), Args.get_size()); + + zen::trace_detail::Bookmark Out; + Out.TimeUs = CycleToTimeUs(Ev.Cycle()); + Out.Line = Spec.Line; + Out.File = Spec.File; + Out.Text = std::move(Text); + m_Bookmarks.push_back(std::move(Out)); + } + + uint32_t CreatePartialRegion(uint32_t TimeUs, std::string Name, std::string Category) + { + zen::trace_detail::RegionEntry Entry; + Entry.BeginUs = TimeUs; + Entry.EndUs = ~uint32_t(0); // sentinel: still open + Entry.Depth = 0; + Entry.Reserved = 0; + Entry.Name = std::move(Name); + Entry.Category = std::move(Category); + uint32_t Idx = uint32_t(m_Regions.size()); + m_Regions.push_back(std::move(Entry)); + return Idx; + } + + // Decodes the raw array bytes of a RegionName field into a std::string. + // UE emits RegionName as either AnsiString (1-byte) or WideString (2-byte) + // depending on the trace's age -- for the 2-byte case we do the same + // lossy ASCII fold tourist's FieldStr does, which is all we need for + // display. + static std::string DecodeRegionName(const Array<uint8[]>& Data) + { + const uint8_t* p = Data.get(); + size_t size = Data.get_size(); + uint32_t count = Data.get_count(); + if (!p || size == 0 || count == 0) + { + return {}; + } + if (size == count) + { + // 1 byte per element -- AnsiString. + return std::string(reinterpret_cast<const char*>(p), count); + } + if (size == count * 2) + { + // 2 bytes per element -- WideString. Lossy ASCII fold. + std::string out; + out.reserve(count); + const char16_t* w = reinterpret_cast<const char16_t*>(p); + for (uint32_t i = 0; i < count; ++i) + { + out.push_back(w[i] < 0x80 ? char(w[i]) : '?'); + } + return out; + } + return {}; + } + + void OnRegionBegin(const Misc_RegionBegin& Ev) + { + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + Array<uint8[]> NameArr = Ev.RegionName(); + std::string Name = DecodeRegionName(NameArr); + std::string Category = DecodeRegionName(Ev.Category()); + uint32_t Idx = CreatePartialRegion(TimeUs, Name, std::move(Category)); + m_OpenByName[Name].push_back(Idx); + } + + void OnRegionBeginWithId(const Misc_RegionBeginWithId& Ev) + { + // Despite its name, CycleAndId is just Cycles64() -- a plain 64-bit + // cycle count that doubles as a unique region identifier. The caller + // keeps the returned value and passes it back as RegionId at end. + uint64_t CycleAndId = Ev.CycleAndId(); + uint32_t TimeUs = CycleToTimeUs(CycleAndId); + Array<uint8[]> NameArr = Ev.RegionName(); + std::string Name = DecodeRegionName(NameArr); + std::string Category = DecodeRegionName(Ev.Category()); + uint32_t Idx = CreatePartialRegion(TimeUs, std::move(Name), std::move(Category)); + m_OpenById[CycleAndId] = Idx; + } + + void OnRegionEnd(const Misc_RegionEnd& Ev) + { + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + Array<uint8[]> NameArr = Ev.RegionName(); + std::string Name = DecodeRegionName(NameArr); + auto It = m_OpenByName.find(Name); + if (It == m_OpenByName.end() || It->second.empty()) + { + return; + } + uint32_t Idx = It->second.back(); + It->second.pop_back(); + m_Regions[Idx].EndUs = TimeUs; + } + + void OnRegionEndWithId(const Misc_RegionEndWithId& Ev) + { + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + uint64_t Id = Ev.RegionId(); + auto It = m_OpenById.find(Id); + if (It == m_OpenById.end()) + { + return; + } + m_Regions[It->second].EndUs = TimeUs; + m_OpenById.erase(It); + } + + const TraceTiming* m_Timing = nullptr; + eastl::hash_map<uint64_t, BookmarkSpec> m_Specs; + eastl::vector<zen::trace_detail::Bookmark> m_Bookmarks; + eastl::vector<zen::trace_detail::RegionEntry> m_Regions; + eastl::hash_map<std::string, eastl::vector<uint32_t>> m_OpenByName; + eastl::hash_map<uint64_t, uint32_t> m_OpenById; +}; + +////////////////////////////////////////////////////////////////////////////// +// CsvProfiler analyzer -- parses CSV stat categories, definitions, timing, +// custom values, events, capture markers, and metadata. + +class CsvProfilerAnalyzer : public Analyzer +{ +public: + explicit CsvProfilerAnalyzer(const TraceTiming* Timing = nullptr) : m_Timing(Timing) {} + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnRegisterCategory); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnDefineInlineStat); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnDefineDeclaredStat); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnBeginStat); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnEndStat); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnBeginExclusiveStat); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnEndExclusiveStat); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnCustomStatInt); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnCustomStatFloat); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnEvent); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnBeginCapture); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnEndCapture); + Subs.emplace_back(this, &CsvProfilerAnalyzer::OnMetadata); + } + + eastl::vector<zen::trace_detail::TraceModel::CsvCategory>& MutableCategories() { return m_Categories; } + eastl::vector<zen::trace_detail::TraceModel::CsvStatDef>& MutableStatDefs() { return m_StatDefs; } + eastl::vector<zen::trace_detail::TraceModel::CsvEvent>& MutableEvents() { return m_Events; } + eastl::vector<zen::trace_detail::TraceModel::CsvMeta>& MutableMetadata() { return m_Metadata; } + + // Build the per-stat+thread time series from the accumulated samples. + eastl::vector<zen::trace_detail::TraceModel::CsvSeries> BuildTimeSeries() + { + eastl::vector<zen::trace_detail::TraceModel::CsvSeries> Result; + for (auto& [Key, Samples] : m_SeriesMap) + { + eastl::sort(Samples.begin(), Samples.end(), [](const auto& A, const auto& B) { return A.TimeUs < B.TimeUs; }); + zen::trace_detail::TraceModel::CsvSeries S; + S.StatId = Key.StatId; + S.ThreadId = Key.ThreadId; + S.Samples = std::move(Samples); + Result.push_back(std::move(S)); + } + return Result; + } + +private: + uint32_t CycleToTimeUs(uint64_t Cycle) const + { + if (!m_Timing || m_Timing->Freq == 0) + { + return 0; + } + uint64_t Elapsed = (Cycle >= m_Timing->Base) ? (Cycle - m_Timing->Base) : 0; + return uint32_t(Elapsed * 1'000'000 / m_Timing->Freq); + } + + static std::string DecodeAnsiName(const Array<uint8[]>& Data) + { + const uint8_t* P = Data.get(); + size_t Size = Data.get_size(); + if (!P || Size == 0) + { + return {}; + } + return std::string(reinterpret_cast<const char*>(P), Size); + } + + static std::string DecodeWideName(const Array<uint8[]>& Data) + { + const uint8_t* P = Data.get(); + size_t Size = Data.get_size(); + uint32_t Count = Data.get_count(); + if (!P || Size == 0 || Count == 0) + { + return {}; + } + uint32_t ElemSize = Data.get_element_size(); + if (ElemSize == 2) + { + std::string Out; + Out.reserve(Count); + const char16_t* W = reinterpret_cast<const char16_t*>(P); + for (uint32_t I = 0; I < Count; ++I) + { + Out.push_back(W[I] < 0x80 ? char(W[I]) : '?'); + } + return Out; + } + return std::string(reinterpret_cast<const char*>(P), Size); + } + + struct SeriesKey + { + uint64_t StatId; + uint32_t ThreadId; + bool operator==(const SeriesKey& O) const { return StatId == O.StatId && ThreadId == O.ThreadId; } + }; + struct SeriesKeyHash + { + size_t operator()(const SeriesKey& K) const + { + return eastl::hash<uint64_t>{}(K.StatId) ^ (eastl::hash<uint32_t>{}(K.ThreadId) * 2654435761u); + } + }; + + void AddSample(uint64_t StatId, uint32_t ThreadId, uint32_t TimeUs, float Value) + { + m_SeriesMap[SeriesKey{StatId, ThreadId}].push_back({TimeUs, Value}); + } + + // -- Event handlers ----------------------------------------------- + + void OnRegisterCategory(const CsvProfiler_RegisterCategory& Ev) + { + zen::trace_detail::TraceModel::CsvCategory Cat; + Cat.Index = Ev.Index(); + Cat.Name = DecodeAnsiName(Ev.Name()); + m_Categories.push_back(std::move(Cat)); + } + + void OnDefineInlineStat(const CsvProfiler_DefineInlineStat& Ev) + { + DefineStat(Ev.StatId(), Ev.CategoryIndex(), DecodeAnsiName(Ev.Name())); + } + + void OnDefineDeclaredStat(const CsvProfiler_DefineDeclaredStat& Ev) + { + DefineStat(Ev.StatId(), Ev.CategoryIndex(), DecodeAnsiName(Ev.Name())); + } + + void DefineStat(uint64_t StatId, int32_t CategoryIndex, std::string Name) + { + if (m_StatIdToIndex.count(StatId)) + { + return; // already defined + } + m_StatIdToIndex[StatId] = uint32_t(m_StatDefs.size()); + zen::trace_detail::TraceModel::CsvStatDef Def; + Def.StatId = StatId; + Def.CategoryIndex = CategoryIndex; + Def.Name = std::move(Name); + m_StatDefs.push_back(std::move(Def)); + } + + void OnBeginStat(const CsvProfiler_BeginStat& Ev) + { + uint32_t ThreadId = Ev.get_thread_id(); + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + m_OpenStacks[{Ev.StatId(), ThreadId}].push_back(TimeUs); + } + + void OnEndStat(const CsvProfiler_EndStat& Ev) + { + uint32_t ThreadId = Ev.get_thread_id(); + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + auto Key = SeriesKey{Ev.StatId(), ThreadId}; + auto It = m_OpenStacks.find(Key); + if (It == m_OpenStacks.end() || It->second.empty()) + { + return; + } + uint32_t BeginUs = It->second.back(); + It->second.pop_back(); + float DurationMs = float(TimeUs - BeginUs) / 1000.0f; + AddSample(Ev.StatId(), ThreadId, BeginUs, DurationMs); + } + + void OnBeginExclusiveStat(const CsvProfiler_BeginExclusiveStat& Ev) + { + // For basic support, treat exclusive stats like regular stats. + uint32_t ThreadId = Ev.get_thread_id(); + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + m_OpenStacks[{Ev.StatId(), ThreadId}].push_back(TimeUs); + } + + void OnEndExclusiveStat(const CsvProfiler_EndExclusiveStat& Ev) + { + uint32_t ThreadId = Ev.get_thread_id(); + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + auto Key = SeriesKey{Ev.StatId(), ThreadId}; + auto It = m_OpenStacks.find(Key); + if (It == m_OpenStacks.end() || It->second.empty()) + { + return; + } + uint32_t BeginUs = It->second.back(); + It->second.pop_back(); + float DurationMs = float(TimeUs - BeginUs) / 1000.0f; + AddSample(Ev.StatId(), ThreadId, BeginUs, DurationMs); + } + + void OnCustomStatInt(const CsvProfiler_CustomStatInt& Ev) + { + uint32_t ThreadId = Ev.get_thread_id(); + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + AddSample(Ev.StatId(), ThreadId, TimeUs, float(Ev.Value())); + } + + void OnCustomStatFloat(const CsvProfiler_CustomStatFloat& Ev) + { + uint32_t ThreadId = Ev.get_thread_id(); + uint32_t TimeUs = CycleToTimeUs(Ev.Cycle()); + AddSample(Ev.StatId(), ThreadId, TimeUs, Ev.Value()); + } + + void OnEvent(const CsvProfiler_Event& Ev) + { + zen::trace_detail::TraceModel::CsvEvent E; + E.TimeUs = CycleToTimeUs(Ev.Cycle()); + E.CategoryIndex = Ev.CategoryIndex(); + E.Text = DecodeWideName(Ev.Text()); + m_Events.push_back(std::move(E)); + } + + void OnBeginCapture(const CsvProfiler_BeginCapture& Ev) { m_CaptureStartUs = CycleToTimeUs(Ev.Cycle()); } + + void OnEndCapture(const CsvProfiler_EndCapture& Ev) { m_CaptureEndUs = CycleToTimeUs(Ev.Cycle()); } + + void OnMetadata(const CsvProfiler_Metadata& Ev) + { + zen::trace_detail::TraceModel::CsvMeta M; + M.Key = DecodeWideName(Ev.Key()); + M.Value = DecodeWideName(Ev.Value()); + m_Metadata.push_back(std::move(M)); + } + + const TraceTiming* m_Timing = nullptr; + + eastl::vector<zen::trace_detail::TraceModel::CsvCategory> m_Categories; + eastl::vector<zen::trace_detail::TraceModel::CsvStatDef> m_StatDefs; + eastl::hash_map<uint64_t, uint32_t> m_StatIdToIndex; + + // Timing stacks: (StatId, ThreadId) -> stack of begin times + eastl::hash_map<SeriesKey, eastl::vector<uint32_t>, SeriesKeyHash> m_OpenStacks; + + // Accumulated samples: (StatId, ThreadId) -> samples + eastl::hash_map<SeriesKey, eastl::vector<zen::trace_detail::TraceModel::CsvSample>, SeriesKeyHash> m_SeriesMap; + + eastl::vector<zen::trace_detail::TraceModel::CsvEvent> m_Events; + eastl::vector<zen::trace_detail::TraceModel::CsvMeta> m_Metadata; + + uint32_t m_CaptureStartUs = 0; + uint32_t m_CaptureEndUs = 0; +}; + +////////////////////////////////////////////////////////////////////////////// +// Analyzers + +class CpuAnalyzer : public Analyzer +{ +public: + CpuAnalyzer(EventStats& Stats, NameDepot& Names, const MetadataRegistry* Metadata) + : m_Names(Names) + , m_Stats(Stats) + , m_Metadata(Metadata) + { + Names.Add(NO_NAME, "???"); + } + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &CpuAnalyzer::OnNewTrace); + Subs.emplace_back(this, &CpuAnalyzer::OnCpuSpec); + Subs.emplace_back(this, &CpuAnalyzer::OnCpuBatch); + Subs.emplace_back(this, &CpuAnalyzer::OnCpuBatchV2); + Subs.emplace_back(this, &CpuAnalyzer::OnCpuBatchV3); + } + +private: + static constexpr uint32 NO_INDEX = ~0u; + static constexpr uint64 NO_NAME = ~0ull; + static constexpr uint32 METADATA_BIT = 0x8000'0000u; + + uint64 ResolveNameHash(uint32 PackedId) + { + const bool IsMetadata = (PackedId & METADATA_BIT) != 0; + const uint32 Id = PackedId & ~METADATA_BIT; + + if (IsMetadata && m_Metadata) + { + auto CachedIt = m_MetadataNames.find(Id); + if (CachedIt != m_MetadataNames.end()) + { + return CachedIt->second; + } + + const MetadataEntry* Entry = m_Metadata->Lookup(Id); + if (Entry) + { + auto BaseIt = m_Specs.find(Entry->SpecId); + StringView BaseName = (BaseIt != m_Specs.end()) ? m_Names.Get(BaseIt->second) : StringView("???"); + std::string Formatted(BaseName); + std::string Values = FormatMetadataValues(Entry->Bytes.data(), Entry->Bytes.size()); + if (!Values.empty()) + { + Formatted += " - "; + Formatted += Values; + } + uint64 Hash = m_Names.Add(StringView(Formatted)); + m_MetadataNames[Id] = Hash; + return Hash; + } + return NO_NAME; + } + + auto It = m_Specs.find(Id); + return (It != m_Specs.end()) ? It->second : NO_NAME; + } + + struct EventStack + { + uint32 Tail = NO_INDEX; + }; + + struct ScopeEvent + { + uint32 Id; + uint32 TimeUs; + union + { + uint32 Next; + uint32 Index; + }; + }; + + struct EventPool + { + uint32 Alloc() + { + if (m_FreeHead == NO_INDEX) + { + uint32 Idx = uint32(m_Pool.size()); + m_Pool.push_back({.Index = Idx}); + return Idx; + } + uint32 Idx = m_FreeHead; + m_FreeHead = m_Pool[Idx].Index; + return Idx; + } + + void Free(uint32 Idx) + { + m_Pool[Idx].Index = m_FreeHead; + m_FreeHead = Idx; + } + + ScopeEvent& Get(uint32 Idx) { return m_Pool[Idx]; } + + eastl::vector<ScopeEvent> m_Pool; + uint32 m_FreeHead = NO_INDEX; + }; + + void OnNewTrace(const $Trace_NewTrace& NewTrace) + { + m_Freq = NewTrace.CycleFrequency(); + m_Base = NewTrace.StartCycle(); + m_UsDiv = m_Freq / 1'000'000; + if (m_UsDiv == 0) + { + m_UsDiv = 1; + } + m_UsDivRecip = ReciprocalU64(m_UsDiv); + } + + void OnCpuSpec(const CpuProfiler_EventSpec& Spec) + { + uint32 SpecId = Spec.Id(); + FieldStr SpecName = Spec.Name(); + + StringView NameView = SpecName.as_view(); + + if (NameView.starts_with("Frame ")) + { + NameView = "Frame"; + } + if (size_t Pos = NameView.find("\""); Pos != StringView::npos) + { + NameView = NameView.substr(0, Pos); + } + if (size_t Pos = NameView.find("\\"); Pos != StringView::npos) + { + NameView = NameView.substr(0, Pos); + } + + m_Specs[SpecId] = m_Names.Add(NameView); + } + + void OnCpuBatch(const CpuProfiler_EventBatch& Batch) + { + uint32 ThreadId = Batch.get_thread_id(); + Array<uint8[]> Data = Batch.Data(); + AbsorbBatch(/*Version=*/1, ThreadId, Data); + } + + void OnCpuBatchV2(const CpuProfiler_EventBatchV2& Batch) + { + uint32 ThreadId = Batch.get_thread_id(); + Array<uint8[]> Data = Batch.Data(); + AbsorbBatch(/*Version=*/2, ThreadId, Data); + } + + void OnCpuBatchV3(const CpuProfiler_EventBatchV3& Batch) + { + uint32 ThreadId = Batch.get_thread_id(); + Array<uint8[]> Data = Batch.Data(); + AbsorbBatch(/*Version=*/3, ThreadId, Data); + } + + // Decodes a CpuProfiler scope batch. Mirrors UE's reference + // TraceServices/.../CpuProfilerTraceAnalysis.cpp ProcessBuffer / + // ProcessBufferV2. + // + // Version 1 (`CpuProfiler.EventBatch`): cycle is `value >> 1`; bit 0 is + // IsEnter; IsEnter events carry a SpecId varint. + // + // Version 2 (`CpuProfiler.EventBatchV2`, UE 5.1..5.5) and Version 3 + // (`CpuProfiler.EventBatchV3`, UE 5.6+): cycle is `value >> 2`; bit 0 is + // IsEnter, bit 1 is IsCoroutine. Coroutine begin events carry CoroutineId + // and TimerScopeDepth varints; coroutine end events carry a single + // TimerScopeDepth varint. V3 additionally reserves the low bit of the + // SpecId to mark metadata-bearing timers, so SpecId must be shifted + // right by 1 to recover the actual spec id. + void AbsorbBatch(uint32 Version, uint32 ThreadId, const Array<uint8[]>& Data) + { + const uint8* Cursor = Data.get(); + const uint8* End = Cursor + Data.get_size(); + + auto Decode = [&]() { + uint64 Value = 0; + for (uint32 I = 1, J = 0; I; J += 7) + { + I = *Cursor++; + Value |= uint64(I & 0x7f) << J; + I &= 0x80; + } + return Value; + }; + + if (ThreadId >= m_Threads.size()) + { + m_Threads.resize(ThreadId + 1); + } + EventStack& Stack = m_Threads[ThreadId]; + + const uint32 CycleShift = (Version == 1) ? 1u : 2u; + + uint64 Base = m_Base; + + uint64 Cycle = ~Base + 1; + while (Cursor < End) + { + uint64 Value = Decode(); + uint32 IsEnter = (Value & 0b01); + + if (Version > 1 && (Value & 0b10)) + { + // Coroutine event -- not visualised, but the trailing varints + // still need to be consumed so we stay in sync with the + // stream. + if (IsEnter) + { + (void)Decode(); // CoroutineId + (void)Decode(); // TimerScopeDepth + } + else + { + (void)Decode(); // TimerScopeDepth + } + continue; + } + + uint64 EventId = IsEnter ? Decode() : ~0ull; + + Cycle += (Value >> CycleShift); + uint32 TimeUs = m_UsDivRecip.Divide(Cycle + (m_UsDiv >> 1)); + + if (IsEnter) + { + uint32 ScopeId = uint32(EventId); + bool IsMetadata = false; + if (Version == 3) + { + IsMetadata = (ScopeId & 1u) != 0; + ScopeId >>= 1; + } + uint32 EvIdx = m_Events.Alloc(); + ScopeEvent& Ev = m_Events.Get(EvIdx); + // Pack the metadata flag in the high bit so the close path + // can distinguish metadata-id scopes from regular ones without + // an extra field. + Ev.Id = IsMetadata ? (ScopeId | 0x8000'0000u) : ScopeId; + Ev.TimeUs = TimeUs; + Ev.Next = Stack.Tail; + Stack.Tail = EvIdx; + continue; + } + + if (Stack.Tail == NO_INDEX) + { + continue; + } + + ScopeEvent& Ev = m_Events.Get(Stack.Tail); + uint32 DurationUs = TimeUs - Ev.TimeUs; + uint64 NameHash = ResolveNameHash(Ev.Id); + m_Stats.Record(NameHash, DurationUs); + + uint32 NextIdx = Ev.Next; + m_Events.Free(Stack.Tail); + Stack.Tail = NextIdx; + } + } + + uint64 m_Freq = 0; + uint64 m_Base = 0; + uint64 m_UsDiv = 1; + ReciprocalU64 m_UsDivRecip; + eastl::hash_map<uint32, uint64> m_Specs; + NameDepot& m_Names; + EventPool m_Events; + eastl::vector<EventStack> m_Threads; + EventStats& m_Stats; + const MetadataRegistry* m_Metadata = nullptr; + // Caches the resolved name hash for each MetadataId so we don't + // re-format the same CBOR payload on every scope-close. + eastl::hash_map<uint32, uint64> m_MetadataNames; +}; + +////////////////////////////////////////////////////////////////////////////// +// Per-event CPU scope capture for the interactive trace viewer. +// +// Mirrors CpuAnalyzer's decode loop but instead of aggregating statistics, +// it records one TimelineScope per closed CPU scope so the viewer can draw a +// flame graph. Scope names are interned into a flat vector so each event only +// stores a compact uint32 NameId. + +class TimelineAnalyzer : public Analyzer +{ +public: + explicit TimelineAnalyzer(const MetadataRegistry* Metadata = nullptr, TraceTiming* SharedTiming = nullptr) + : m_SharedTiming(SharedTiming) + , m_Metadata(Metadata) + { + } + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &TimelineAnalyzer::OnNewTrace); + Subs.emplace_back(this, &TimelineAnalyzer::OnCpuSpec); + Subs.emplace_back(this, &TimelineAnalyzer::OnCpuBatch); + Subs.emplace_back(this, &TimelineAnalyzer::OnCpuBatchV2); + Subs.emplace_back(this, &TimelineAnalyzer::OnCpuBatchV3); + } + + struct ThreadData + { + eastl::vector<zen::trace_detail::TimelineScope> Scopes; + // Open-scope stack: parallel arrays keeping begin time and name id. + eastl::vector<uint32_t> OpenBeginUs; + eastl::vector<uint32_t> OpenNameIds; + }; + + const eastl::vector<std::string>& ScopeNames() const { return m_ScopeNames; } + const eastl::map<uint32_t, ThreadData>& Threads() const { return m_Threads; } + uint32_t MinBeginUs() const { return m_MinBeginUs; } + uint32_t MaxEndUs() const { return m_MaxEndUs; } + +private: + static constexpr uint32_t INVALID_NAME = ~0u; + + uint32_t InternName(StringView Name) + { + String Key(Name); + auto [It, Inserted] = m_NameIndex.try_emplace(std::move(Key), 0); + if (Inserted) + { + It->second = uint32_t(m_ScopeNames.size()); + m_ScopeNames.emplace_back(Name); + } + return It->second; + } + + void OnNewTrace(const $Trace_NewTrace& NewTrace) + { + m_Freq = NewTrace.CycleFrequency(); + m_Base = NewTrace.StartCycle(); + m_UsDiv = m_Freq / 1'000'000; + if (m_UsDiv == 0) + { + m_UsDiv = 1; + } + m_UsDivRecip = ReciprocalU64(m_UsDiv); + if (m_SharedTiming) + { + m_SharedTiming->Freq = m_Freq; + m_SharedTiming->Base = m_Base; + m_SharedTiming->UsDiv = m_UsDiv; + } + } + + void OnCpuSpec(const CpuProfiler_EventSpec& Spec) + { + uint32 SpecId = Spec.Id(); + FieldStr SpecName = Spec.Name(); + + StringView NameView = SpecName.as_view(); + + if (NameView.starts_with("Frame ")) + { + NameView = "Frame"; + } + if (size_t Pos = NameView.find("\""); Pos != StringView::npos) + { + NameView = NameView.substr(0, Pos); + } + if (size_t Pos = NameView.find("\\"); Pos != StringView::npos) + { + NameView = NameView.substr(0, Pos); + } + + m_Specs[SpecId] = InternName(NameView); + } + + void OnCpuBatch(const CpuProfiler_EventBatch& Batch) + { + uint32 ThreadId = Batch.get_thread_id(); + Array<uint8[]> Data = Batch.Data(); + AbsorbBatch(/*Version=*/1, ThreadId, Data); + } + + void OnCpuBatchV2(const CpuProfiler_EventBatchV2& Batch) + { + uint32 ThreadId = Batch.get_thread_id(); + Array<uint8[]> Data = Batch.Data(); + AbsorbBatch(/*Version=*/2, ThreadId, Data); + } + + void OnCpuBatchV3(const CpuProfiler_EventBatchV3& Batch) + { + uint32 ThreadId = Batch.get_thread_id(); + Array<uint8[]> Data = Batch.Data(); + AbsorbBatch(/*Version=*/3, ThreadId, Data); + } + + TraceTiming* m_SharedTiming = nullptr; + + // See CpuAnalyzer::AbsorbBatch for a detailed description of the wire + // format for each version. + void AbsorbBatch(uint32 Version, uint32 ThreadId, const Array<uint8[]>& Data) + { + const uint8* Cursor = Data.get(); + const uint8* End = Cursor + Data.get_size(); + + auto Decode = [&]() { + uint64 Value = 0; + for (uint32 I = 1, J = 0; I; J += 7) + { + I = *Cursor++; + Value |= uint64(I & 0x7f) << J; + I &= 0x80; + } + return Value; + }; + + ThreadData& Thread = m_Threads[ThreadId]; + + const uint32 CycleShift = (Version == 1) ? 1u : 2u; + + uint64 Base = m_Base; + + uint64 Cycle = ~Base + 1; + while (Cursor < End) + { + uint64 Value = Decode(); + uint32 IsEnter = (Value & 0b01); + + if (Version > 1 && (Value & 0b10)) + { + // Coroutine event -- consume the trailing varints so we stay + // in sync with the stream, but drop the event on the floor. + if (IsEnter) + { + (void)Decode(); // CoroutineId + (void)Decode(); // TimerScopeDepth + } + else + { + (void)Decode(); // TimerScopeDepth + } + continue; + } + + uint64 EventId = IsEnter ? Decode() : ~0ull; + + Cycle += (Value >> CycleShift); + uint32 TimeUs = m_UsDivRecip.Divide(Cycle + (m_UsDiv >> 1)); + + if (IsEnter) + { + uint32 ScopeId = uint32(EventId); + bool IsMetadata = false; + if (Version == 3) + { + IsMetadata = (ScopeId & 1u) != 0; + ScopeId >>= 1; + } + uint32_t NameId = IsMetadata ? ResolveMetadataNameId(ScopeId) : LookupSpecNameId(ScopeId); + Thread.OpenBeginUs.push_back(TimeUs); + Thread.OpenNameIds.push_back(NameId); + continue; + } + + if (Thread.OpenBeginUs.empty()) + { + continue; + } + + uint32_t BeginUs = Thread.OpenBeginUs.back(); + uint32_t NameId = Thread.OpenNameIds.back(); + Thread.OpenBeginUs.pop_back(); + Thread.OpenNameIds.pop_back(); + + if (NameId == INVALID_NAME) + { + continue; + } + + uint16_t Depth = uint16_t(Thread.OpenBeginUs.size()); + Thread.Scopes.push_back(zen::trace_detail::TimelineScope{ + .BeginUs = BeginUs, + .DurationUs = TimeUs - BeginUs, + .NameId = NameId, + .Depth = Depth, + .MergeCount = 0, + }); + + if (BeginUs < m_MinBeginUs) + { + m_MinBeginUs = BeginUs; + } + if (TimeUs > m_MaxEndUs) + { + m_MaxEndUs = TimeUs; + } + } + } + + uint32_t LookupSpecNameId(uint32 SpecId) const + { + auto It = m_Specs.find(SpecId); + return (It != m_Specs.end()) ? It->second : INVALID_NAME; + } + + uint32_t ResolveMetadataNameId(uint32 MetadataId) + { + if (!m_Metadata) + { + return INVALID_NAME; + } + + auto CachedIt = m_MetadataNameIds.find(MetadataId); + if (CachedIt != m_MetadataNameIds.end()) + { + return CachedIt->second; + } + + const MetadataEntry* Entry = m_Metadata->Lookup(MetadataId); + if (!Entry) + { + return INVALID_NAME; + } + + auto BaseIt = m_Specs.find(Entry->SpecId); + const std::string& BaseName = (BaseIt != m_Specs.end()) ? m_ScopeNames[BaseIt->second] : kUnknownName; + std::string Formatted = BaseName; + std::string Values = FormatMetadataValues(Entry->Bytes.data(), Entry->Bytes.size()); + if (!Values.empty()) + { + Formatted += " - "; + Formatted += Values; + } + + uint32_t NameId = InternName(StringView(Formatted.data(), Formatted.size())); + m_MetadataNameIds[MetadataId] = NameId; + return NameId; + } + + static inline const std::string kUnknownName{"???"}; + + uint64 m_Freq = 0; + uint64 m_Base = 0; + uint64 m_UsDiv = 1; + ReciprocalU64 m_UsDivRecip; + uint32_t m_MinBeginUs = ~0u; + uint32_t m_MaxEndUs = 0; + eastl::hash_map<uint32, uint32_t> m_Specs; + eastl::hash_map<String, uint32_t> m_NameIndex; + eastl::vector<std::string> m_ScopeNames; + eastl::map<uint32_t, ThreadData> m_Threads; + const MetadataRegistry* m_Metadata = nullptr; + eastl::hash_map<uint32_t, uint32_t> m_MetadataNameIds; +}; + +////////////////////////////////////////////////////////////////////////////// + +} // anonymous namespace + +std::string +zen::trace_detail::SafeFieldStr(FieldStr&& Field) +{ + try + { + std::string_view View = Field.as_view(); + // Some trace writers include the NUL terminator in the field length + // (see UE trace ToAnsiCheap / ThreadRegister). Strip any trailing NULs + // so downstream consumers don't see garbage. + while (!View.empty() && View.back() == '\0') + { + View.remove_suffix(1); + } + return std::string(View); + } + catch (const std::exception& E) + { + ZEN_DEBUG("Failed to decode trace string field: {}", E.what()); + return {}; + } +} + +namespace { + +// Derive a thread group name from a thread name by stripping a trailing +// integer suffix (optionally preceded by a separator). E.g. "IoPool Worker 3" +// -> "IoPool Worker", "DbWorker_12" -> "DbWorker", "HttpThread42" -> +// "HttpThread". Returns an empty string if no suffix is present or the +// resulting prefix would be empty. +static std::string +SynthesizeThreadGroupFromName(std::string_view Name) +{ + size_t I = Name.size(); + while (I > 0 && Name[I - 1] >= '0' && Name[I - 1] <= '9') + { + --I; + } + if (I == Name.size()) + { + return {}; // no trailing digits + } + if (I > 0) + { + char C = Name[I - 1]; + if (C == '_' || C == '-' || C == '.' || C == ':' || C == '#' || C == '/' || C == ' ' || C == '\t') + { + --I; + } + } + while (I > 0 && (Name[I - 1] == ' ' || Name[I - 1] == '\t')) + { + --I; + } + if (I == 0) + { + return {}; // pure-numeric name + } + return std::string(Name.substr(0, I)); +} + +class SessionAnalyzer : public Analyzer +{ +public: + zen::trace_detail::SessionInfo Session; + eastl::map<uint32_t, zen::trace_detail::ThreadInfoEntry> ThreadNames; + eastl::map<uint32_t, zen::trace_detail::ChannelInfo> Channels; + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &SessionAnalyzer::OnSession); + Subs.emplace_back(this, &SessionAnalyzer::OnThreadGroupBegin); + Subs.emplace_back(this, &SessionAnalyzer::OnThreadGroupEnd); + Subs.emplace_back(this, &SessionAnalyzer::OnThreadInfo); + Subs.emplace_back(this, &SessionAnalyzer::OnChannelAnnounce); + Subs.emplace_back(this, &SessionAnalyzer::OnChannelToggle); + } + +private: + eastl::vector<String> m_GroupStack; + + void OnSession(const Diagnostics_Session2& Ev) + { + Session.Platform = SafeFieldStr(Ev.Platform()); + Session.AppName = SafeFieldStr(Ev.AppName()); + Session.ProjectName = SafeFieldStr(Ev.ProjectName()); + Session.CommandLine = SafeFieldStr(Ev.CommandLine()); + Session.Branch = SafeFieldStr(Ev.Branch()); + Session.BuildVersion = SafeFieldStr(Ev.BuildVersion()); + Session.Changelist = Ev.Changelist(); + Session.ConfigurationType = Ev.ConfigurationType(); + Session.HasSession = true; + } + + void OnThreadGroupBegin(const $Trace_ThreadGroupBegin& Ev) { m_GroupStack.push_back(SafeFieldStr(Ev.Name())); } + + void OnThreadGroupEnd(const $Trace_ThreadGroupEnd&) + { + if (!m_GroupStack.empty()) + { + m_GroupStack.pop_back(); + } + } + + void OnThreadInfo(const $Trace_ThreadInfo& Ev) + { + uint32_t Tid = Ev.ThreadId(); + zen::trace_detail::ThreadInfoEntry& Info = ThreadNames[Tid]; + Info.ThreadId = Tid; + Info.Name = SafeFieldStr(Ev.Name()); + Info.GroupName = m_GroupStack.empty() ? "" : m_GroupStack.back(); + if (Info.GroupName.empty()) + { + Info.GroupName = SynthesizeThreadGroupFromName(Info.Name); + } + Info.SystemId = Ev.SystemId(); + Info.SortHint = Ev.SortHint(); + } + + void OnChannelAnnounce(const Trace_ChannelAnnounce& Ev) + { + uint32_t Id = Ev.Id(); + zen::trace_detail::ChannelInfo& Info = Channels[Id]; + Info.Name = SafeFieldStr(Ev.Name()); + Info.Enabled = Ev.IsEnabled() != 0; + Info.ReadOnly = Ev.ReadOnly() != 0; + } + + void OnChannelToggle(const Trace_ChannelToggle& Ev) + { + uint32_t Id = Ev.Id(); + auto It = Channels.find(Id); + if (It != Channels.end()) + { + It->second.Enabled = Ev.IsEnabled() != 0; + } + } +}; + +////////////////////////////////////////////////////////////////////////////// +// Module analyzer +// +// Captures Diagnostics.Module{Init,Load,Unload} so TraceModel::Modules has a +// populated list of loaded DLLs. These events are NoSync+Important so they +// don't carry a Cycle field (no load/unload timestamps available) but they +// do survive reconnects and the trim filter. The analyzer is intentionally +// passive -- we stash the raw data here and leave symbolication and memory +// attribution to whatever consumes TraceModel::Modules later. + +class ModuleAnalyzer : public Analyzer +{ +public: + eastl::map<uint64_t, zen::trace_detail::ModuleInfo> ModulesByBase; + std::string SymbolFormat; + uint8_t BaseShift = 0; + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &ModuleAnalyzer::OnModuleInit); + Subs.emplace_back(this, &ModuleAnalyzer::OnModuleLoad); + Subs.emplace_back(this, &ModuleAnalyzer::OnModuleUnload); + } + +private: + void OnModuleInit(const Diagnostics_ModuleInit& Ev) + { + SymbolFormat = SafeFieldStr(Ev.SymbolFormat()); + BaseShift = Ev.ModuleBaseShift(); + } + + void OnModuleLoad(const Diagnostics_ModuleLoad& Ev) + { + // Older traces stored Base as a 32-bit value shifted right by + // ModuleBaseShift to fit. Modern traces set BaseShift to zero and + // Base is a full 64-bit address; applying the shift then is a + // harmless no-op. + uint64_t Base = uint64_t(Ev.Base()) << BaseShift; + + zen::trace_detail::ModuleInfo& Info = ModulesByBase[Base]; + Info.FullPath = SafeFieldStr(Ev.Name()); + Info.Base = Base; + Info.Size = Ev.Size(); + Info.Unloaded = false; + + // Extract the basename without pulling in the whole filesystem + // library for a single operation. UE emits forward- or backslashes + // depending on platform, so handle both. + const std::string& Path = Info.FullPath; + size_t Cut = Path.find_last_of("/\\"); + Info.Name = (Cut == std::string::npos) ? Path : Path.substr(Cut + 1); + + ::Array<uint8[]> ImageId = Ev.ImageId(); + const uint8* IdPtr = ImageId.get(); + const uint32 IdSize = ImageId.get_size(); + Info.ImageId.assign(IdPtr, IdPtr + IdSize); + } + + void OnModuleUnload(const Diagnostics_ModuleUnload& Ev) + { + uint64_t Base = uint64_t(Ev.Base()) << BaseShift; + auto It = ModulesByBase.find(Base); + if (It != ModulesByBase.end()) + { + It->second.Unloaded = true; + } + } +}; + +////////////////////////////////////////////////////////////////////////////// +// Trim analyzer +// +// Decodes CpuProfiler batch events to extract per-batch timestamp ranges AND +// to track open/close scope bracketing per thread. The scope tracker lets us +// identify "must-keep" packets: any packet containing the Leave event for a +// scope whose Enter was at or before the user's trim EndUs. Preserving those +// Leaves is what lets the downstream TimelineAnalyzer (and Unreal Insights) +// still render long-running scopes that span the window end -- if we dropped +// their closing event the scope would sit unmatched on the open-scope stack +// and not render at all. +// +// Attribution to raw packet indices is approximate due to Tourist's internal +// per-thread packet buffering; the trim driver processes the trace one packet +// at a time (bundle of size 1) to keep it as tight as possible. Packets that +// never get an attributed time range are conservatively retained by the +// caller. + +class TrimAnalyzer : public Analyzer +{ +public: + // Maps packet index (matching both Tourist's Packet::get_index() and our + // raw walker's vector position) -> (MinUs, MaxUs) of all events attributed + // to that packet. + struct Range + { + uint32_t MinUs = ~0u; + uint32_t MaxUs = 0; + }; + + eastl::hash_map<uint32_t, Range> PacketRanges; + + // Maps thread id -> the maximum packet index that contains a Leave event + // for a scope whose matching Enter was at or before EndUs. These packets + // must be retained so the downstream analyzer can close the scope. + eastl::hash_map<uint32_t, uint32_t> MustKeepPacketByThread; + + // Set by the trim driver from TraceTrimArgs::EndSec before the analysis + // pass begins. Used by the scope tracker to decide which leaves are + // "must keep". + uint32_t EndUs = ~0u; + + // Updated by the trim driver before each Proto.read call when the next + // packet is on a normal thread. Maps normal-thread id -> the most + // recently scattered packet's index. + eastl::hash_map<uint32_t, uint32_t> LastPacketIndexByThread; + + void subscribe(Vector<Subscription>& Subs) override + { + Subs.emplace_back(this, &TrimAnalyzer::OnNewTrace); + Subs.emplace_back(this, &TrimAnalyzer::OnCpuBatch); + Subs.emplace_back(this, &TrimAnalyzer::OnCpuBatchV2); + Subs.emplace_back(this, &TrimAnalyzer::OnCpuBatchV3); + } + + bool HasTimeBase() const { return m_Freq != 0; } + +private: + void OnNewTrace(const $Trace_NewTrace& NewTrace) + { + m_Freq = NewTrace.CycleFrequency(); + m_Base = NewTrace.StartCycle(); + m_UsDiv = (m_Freq > 0) ? (m_Freq / 1'000'000) : 1; + if (m_UsDiv == 0) + { + m_UsDiv = 1; + } + m_UsDivRecip = ReciprocalU64(m_UsDiv); + } + + void OnCpuBatch(const CpuProfiler_EventBatch& Batch) { AbsorbBatchTimes(/*Version=*/1, Batch.get_thread_id(), Batch.Data()); } + + void OnCpuBatchV2(const CpuProfiler_EventBatchV2& Batch) { AbsorbBatchTimes(/*Version=*/2, Batch.get_thread_id(), Batch.Data()); } + + void OnCpuBatchV3(const CpuProfiler_EventBatchV3& Batch) { AbsorbBatchTimes(/*Version=*/3, Batch.get_thread_id(), Batch.Data()); } + + // Decodes cycle deltas in a CpuProfiler batch to find the timestamp range + // AND to maintain a per-thread open-scope stack. Mirrors the wire format + // documented in CpuAnalyzer::AbsorbBatch. Scope ids are decoded just far + // enough to keep the varint cursor in sync; we don't store them. + void AbsorbBatchTimes(uint32 Version, uint32 ThreadId, const Array<uint8[]>& Data) + { + if (m_Freq == 0) + { + return; + } + + auto It = LastPacketIndexByThread.find(ThreadId); + if (It == LastPacketIndexByThread.end()) + { + return; + } + const uint32_t PacketIndex = It->second; + + // The open-scope stack is maintained across every batch on a thread. + // Each entry stores the Enter time in microseconds from trace start. + eastl::vector<uint32_t>& OpenStack = m_OpenScopes[ThreadId]; + + const uint8* Cursor = Data.get(); + const uint8* End = Cursor + Data.get_size(); + + auto Decode = [&]() { + uint64 Value = 0; + for (uint32 I = 1, J = 0; I; J += 7) + { + I = *Cursor++; + Value |= uint64(I & 0x7f) << J; + I &= 0x80; + } + return Value; + }; + + const uint32 CycleShift = (Version == 1) ? 1u : 2u; + const uint64 Base = m_Base; + + uint32_t BatchMinUs = ~0u; + uint32_t BatchMaxUs = 0; + bool HasAny = false; + + uint64 Cycle = ~Base + 1; + while (Cursor < End) + { + uint64 Value = Decode(); + uint32 IsEnter = (Value & 0b01); + + if (Version > 1 && (Value & 0b10)) + { + // Coroutine event -- consume the trailing varints and skip. + // These don't participate in the scope bracket tracking; the + // existing TimelineAnalyzer ignores them for the same reason. + if (IsEnter) + { + (void)Decode(); // CoroutineId + (void)Decode(); // TimerScopeDepth + } + else + { + (void)Decode(); // TimerScopeDepth + } + continue; + } + + if (IsEnter) + { + (void)Decode(); // EventId / SpecId + } + + Cycle += (Value >> CycleShift); + uint32_t TimeUs = m_UsDivRecip.Divide(Cycle + (m_UsDiv >> 1)); + + if (!HasAny || TimeUs < BatchMinUs) + { + BatchMinUs = TimeUs; + } + if (!HasAny || TimeUs > BatchMaxUs) + { + BatchMaxUs = TimeUs; + } + HasAny = true; + + if (IsEnter) + { + OpenStack.push_back(TimeUs); + } + else if (!OpenStack.empty()) + { + uint32_t EnterTimeUs = OpenStack.back(); + OpenStack.pop_back(); + + // If the scope started at or before the window end, we need + // its closing Leave event to survive so the downstream + // analyzer can render it. Mark the current packet (the one + // holding this Leave) as must-keep for the thread. + if (EnterTimeUs <= EndUs) + { + uint32_t& MustKeep = MustKeepPacketByThread[ThreadId]; + if (PacketIndex > MustKeep) + { + MustKeep = PacketIndex; + } + } + } + } + + if (!HasAny) + { + return; + } + + Range& R = PacketRanges[PacketIndex]; + R.MinUs = std::min(R.MinUs, BatchMinUs); + R.MaxUs = std::max(R.MaxUs, BatchMaxUs); + } + + uint64 m_Freq = 0; + uint64 m_Base = 0; + uint64 m_UsDiv = 1; + ReciprocalU64 m_UsDivRecip; + + // Per-thread open scope stack, carrying the Enter times in microseconds + // from trace start. Entries are pushed on Enter and popped on Leave; the + // stack may contain unclosed entries when decoding ends (scopes that + // outlive the captured trace). + eastl::hash_map<uint32_t, eastl::vector<uint32_t>> m_OpenScopes; +}; + +////////////////////////////////////////////////////////////////////////////// +// Common trace iteration + +struct TraceSummary +{ + eastl::map<uint32_t, std::pair<std::string, uint64_t>> TypeInfo; + eastl::set<uint16_t> Threads; + uint64_t TotalEvents = 0; +}; + +template<typename ParcelCallback> +static TraceSummary +IterateTrace(::DataSource& Source, ParcelCallback OnParcel, const zen::trace_detail::ProgressCallback& OnProgress = {}) +{ + TraceSummary Summary; + + try + { + uint64_t TotalFileBytes = uint64_t(std::max(Source.get_size(), int64(0))); + + ::Allocator TraceAllocator; + ::Preamble Pream(Source, TraceAllocator); + ::Transport Xport = Pream.get_transport(); + ::Protocol Proto = Pream.get_protocol(); + + ::Packet Packets[128]; + ::EventParcel Parcel; + + while (::Bundle Bndl = Xport.read_packets(Packets)) + { + Parcel.reset(); + Proto.read(Parcel, Bndl); + + OnParcel(Parcel); + + for (const ::Type* TraceType : Parcel.new_types) + { + auto [LoggerName, EventName] = TraceType->get_name(); + std::string TypeName = fmt::format("{}.{}", std::string_view(LoggerName), std::string_view(EventName)); + Summary.TypeInfo[TraceType->get_uid()] = {std::move(TypeName), 0}; + } + + for (const ::Event& Ev : Parcel.events) + { + Summary.TotalEvents++; + Summary.Threads.insert(Ev.thread_id); + + auto It = Summary.TypeInfo.find(Ev.uid); + if (It != Summary.TypeInfo.end()) + { + It->second.second++; + } + } + + if (OnProgress) + { + OnProgress(Xport.tell(), TotalFileBytes, Summary.TotalEvents); + } + } + } + catch (const DataStream::Eof&) + { + } + catch (const Exception::StreamError& E) + { + throw std::runtime_error(fmt::format("Trace stream error at position {}: {} (value: {})", E.position, E.message, E.value)); + } + + return Summary; +} + +// Print session metadata +static void +PrintSessionInfo(const SessionAnalyzer& SessionAn) +{ + const zen::trace_detail::SessionInfo& Sess = SessionAn.Session; + if (!Sess.HasSession) + { + return; + } + + ZEN_CONSOLE("Platform: {}", Sess.Platform); + ZEN_CONSOLE("App: {}", Sess.AppName); + if (!Sess.ProjectName.empty()) + { + ZEN_CONSOLE("Project: {}", Sess.ProjectName); + } + if (!Sess.Branch.empty()) + { + ZEN_CONSOLE("Branch: {}", Sess.Branch); + } + if (!Sess.BuildVersion.empty()) + { + ZEN_CONSOLE("Build: {}", Sess.BuildVersion); + } + if (Sess.Changelist) + { + ZEN_CONSOLE("Changelist: {}", Sess.Changelist); + } + if (!Sess.CommandLine.empty()) + { + ZEN_CONSOLE("CommandLine: {}", Sess.CommandLine); + } + ZEN_CONSOLE(""); +} + +// Print thread names +static void +PrintThreadInfo(const SessionAnalyzer& SessionAn) +{ + if (SessionAn.ThreadNames.empty()) + { + return; + } + + eastl::vector<std::pair<uint32_t, const zen::trace_detail::ThreadInfoEntry*>> ThreadsSorted; + for (const auto& [Tid, Info] : SessionAn.ThreadNames) + { + ThreadsSorted.emplace_back(Tid, &Info); + } + eastl::sort(ThreadsSorted.begin(), ThreadsSorted.end(), [](const auto& A, const auto& B) { + return A.second->SortHint < B.second->SortHint; + }); + + ZEN_CONSOLE(""); + ZEN_CONSOLE("Threads:"); + ZEN_CONSOLE(""); + ZEN_CONSOLE("{:>6} {:>10} {}", "TID", "SystemID", "Name"); + ZEN_CONSOLE("{:-<{}}", "", 6 + 10 + 40 + 4); + for (const auto& [Tid, Info] : ThreadsSorted) + { + ZEN_CONSOLE("{:>6} {:>10} {}", Tid, Info->SystemId, Info->Name); + } +} + +// Print trace channel info +static void +PrintChannelInfo(const SessionAnalyzer& SessionAn) +{ + if (SessionAn.Channels.empty()) + { + return; + } + + eastl::vector<const zen::trace_detail::ChannelInfo*> ChannelsSorted; + for (const auto& [Id, Info] : SessionAn.Channels) + { + ChannelsSorted.push_back(&Info); + } + eastl::sort(ChannelsSorted.begin(), ChannelsSorted.end(), [](const auto* A, const auto* B) { return A->Name < B->Name; }); + + ZEN_CONSOLE(""); + ZEN_CONSOLE("Trace Channels:"); + ZEN_CONSOLE(""); + for (const zen::trace_detail::ChannelInfo* Ch : ChannelsSorted) + { + std::string_view State = Ch->Enabled ? "enabled" : "disabled"; + if (Ch->ReadOnly) + { + ZEN_CONSOLE(" {} ({}, read-only)", Ch->Name, State); + } + else + { + ZEN_CONSOLE(" {} ({})", Ch->Name, State); + } + } +} + +} // namespace + +////////////////////////////////////////////////////////////////////////////// + +namespace zen::trace_detail { + +std::filesystem::path +ResolveTraceFile(const std::filesystem::path& Input, cxxopts::Options& HelpOptions) +{ + if (Input.empty()) + { + throw zen::OptionParseException("File path is required", HelpOptions.help()); + } + + std::filesystem::path FilePath = std::filesystem::absolute(Input); + if (!std::filesystem::exists(FilePath)) + { + throw std::runtime_error(fmt::format("File not found: {}", FilePath)); + } + + return FilePath; +} + +void +RunInspect(const std::filesystem::path& FilePath) +{ + ::DataSource Source(FilePath); + + SessionAnalyzer SessionAn; + ::Dispatcher Dispatch; + Dispatch.add_analyzer(SessionAn); + + // Collect type schemas + struct TypeSchema + { + std::string FullName; + uint32_t Uid = 0; + uint32_t FieldCount = 0; + uint32_t Flags = 0; + uint64_t EventCount = 0; + eastl::vector<std::string> FieldNames; + eastl::vector<uint32_t> FieldSizes; + eastl::vector<uint32_t> FieldTypeInfos; + }; + + eastl::map<uint32_t, TypeSchema> Schemas; + + TraceSummary Summary = IterateTrace(Source, [&](const ::EventParcel& Parcel) { + Dispatch.on_parcel(Parcel); + + for (const ::Type* TraceType : Parcel.new_types) + { + auto [LoggerName, EventName] = TraceType->get_name(); + uint32_t Uid = TraceType->get_uid(); + + TypeSchema& Schema = Schemas[Uid]; + Schema.FullName = fmt::format("{}.{}", std::string_view(LoggerName), std::string_view(EventName)); + Schema.Uid = Uid; + Schema.FieldCount = TraceType->get_field_count(); + Schema.Flags = 0; + if (TraceType->has_flag(TYPE_FLAG_IMPORTANT)) + { + Schema.Flags |= TYPE_FLAG_IMPORTANT; + } + if (TraceType->has_flag(TYPE_FLAG_AUX)) + { + Schema.Flags |= TYPE_FLAG_AUX; + } + + for (uint32_t I = 0; I < Schema.FieldCount; I++) + { + auto [FieldName, Field] = TraceType->get_field_info(I); + Schema.FieldNames.emplace_back(FieldName); + Schema.FieldSizes.push_back(Field.get_size()); + Schema.FieldTypeInfos.push_back(Field.get_type_info()); + } + } + + for (const ::Event& Ev : Parcel.events) + { + auto It = Schemas.find(Ev.uid); + if (It != Schemas.end()) + { + It->second.EventCount++; + } + } + }); + + // -- Session info -- + PrintSessionInfo(SessionAn); + + ZEN_CONSOLE("Trace: {}", FilePath); + ZEN_CONSOLE("Size: {}", zen::NiceBytes(uint64_t(std::filesystem::file_size(FilePath)))); + ZEN_CONSOLE("Events: {}", zen::ThousandsNum(Summary.TotalEvents)); + ZEN_CONSOLE("Threads: {}", Summary.Threads.size()); + ZEN_CONSOLE("Types: {}", Schemas.size()); + + // -- Thread names -- + PrintThreadInfo(SessionAn); + + // -- Trace channels -- + PrintChannelInfo(SessionAn); + + // -- Event schemas -- + ZEN_CONSOLE(""); + ZEN_CONSOLE("Event Schemas:"); + ZEN_CONSOLE(""); + + eastl::vector<const TypeSchema*> SortedSchemas; + SortedSchemas.reserve(Schemas.size()); + for (const auto& [Uid, Schema] : Schemas) + { + SortedSchemas.push_back(&Schema); + } + eastl::sort(SortedSchemas.begin(), SortedSchemas.end(), [](const auto* A, const auto* B) { return A->FullName < B->FullName; }); + + auto FieldTypeStr = [](uint32_t TypeInfo, uint32_t Size) -> std::string_view { + uint32_t Cat = TypeInfo & TYPE_INFO_CAT_MASK; + if (Cat == TYPE_INFO_CAT_ARRAY) + { + return "array"; + } + if (Cat == TYPE_INFO_CAT_FLOAT) + { + return (Size == 8) ? "float64" : "float32"; + } + bool IsSigned = (TypeInfo & TYPE_INFO_SPECIAL_MASK) == TYPE_INFO_SPECIAL_SIGNED; + switch (Size) + { + case 1: + return IsSigned ? "int8" : "uint8"; + case 2: + return IsSigned ? "int16" : "uint16"; + case 4: + return IsSigned ? "int32" : "uint32"; + case 8: + return IsSigned ? "int64" : "uint64"; + default: + return "unknown"; + } + }; + + for (const TypeSchema* Schema : SortedSchemas) + { + std::string Flags; + if (Schema->Flags & TYPE_FLAG_IMPORTANT) + { + Flags += " [important]"; + } + if (Schema->Flags & TYPE_FLAG_AUX) + { + Flags += " [aux]"; + } + + ZEN_CONSOLE("{} (uid={}, events={}){}", Schema->FullName, Schema->Uid, zen::ThousandsNum(Schema->EventCount), Flags); + + for (uint32_t I = 0; I < Schema->FieldCount; I++) + { + ZEN_CONSOLE(" {} {}", FieldTypeStr(Schema->FieldTypeInfos[I], Schema->FieldSizes[I]), Schema->FieldNames[I]); + } + + if (Schema->FieldCount > 0) + { + ZEN_CONSOLE(""); + } + } +} + +// Build a single LOD level by merging Lod0 scopes below the given resolution. +// Lod0 must already be sorted by BeginUs. Safe to call concurrently for +// different (Level, Resolution) pairs sharing the same Lod0. +static void +BuildSingleLod(const eastl::vector<TimelineScope>& Lod0, TimelineDetailLevel& Level, uint32_t Resolution) +{ + Level.ResolutionUs = Resolution; + + // Per-depth merge accumulators. Since depths are typically small (< 64), + // a flat array indexed by depth is more cache-friendly than a hash map. + struct PendingMerge + { + uint32_t BeginUs = 0; + uint32_t EndUs = 0; + uint32_t NameId = 0; + uint32_t MaxChildDur = 0; + uint16_t Depth = 0; + uint16_t Count = 0; + bool Active = false; + }; + + eastl::vector<PendingMerge> Pending(64); // grows if needed + + auto FlushPending = [&Level](PendingMerge& P) { + if (!P.Active) + { + return; + } + Level.Scopes.push_back(TimelineScope{ + .BeginUs = P.BeginUs, + .DurationUs = P.EndUs - P.BeginUs, + .NameId = P.NameId, + .Depth = P.Depth, + .MergeCount = P.Count, + }); + P.Active = false; + }; + + // Single O(n) sweep over LOD 0 scopes (sorted by BeginUs). For each + // depth, merge adjacent small scopes that fall within one resolution + // bucket of each other. Large scopes (>= Resolution) pass through. + for (const TimelineScope& Scope : Lod0) + { + uint16_t Depth = Scope.Depth; + if (Depth >= Pending.size()) + { + Pending.resize(Depth + 1); + } + + if (Scope.DurationUs >= Resolution) + { + // Large scope -- flush any pending merge for this depth, + // then emit the scope un-merged. + FlushPending(Pending[Depth]); + Level.Scopes.push_back(TimelineScope{ + .BeginUs = Scope.BeginUs, + .DurationUs = Scope.DurationUs, + .NameId = Scope.NameId, + .Depth = Scope.Depth, + .MergeCount = 1, + }); + continue; + } + + PendingMerge& P = Pending[Depth]; + uint32_t EndUs = Scope.BeginUs + Scope.DurationUs; + + if (P.Active && Scope.BeginUs < P.EndUs + Resolution) + { + // Extend the pending merge. + if (EndUs > P.EndUs) + { + P.EndUs = EndUs; + } + ++P.Count; + if (Scope.DurationUs > P.MaxChildDur) + { + P.MaxChildDur = Scope.DurationUs; + P.NameId = Scope.NameId; + } + } + else + { + // Start a new pending merge (flush previous if any). + FlushPending(P); + P.BeginUs = Scope.BeginUs; + P.EndUs = EndUs; + P.NameId = Scope.NameId; + P.MaxChildDur = Scope.DurationUs; + P.Depth = Scope.Depth; + P.Count = 1; + P.Active = true; + } + } + + // Flush remaining per-depth accumulators. + for (PendingMerge& P : Pending) + { + FlushPending(P); + } + + // Sort by (BeginUs, Depth) -- the per-depth flush may have interleaved + // entries from different depths. Tie-breaking on depth keeps the + // ordering consistent with LOD 0 (parents before nested children) so + // the front-end never sees a child rendered before its parent. + eastl::sort(Level.Scopes.begin(), Level.Scopes.end(), [](const TimelineScope& A, const TimelineScope& B) { + if (A.BeginUs != B.BeginUs) + { + return A.BeginUs < B.BeginUs; + } + return A.Depth < B.Depth; + }); +} + +void +BuildTimelineLods(ThreadTimeline& Timeline) +{ + if (Timeline.Scopes.empty()) + { + return; + } + + for (size_t LodIdx = 0; LodIdx < kTimelineLodCount; ++LodIdx) + { + BuildSingleLod(Timeline.Scopes, Timeline.DetailLevels[LodIdx], kTimelineLodResolutions[LodIdx]); + } +} + +namespace { + + // Post-iteration phases, extracted from BuildTraceModel for clarity. Each one + // runs after the event-iteration pass has populated the analyzers and mutates + // only the pieces of TraceModel it owns. + + void ComputeScopeStats(const TimelineAnalyzer& TimelineAn, TraceModel& Model) + { + const eastl::vector<std::string>& ScopeNames = TimelineAn.ScopeNames(); + eastl::vector<Distribution> Dists(ScopeNames.size()); + eastl::vector<uint32_t> Mins(ScopeNames.size(), ~0u); + eastl::vector<uint32_t> Maxs(ScopeNames.size(), 0u); + + for (const auto& [Tid, Thread] : TimelineAn.Threads()) + { + for (const TimelineScope& Scope : Thread.Scopes) + { + if (Scope.NameId >= Dists.size()) + { + continue; + } + Dists[Scope.NameId].add(double(Scope.DurationUs)); + Mins[Scope.NameId] = std::min(Mins[Scope.NameId], Scope.DurationUs); + Maxs[Scope.NameId] = std::max(Maxs[Scope.NameId], Scope.DurationUs); + } + } + + Model.ScopeStats.reserve(ScopeNames.size()); + for (size_t I = 0; I < ScopeNames.size(); ++I) + { + if (Dists[I].Count() == 0) + { + continue; + } + CpuScopeStat Entry; + Entry.Name = ScopeNames[I]; + Entry.Count = Dists[I].Count(); + Entry.MinUs = Mins[I]; + Entry.MaxUs = Maxs[I]; + Entry.MeanUs = Dists[I].Mean(); + Entry.StdDevUs = Dists[I].StdDev(); + Model.ScopeStats.push_back(std::move(Entry)); + } + eastl::sort(Model.ScopeStats.begin(), Model.ScopeStats.end(), [](const CpuScopeStat& A, const CpuScopeStat& B) { + return A.Count > B.Count; + }); + } + + // Translate each LogEntry's captured CategoryIndex (a sequential id keyed on + // the source category pointer) into the flat LogCategories index the frontend + // consumes. Entries whose category pointer never got a matching LogCategory + // event are bucketed into a synthetic "(unknown)" category. + void ResolveLogCategories(LogAnalyzer& LogAn, TraceModel& Model) + { + const eastl::hash_map<uint64_t, uint32_t>& CategoryPtrToSeqIdx = LogAn.CategoryPointerIndex(); + + eastl::hash_map<uint64_t, uint32_t> RealPtrToFlatIdx; + Model.LogCategories = LogAn.BuildCategories(RealPtrToFlatIdx); + + const uint32_t UnknownIdx = uint32_t(Model.LogCategories.size()); + Model.LogCategories.push_back(LogCategoryInfo{.Name = "(unknown)", .DefaultVerbosity = 0}); + + eastl::vector<uint32_t> SeqToFlat(CategoryPtrToSeqIdx.size(), UnknownIdx); + for (const auto& [Ptr, SeqIdx] : CategoryPtrToSeqIdx) + { + auto It = RealPtrToFlatIdx.find(Ptr); + if (It != RealPtrToFlatIdx.end()) + { + SeqToFlat[SeqIdx] = It->second; + } + } + + Model.LogEntries = LogAn.MutableEntries(); + for (LogEntry& E : Model.LogEntries) + { + E.CategoryIndex = (E.CategoryIndex < SeqToFlat.size()) ? SeqToFlat[E.CategoryIndex] : UnknownIdx; + } + + eastl::sort(Model.LogEntries.begin(), Model.LogEntries.end(), [](const LogEntry& A, const LogEntry& B) { + return A.TimeUs < B.TimeUs; + }); + } + + // Finalize any still-open regions, group by category, and greedily pack each + // category's regions into non-overlapping lanes so the frontend can stack them + // without re-running collision detection. + void BuildRegionCategories(eastl::vector<RegionEntry>&& AllRegions, uint32_t TraceEndUs, TraceModel& Model) + { + for (RegionEntry& R : AllRegions) + { + if (R.EndUs == ~uint32_t(0)) + { + R.EndUs = TraceEndUs; + } + if (R.EndUs < R.BeginUs) + { + R.EndUs = R.BeginUs; + } + } + + eastl::map<std::string, eastl::vector<RegionEntry>> ByCategory; + for (RegionEntry& R : AllRegions) + { + ByCategory[R.Category].push_back(std::move(R)); + } + + for (auto& [CatName, Regions] : ByCategory) + { + eastl::sort(Regions.begin(), Regions.end(), [](const RegionEntry& A, const RegionEntry& B) { + if (A.BeginUs != B.BeginUs) + { + return A.BeginUs < B.BeginUs; + } + return A.EndUs < B.EndUs; + }); + + eastl::vector<uint32_t> LaneEndUs; + uint32_t MaxLane = 0; + for (RegionEntry& R : Regions) + { + uint16_t Depth = 0; + bool Assigned = false; + for (size_t I = 0; I < LaneEndUs.size(); ++I) + { + if (LaneEndUs[I] <= R.BeginUs) + { + Depth = uint16_t(I); + LaneEndUs[I] = R.EndUs; + Assigned = true; + break; + } + } + if (!Assigned) + { + Depth = uint16_t(LaneEndUs.size()); + LaneEndUs.push_back(R.EndUs); + } + R.Depth = Depth; + if (Depth + 1u > MaxLane) + { + MaxLane = Depth + 1u; + } + } + + RegionCategory Cat; + Cat.Name = CatName; + Cat.LaneCount = MaxLane; + Cat.Regions = std::move(Regions); + Model.RegionCategories.push_back(std::move(Cat)); + } + + // Sort: uncategorized (empty name) first, then alphabetical. + eastl::sort(Model.RegionCategories.begin(), Model.RegionCategories.end(), [](const RegionCategory& A, const RegionCategory& B) { + if (A.Name.empty() != B.Name.empty()) + { + return A.Name.empty(); + } + return A.Name < B.Name; + }); + } + + // Map callstack frame addresses to (module, offset) pairs using a sorted + // (Base, End) lookup over the already-populated Model.Modules. + void ResolveCallstacks(const ModuleAnalyzer& ModuleAn, + const CallstackAnalyzer& CallstackAn, + AllocationAnalyzer& AllocAn, + TraceModel& Model) + { + const auto& RawCallstacks = CallstackAn.RawCallstacks(); + + struct ModuleLookup + { + uint64_t Base; + uint64_t End; + uint32_t ModelIndex; + }; + eastl::vector<ModuleLookup> Lookup; + Lookup.reserve(ModuleAn.ModulesByBase.size()); + for (const auto& [Base, Info] : ModuleAn.ModulesByBase) + { + for (uint32_t I = 0; I < Model.Modules.size(); ++I) + { + if (Model.Modules[I].Base == Base) + { + Lookup.push_back({Base, Base + Info.Size, I}); + break; + } + } + } + eastl::sort(Lookup.begin(), Lookup.end(), [](const ModuleLookup& A, const ModuleLookup& B) { return A.Base < B.Base; }); + + auto ResolveFrame = [&Lookup](uint64_t Address) -> ResolvedFrame { + ResolvedFrame F; + F.Address = Address; + auto It = eastl::upper_bound(Lookup.begin(), Lookup.end(), Address, [](uint64_t Addr, const ModuleLookup& M) { + return Addr < M.Base; + }); + if (It != Lookup.begin()) + { + --It; + if (Address < It->End) + { + F.ModuleIndex = It->ModelIndex; + F.Offset = Address - It->Base; + } + } + return F; + }; + + eastl::vector<uint32_t> SortedCallstackIds; + SortedCallstackIds.reserve(RawCallstacks.size()); + for (const auto& [Id, RawFrames] : RawCallstacks) + { + ZEN_UNUSED(RawFrames); + SortedCallstackIds.push_back(Id); + } + eastl::sort(SortedCallstackIds.begin(), SortedCallstackIds.end()); + + Model.Callstacks.reserve(RawCallstacks.size()); + for (uint32_t Id : SortedCallstackIds) + { + auto RawIt = RawCallstacks.find(Id); + ZEN_ASSERT(RawIt != RawCallstacks.end()); + const eastl::vector<uint64_t>& RawFrames = RawIt->second; + + CallstackEntry Entry; + Entry.Id = Id; + Entry.Frames.reserve(RawFrames.size()); + for (uint64_t Addr : RawFrames) + { + Entry.Frames.push_back(ResolveFrame(Addr)); + } + Model.Callstacks.push_back(std::move(Entry)); + } + + Model.CallstackStats = AllocAn.BuildCallstackStats(); + Model.ChurnStats = AllocAn.BuildChurnStats(~uint64_t(0)); + Model.AllocSizeHistogram = AllocAn.BuildSizeHistogram(); + } + +} // namespace + +TraceModel +BuildTraceModel(const std::filesystem::path& FilePath, WorkerThreadPool& ThreadPool, const ProgressCallback& OnProgress) +{ + ::DataSource Source(FilePath); + + TraceTiming Timing; + + SessionAnalyzer SessionAn; + ModuleAnalyzer ModuleAn; + MetadataRegistry MetadataReg; + TimelineAnalyzer TimelineAn(&MetadataReg, &Timing); + LogAnalyzer LogAn(&Timing); + BookmarksAnalyzer BookmarkAn(&Timing); + CsvProfilerAnalyzer CsvAn(&Timing); + AllocationAnalyzer AllocAn(&Timing); + CallstackAnalyzer CallstackAn; + + // Tourist's Dispatcher only supports one subscription per event type, so we + // cannot run CpuAnalyzer alongside TimelineAnalyzer -- CpuAnalyzer would + // claim the CpuProfiler.Event* types first and TimelineAnalyzer would + // never receive any events. Instead, TimelineAnalyzer captures every + // scope interval and we derive the aggregate statistics from those + // intervals in a cheap post-pass below. + ::Dispatcher Dispatch; + Dispatch.add_analyzer(SessionAn); + Dispatch.add_analyzer(ModuleAn); + Dispatch.add_analyzer(MetadataReg); + Dispatch.add_analyzer(TimelineAn); + Dispatch.add_analyzer(LogAn); + Dispatch.add_analyzer(BookmarkAn); + Dispatch.add_analyzer(CsvAn); + Dispatch.add_analyzer(AllocAn); + Dispatch.add_analyzer(CallstackAn); + + zen::Stopwatch Timer; + TraceSummary Summary = IterateTrace( + Source, + [&](const ::EventParcel& Parcel) { Dispatch.on_parcel(Parcel); }, + OnProgress); + ZEN_INFO("Trace iteration complete: {} events in {}", + zen::ThousandsNum(Summary.TotalEvents), + zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + + { + uint32_t StartUs = (TimelineAn.MinBeginUs() == ~0u) ? 0u : TimelineAn.MinBeginUs(); + uint32_t EndUs = TimelineAn.MaxEndUs(); + uint64_t DurationMs = (EndUs > StartUs) ? (uint64_t(EndUs - StartUs) + 500) / 1000 : 0; + ZEN_INFO("Trace duration: {}", zen::NiceTimeSpanMs(DurationMs)); + } + + TraceModel Model; + Model.FilePath = FilePath; + Model.FileSize = uint64_t(std::filesystem::file_size(FilePath)); + Model.TotalEvents = Summary.TotalEvents; + Model.ParseTimeMs = Timer.GetElapsedTimeMs(); + Model.Session = SessionAn.Session; + + // Event type counts (sorted by count descending) + Model.EventTypeCounts.reserve(Summary.TypeInfo.size()); + for (auto& [Uid, Info] : Summary.TypeInfo) + { + Model.EventTypeCounts.push_back({std::move(Info.first), Info.second}); + } + eastl::sort(Model.EventTypeCounts.begin(), Model.EventTypeCounts.end(), [](const auto& A, const auto& B) { return A.Count > B.Count; }); + + // Flatten and sort threads by sort hint + Model.Threads.reserve(SessionAn.ThreadNames.size()); + for (const auto& [Tid, Info] : SessionAn.ThreadNames) + { + Model.Threads.push_back(Info); + } + eastl::sort(Model.Threads.begin(), Model.Threads.end(), [](const ThreadInfoEntry& A, const ThreadInfoEntry& B) { + return A.SortHint < B.SortHint; + }); + + // Flatten and sort channels by name + Model.Channels.reserve(SessionAn.Channels.size()); + for (const auto& [Id, Info] : SessionAn.Channels) + { + Model.Channels.push_back(Info); + } + eastl::sort(Model.Channels.begin(), Model.Channels.end(), [](const ChannelInfo& A, const ChannelInfo& B) { return A.Name < B.Name; }); + + { + ExtendableStringBuilder<512> Enabled; + for (const ChannelInfo& Ch : Model.Channels) + { + if (Ch.Enabled) + { + if (Enabled.Size() > 0) + { + Enabled.Append(", "); + } + Enabled.Append(Ch.Name); + } + } + if (Enabled.Size() > 0) + { + ZEN_INFO("Enabled channels: {}", Enabled); + } + } + + // Flatten and sort modules by name + Model.Modules.reserve(ModuleAn.ModulesByBase.size()); + for (const auto& [Base, Info] : ModuleAn.ModulesByBase) + { + Model.Modules.push_back(Info); + } + eastl::sort(Model.Modules.begin(), Model.Modules.end(), [](const ModuleInfo& A, const ModuleInfo& B) { return A.Name < B.Name; }); + + // CPU scope statistics and timeline building read from TimelineAn + // independently and write to separate Model fields, so overlap them. + Model.ScopeNames = TimelineAn.ScopeNames(); + + ZEN_INFO("Computing CPU scope statistics ({} scope names)", TimelineAn.ScopeNames().size()); + + // Kick off scope stats on a worker -- runs concurrently with the + // timeline copy + sort below. + Latch StatsLatch(1); + ThreadPool.ScheduleWork( + [&StatsLatch, &Model, &TimelineAn]() { + auto _ = MakeGuard([&StatsLatch]() { StatsLatch.CountDown(); }); + ComputeScopeStats(TimelineAn, Model); + }, + WorkerThreadPool::EMode::EnableBacklog); + + // Timelines -- build per-thread sort + LODs in parallel. + { + const auto& Threads = TimelineAn.Threads(); + size_t TotalScopes = 0; + for (const auto& [Tid, Thread] : Threads) + { + TotalScopes += Thread.Scopes.size(); + } + ZEN_INFO("Building timelines: {} threads, {} scopes (sort + LODs)", Threads.size(), zen::ThousandsNum(TotalScopes)); + Model.Timelines.resize(Threads.size()); + + // Populate timeline metadata on the main thread (cheap lookups). + size_t Idx = 0; + for (const auto& [Tid, Thread] : Threads) + { + ThreadTimeline& Timeline = Model.Timelines[Idx++]; + Timeline.ThreadId = Tid; + auto It = SessionAn.ThreadNames.find(Tid); + if (It != SessionAn.ThreadNames.end()) + { + Timeline.Name = It->second.Name; + Timeline.SortHint = It->second.SortHint; + } + Timeline.Scopes = Thread.Scopes; + } + + // Phase 1: Sort LOD 0 scopes per thread. + // ParallelSort fans out internally using the pool, so it must be + // called from the main thread to avoid nested fan-out deadlocks. + // Small timelines are dispatched to workers first (they just call + // eastl::sort -- no nesting). Then large ones are sorted one at a + // time from the main thread with full pool utilisation each. + // + // Tie-break on Depth so that scopes which start at the same micro + // timestamp come out parent-first (lower depth wins). This keeps + // the scope ordering well-defined and lets the front-end rely on + // outer scopes appearing before their nested children regardless + // of the order the analyzer happened to emit them. + { + auto Cmp = [](const TimelineScope& A, const TimelineScope& B) { + if (A.BeginUs != B.BeginUs) + { + return A.BeginUs < B.BeginUs; + } + return A.Depth < B.Depth; + }; + + if constexpr (kUseParallelSort) + { + constexpr size_t kParallelThreshold = 65536; + + // Dispatch small timelines to workers. + Latch SmallLatch(1); + for (size_t I = 0; I < Model.Timelines.size(); ++I) + { + if (Model.Timelines[I].Scopes.size() >= kParallelThreshold) + { + continue; + } + SmallLatch.AddCount(1); + ThreadPool.ScheduleWork( + [&SmallLatch, &Cmp, &Timeline = Model.Timelines[I]]() { + auto _ = MakeGuard([&SmallLatch]() { SmallLatch.CountDown(); }); + eastl::sort(Timeline.Scopes.begin(), Timeline.Scopes.end(), Cmp); + }, + WorkerThreadPool::EMode::EnableBacklog); + } + SmallLatch.CountDown(); + SmallLatch.Wait(); + + // Sort large timelines from the main thread so ParallelSort + // can fan out across the (now idle) pool without deadlocking. + for (ThreadTimeline& Timeline : Model.Timelines) + { + if (Timeline.Scopes.size() >= kParallelThreshold) + { + zen::ParallelSort(ThreadPool, Timeline.Scopes.begin(), Timeline.Scopes.end(), Cmp); + } + } + } + else + { + Latch SortLatch(1); + for (size_t I = 0; I < Model.Timelines.size(); ++I) + { + SortLatch.AddCount(1); + ThreadPool.ScheduleWork( + [&SortLatch, &Cmp, &Timeline = Model.Timelines[I]]() { + auto _ = MakeGuard([&SortLatch]() { SortLatch.CountDown(); }); + eastl::sort(Timeline.Scopes.begin(), Timeline.Scopes.end(), Cmp); + }, + WorkerThreadPool::EMode::EnableBacklog); + } + SortLatch.CountDown(); + SortLatch.Wait(); + } + } + + // Phase 2: Build LOD levels -- one task per (thread, LOD) pair. + // Flat dispatch avoids nested fan-out which could deadlock the pool. + Latch LodLatch(1); + for (size_t I = 0; I < Model.Timelines.size(); ++I) + { + if (Model.Timelines[I].Scopes.empty()) + { + continue; + } + for (size_t L = 0; L < kTimelineLodCount; ++L) + { + LodLatch.AddCount(1); + ThreadPool.ScheduleWork( + [&LodLatch, &Timeline = Model.Timelines[I], L]() { + auto _ = MakeGuard([&LodLatch]() { LodLatch.CountDown(); }); + BuildSingleLod(Timeline.Scopes, Timeline.DetailLevels[L], kTimelineLodResolutions[L]); + }, + WorkerThreadPool::EMode::EnableBacklog); + } + } + LodLatch.CountDown(); + LodLatch.Wait(); + } + eastl::sort(Model.Timelines.begin(), Model.Timelines.end(), [](const ThreadTimeline& A, const ThreadTimeline& B) { + return A.SortHint < B.SortHint; + }); + + Model.TraceStartUs = (TimelineAn.MinBeginUs() == ~0u) ? 0u : TimelineAn.MinBeginUs(); + Model.TraceEndUs = TimelineAn.MaxEndUs(); + + // Ensure scope stats computation (kicked off earlier) has finished. + StatsLatch.Wait(); + + ZEN_INFO("Processing {} log entries", zen::ThousandsNum(LogAn.Entries().size())); + ResolveLogCategories(LogAn, Model); + + ZEN_INFO("Sorting {} bookmarks, {} regions", BookmarkAn.MutableBookmarks().size(), BookmarkAn.MutableRegions().size()); + + // Bookmarks: move and sort by TimeUs. + Model.Bookmarks = std::move(BookmarkAn.MutableBookmarks()); + eastl::sort(Model.Bookmarks.begin(), Model.Bookmarks.end(), [](const Bookmark& A, const Bookmark& B) { return A.TimeUs < B.TimeUs; }); + + BuildRegionCategories(std::move(BookmarkAn.MutableRegions()), Model.TraceEndUs, Model); + + // CsvProfiler data + { + Model.CsvCategories = std::move(CsvAn.MutableCategories()); + Model.CsvStatDefs = std::move(CsvAn.MutableStatDefs()); + Model.CsvTimeSeries = CsvAn.BuildTimeSeries(); + Model.CsvEvents = std::move(CsvAn.MutableEvents()); + eastl::sort(Model.CsvEvents.begin(), Model.CsvEvents.end(), [](const auto& A, const auto& B) { return A.TimeUs < B.TimeUs; }); + Model.CsvMetadata = std::move(CsvAn.MutableMetadata()); + ZEN_INFO("CSV profiler: {} categories, {} stats, {} series, {} events", + Model.CsvCategories.size(), + Model.CsvStatDefs.size(), + Model.CsvTimeSeries.size(), + Model.CsvEvents.size()); + } + + // Memory allocation data + { + AllocAn.EmitFinalSample(Model.TraceEndUs); + Model.AllocSummary = AllocAn.Summary(); + + // Flatten heaps map into sorted vector + Model.Heaps.reserve(AllocAn.Heaps().size()); + for (const auto& [Id, Info] : AllocAn.Heaps()) + { + Model.Heaps.push_back(Info); + } + eastl::sort(Model.Heaps.begin(), Model.Heaps.end(), [](const HeapInfo& A, const HeapInfo& B) { return A.Id < B.Id; }); + + // Flatten tags map into sorted vector + Model.Tags.reserve(AllocAn.Tags().size()); + for (const auto& [Tag, Info] : AllocAn.Tags()) + { + Model.Tags.push_back(Info); + } + eastl::sort(Model.Tags.begin(), Model.Tags.end(), [](const TagInfo& A, const TagInfo& B) { return A.Tag < B.Tag; }); + + // Move timeline (already time-ordered from Marker events) + Model.MemoryTimeline = std::move(AllocAn.MutableTimeline()); + + // Flatten per-root-heap stats into sorted vector + Model.HeapStats.reserve(AllocAn.RootHeapStats().size()); + for (const auto& [HeapId, Stat] : AllocAn.RootHeapStats()) + { + Model.HeapStats.push_back(Stat); + } + eastl::sort(Model.HeapStats.begin(), Model.HeapStats.end(), [](const HeapStat& A, const HeapStat& B) { + return A.HeapId < B.HeapId; + }); + + if (Model.AllocSummary.HasMemoryData) + { + ZEN_INFO("Memory: {} allocs, {} frees, peak {}, {} live, {} timeline samples", + zen::ThousandsNum(Model.AllocSummary.TotalAllocs + Model.AllocSummary.TotalReallocAllocs), + zen::ThousandsNum(Model.AllocSummary.TotalFrees + Model.AllocSummary.TotalReallocFrees), + zen::NiceBytes(uint64_t(Model.AllocSummary.PeakBytes)), + zen::ThousandsNum(Model.AllocSummary.LiveAllocations), + zen::ThousandsNum(Model.MemoryTimeline.size())); + } + } + + ResolveCallstacks(ModuleAn, CallstackAn, AllocAn, Model); + ZEN_INFO("Callstacks: {} unique, {} with live allocations", + zen::ThousandsNum(Model.Callstacks.size()), + zen::ThousandsNum(Model.CallstackStats.size())); + + return Model; +} + +////////////////////////////////////////////////////////////////////////////// +// Trace trim +// +// The trim pipeline operates entirely at the raw packet level: a .utrace on +// disk is identical to the wire format (see src/zenserver/trace/tracerecorder.cpp +// for the capture-side passthrough), so trimming reduces to "copy the preamble, +// then copy only the packets we want to keep". We never re-encode or re-emit +// any events, which sidesteps the fact that Tourist has no writer path. +// +// The algorithm: +// +// 1. Slurp the input file into memory and walk raw packets using the +// [size:uint16][thread_id:uint16][payload] framing. This gives an ordered +// list of packet descriptors keyed by file offset. +// +// 2. Classify packets by their on-disk thread_id: +// TID_TYPE -> always keep (type definitions) +// TID_IMPORTANT -> always keep (events of types marked TYPE_FLAG_IMPORTANT, +// i.e. session info, thread names, channel state, +// log categories, CPU specs, etc.) +// TID_SYNC -> always keep (transport barriers) +// TID_NORMAL+ -> keep only if the packet's events overlap the window +// +// 3. For normal-thread packets, run Tourist's reader with a bundle of size 1 +// so each Proto.read() scatters exactly one raw packet before emitting any +// events. Before the read call, we record the file offset of the current +// packet as the "latest packet" for its thread. TrimAnalyzer then decodes +// CpuProfiler batch events and attributes their timestamp ranges back to +// that thread's latest packet. The attribution can drift if Tourist +// buffers multiple packets on one thread, but the failure mode is that +// earlier packets lose attribution and are conservatively retained. +// +// 4. Write the output: the preamble bytes verbatim, followed by the raw +// bytes of each kept packet in original order. There is no trailer; the +// Tourist reader catches DataStream::Eof at the end of the stream. +// +// Coarse per-packet precision is accepted by design: a packet straddling a +// window edge is kept in full. CpuProfiler batches are self-contained per +// packet (each re-derives cycles from the trace-wide StartCycle), so dropping +// packets does not desync delta decoding on surviving ones, and orphaned leave +// events from half-open scopes are silently ignored by decoders. + +namespace { + + struct TrimPacketDesc + { + uint64_t FileOffset = 0; // offset of the [size:uint16] header in the file + uint32_t Size = 0; // total size including the 4-byte header + uint16_t ThreadIdRaw = 0; // thread_id as stored on disk, including PACKET_FLAG_COMPRESSED + }; + + // Parses the .utrace preamble in place to determine the byte offset where + // packets begin. Mirrors Preamble::parse_header in Tourist so we can run the + // raw walker without spinning up a second DataSource. Throws on a malformed + // preamble. + static uint64_t ParsePreambleLength(const uint8_t* Data, uint64_t Size) + { + if (Size < 8) + { + throw zen::runtime_error("Trace file too small to contain a preamble ({} bytes)", Size); + } + + uint32_t Magic = 0; + std::memcpy(&Magic, Data, sizeof(uint32_t)); + if (Magic != 'TRC2') + { + throw zen::runtime_error("Unexpected trace file magic value 0x{:08x}", Magic); + } + + uint16_t MetaSize = 0; + std::memcpy(&MetaSize, Data + 4, sizeof(uint16_t)); + + // magic(4) + meta_size(2) + metadata + transport(1) + protocol(1) + uint64_t PreambleLen = uint64_t(4) + 2 + MetaSize + 1 + 1; + if (PreambleLen > Size) + { + throw zen::runtime_error("Trace preamble extends past end of file ({} > {})", PreambleLen, Size); + } + + return PreambleLen; + } + + // Walks raw packets starting at PreambleLen. Returns one TrimPacketDesc per + // packet in original stream order. The walker stops gracefully on truncated + // data so partial traces still produce a usable packet list. + static eastl::vector<TrimPacketDesc> WalkRawPackets(const uint8_t* Data, uint64_t Size, uint64_t PreambleLen) + { + eastl::vector<TrimPacketDesc> Packets; + uint64_t Offset = PreambleLen; + + while (Offset + 4 <= Size) + { + uint16_t PacketSize = 0; + uint16_t ThreadIdRaw = 0; + std::memcpy(&PacketSize, Data + Offset, sizeof(uint16_t)); + std::memcpy(&ThreadIdRaw, Data + Offset + 2, sizeof(uint16_t)); + + if (PacketSize < 4) + { + // Malformed size; stop walking and accept whatever we have. + break; + } + + if (Offset + PacketSize > Size) + { + // Truncated tail -- drop it. + break; + } + + TrimPacketDesc Desc; + Desc.FileOffset = Offset; + Desc.Size = PacketSize; + Desc.ThreadIdRaw = ThreadIdRaw; + Packets.push_back(Desc); + + Offset += PacketSize; + } + + return Packets; + } + +} // namespace + +void +RunTraceTrim(const TraceTrimArgs& Args) +{ + if (!(Args.EndSec > Args.StartSec)) + { + throw zen::runtime_error("Invalid trim range: start={} end={}", Args.StartSec, Args.EndSec); + } + + // --- Read the input file --- + zen::BasicFile InputFile(Args.InputPath, zen::BasicFile::Mode::kRead); + zen::IoBuffer InputBuffer = InputFile.ReadAll(); + InputFile.Close(); + + const uint8_t* FileBytes = static_cast<const uint8_t*>(InputBuffer.GetData()); + const uint64_t FileSize = InputBuffer.GetSize(); + + const uint64_t PreambleLen = ParsePreambleLength(FileBytes, FileSize); + + // --- Raw packet walk --- + eastl::vector<TrimPacketDesc> Packets = WalkRawPackets(FileBytes, FileSize, PreambleLen); + if (Packets.empty()) + { + throw zen::runtime_error("Trace file contains no packets"); + } + + // Initial keep classification: definitions, important events, sync are + // always retained. Normal-thread packets start as drop candidates and get + // promoted if their decoded time range overlaps the window. + eastl::vector<uint8_t> Keep(Packets.size(), 0); + size_t NumAlwaysKept = 0; + for (size_t I = 0; I < Packets.size(); ++I) + { + uint32_t Tid = Packets[I].ThreadIdRaw & ~PACKET_FLAG_COMPRESSED; + if (Tid == TID_TYPE || Tid == TID_IMPORTANT || Tid == TID_SYNC) + { + Keep[I] = 1; + ++NumAlwaysKept; + } + } + + // --- Time-range classification via Tourist (bundle of 1) --- + TrimAnalyzer TrimAn; + TrimAn.EndUs = (Args.EndSec * 1e6 > double(~uint32_t(0))) ? ~uint32_t(0) : uint32_t(Args.EndSec * 1e6); + ::Dispatcher Dispatch; + Dispatch.add_analyzer(TrimAn); + + { + ::DataSource Source(Args.InputPath); + ::Allocator TraceAllocator; + ::Preamble Pream(Source, TraceAllocator); + ::Transport Xport = Pream.get_transport(); + ::Protocol Proto = Pream.get_protocol(); + + ::Packet OnePacket[1]; + ::EventParcel Parcel; + + try + { + while (::Bundle Bndl = Xport.read_packets(OnePacket)) + { + if (Bndl.empty()) + { + break; + } + + const ::Packet& P = Bndl[0]; + uint32_t Tid = P.get_thread_id(); + + if (Tid >= TID_NORMAL && Tid != TID_SYNC) + { + // Tourist's Packet::get_index() is the same sequential + // packet counter as our raw walker's vector position, + // since both read the stream from the start in order. + TrimAn.LastPacketIndexByThread[Tid] = P.get_index(); + } + + Parcel.reset(); + Proto.read(Parcel, Bndl); + Dispatch.on_parcel(Parcel); + } + } + catch (const DataStream::Eof&) + { + } + catch (const Exception::StreamError& E) + { + throw zen::runtime_error("Trace stream error at position {}: {} (value: {})", E.position, E.message, E.value); + } + } + + // --- Apply the window filter --- + // + // Per-packet filtering in the middle of a thread's stream is unsafe: + // Tourist's event parser holds per-thread continuation state (see + // EventParser::_fragment / _missing in + // thirdparty/tourist/trace/src/protocol.cpp) so an event can straddle a + // packet boundary on a normal thread. Removing a packet from the middle + // leaves subsequent packets on the same thread decoded against the wrong + // position in an in-flight event and Tourist crashes. We therefore only + // drop packets in two safe ways: + // + // 1. Whole-thread drop: a thread whose attributed packets are all + // outside the window has every one of its packets dropped. No + // surviving packet references that thread, so there is no state + // machine to corrupt. + // + // 2. Per-thread tail truncation: for a thread that does have in-window + // activity, drop every packet AFTER the latest in-window packet on + // that thread. Tail drops are safe because no later packet on the + // same thread can be looking forward to the dropped bytes; the + // parser just ends its stream for that thread at the truncation + // point, exactly like a trace that naturally stopped recording. + // + // Threads for which we never attributed any CpuProfiler batch events are + // retained in full; we have no evidence about their time range and + // can't safely drop them. + const uint32_t StartUs = uint32_t(std::max(0.0, Args.StartSec) * 1e6); + const uint32_t EndUs = (Args.EndSec * 1e6 > double(~uint32_t(0))) ? ~uint32_t(0) : uint32_t(Args.EndSec * 1e6); + + struct ThreadInfo + { + bool HasAnyBatch = false; + bool HasInWindowBatch = false; + // First packet index on this thread whose attributed CPU batches are + // *entirely* past EndUs. Every packet on this thread with an index + // >= this value is safe to tail-drop. Defaults to size_t(-1) (no cut + // point) when the thread has no such packet. + size_t FirstPastWindowIdx = size_t(-1); + }; + eastl::hash_map<uint32_t, ThreadInfo> ThreadInfos; + + for (size_t I = 0; I < Packets.size(); ++I) + { + uint32_t Tid = Packets[I].ThreadIdRaw & ~PACKET_FLAG_COMPRESSED; + if (Tid < TID_NORMAL || Tid == TID_SYNC) + { + continue; + } + + auto RangeIt = TrimAn.PacketRanges.find(uint32_t(I)); + if (RangeIt == TrimAn.PacketRanges.end()) + { + continue; + } + + ThreadInfo& Info = ThreadInfos[Tid]; + Info.HasAnyBatch = true; + const auto& Range = RangeIt->second; + if (Range.MaxUs >= StartUs && Range.MinUs <= EndUs) + { + Info.HasInWindowBatch = true; + } + if (Range.MinUs > EndUs && I < Info.FirstPastWindowIdx) + { + Info.FirstPastWindowIdx = I; + } + } + + size_t NumThreadsKept = 0; + size_t NumThreadsDropped = 0; + for (const auto& [Tid, Info] : ThreadInfos) + { + if (Info.HasInWindowBatch) + { + ++NumThreadsKept; + } + else + { + ++NumThreadsDropped; + } + } + + size_t NumInWindow = 0; + size_t NumTailDropped = 0; + size_t NumUnattributed = 0; + size_t NumDropped = 0; + + for (size_t I = 0; I < Packets.size(); ++I) + { + if (Keep[I]) + { + continue; + } + + uint32_t Tid = Packets[I].ThreadIdRaw & ~PACKET_FLAG_COMPRESSED; + auto It = ThreadInfos.find(Tid); + if (It == ThreadInfos.end() || !It->second.HasAnyBatch) + { + // We have no evidence for this thread's time range. Retain all + // its packets conservatively to avoid breaking Tourist's per- + // thread parser state. + Keep[I] = 1; + ++NumUnattributed; + continue; + } + + if (!It->second.HasInWindowBatch) + { + // Thread's attributed packets are all outside the window -- drop + // every packet on this thread. + ++NumDropped; + continue; + } + + if (I >= It->second.FirstPastWindowIdx) + { + // Past the first entirely-after-window packet on this thread -- + // candidate for tail truncation. Before dropping, check whether + // this packet carries a Leave event that closes a scope whose + // Enter was at or before the window end. If so, we MUST keep it + // so the downstream analyzer can render the long-running scope; + // otherwise the scope would sit unmatched on the open stack. + auto MustKeepIt = TrimAn.MustKeepPacketByThread.find(Tid); + if (MustKeepIt != TrimAn.MustKeepPacketByThread.end() && I <= MustKeepIt->second) + { + Keep[I] = 1; + ++NumInWindow; + continue; + } + + ++NumTailDropped; + continue; + } + + Keep[I] = 1; + ++NumInWindow; + } + + // --- Write output --- + std::error_code Ec; + std::filesystem::create_directories(Args.OutputPath.parent_path(), Ec); + + zen::BasicFile OutputFile(Args.OutputPath, zen::BasicFile::Mode::kTruncate); + + uint64_t OutOffset = 0; + OutputFile.Write(FileBytes, PreambleLen, OutOffset); + OutOffset += PreambleLen; + + uint64_t KeptBytes = 0; + for (size_t I = 0; I < Packets.size(); ++I) + { + if (!Keep[I]) + { + continue; + } + OutputFile.Write(FileBytes + Packets[I].FileOffset, Packets[I].Size, OutOffset); + OutOffset += Packets[I].Size; + KeptBytes += Packets[I].Size; + } + + OutputFile.Flush(); + OutputFile.Close(); + + ZEN_CONSOLE("Trimmed trace written to {}", Args.OutputPath); + ZEN_CONSOLE(" Input: {} ({} packets)", zen::NiceBytes(FileSize), zen::ThousandsNum(Packets.size())); + ZEN_CONSOLE(" Output: {} ({} packets)", + zen::NiceBytes(OutOffset), + zen::ThousandsNum(NumAlwaysKept + NumInWindow + NumUnattributed)); + ZEN_CONSOLE(" Always kept: {} packets (types / important / sync)", zen::ThousandsNum(NumAlwaysKept)); + ZEN_CONSOLE(" Thread kept: {} packets from {} threads with in-window activity", + zen::ThousandsNum(NumInWindow), + zen::ThousandsNum(NumThreadsKept)); + ZEN_CONSOLE(" Thread dropped: {} packets from {} threads with no in-window activity", + zen::ThousandsNum(NumDropped), + zen::ThousandsNum(NumThreadsDropped)); + ZEN_CONSOLE(" Tail dropped: {} packets past the latest in-window packet on their thread", zen::ThousandsNum(NumTailDropped)); + ZEN_CONSOLE(" Unattributed: {} packets (retained conservatively)", zen::ThousandsNum(NumUnattributed)); + ZEN_UNUSED(KeptBytes); + + // --- Diagnostic: summarise the attributed time range distribution --- + { + uint32_t GlobalMin = ~0u; + uint32_t GlobalMax = 0; + for (const auto& [Idx, R] : TrimAn.PacketRanges) + { + GlobalMin = std::min(GlobalMin, R.MinUs); + GlobalMax = std::max(GlobalMax, R.MaxUs); + } + ZEN_CONSOLE(" Attributed: {} packets, window {:.3f}s .. {:.3f}s", + zen::ThousandsNum(TrimAn.PacketRanges.size()), + double(GlobalMin) / 1e6, + double(GlobalMax) / 1e6); + } +} + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_model.h b/src/zen/trace/trace_model.h new file mode 100644 index 000000000..bd6dcc674 --- /dev/null +++ b/src/zen/trace/trace_model.h @@ -0,0 +1,314 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "trace_memory.h" +#include "zen.h" + +#include <zencore/workthreadpool.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/vector.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <cstdint> +#include <filesystem> +#include <functional> +#include <string> + +namespace zen::trace_detail { + +// Shared trace timing state. Tourist's Dispatcher only allows one subscriber +// per event type, so only one analyzer can own the `$Trace.NewTrace` +// subscription. Other analyzers that need to convert absolute Cycle64 values +// read from this shared struct, which the owning analyzer fills in during its +// OnNewTrace callback. +struct TraceTiming +{ + uint64_t Freq = 0; + uint64_t Base = 0; + uint64_t UsDiv = 1; + + uint32_t CycleToTimeUs(uint64_t Cycle) const + { + uint64_t CycleFromStart = (Cycle >= Base) ? (Cycle - Base) : 0; + uint64_t D = (UsDiv > 0) ? UsDiv : 1; + return uint32_t((CycleFromStart + (D >> 1)) / D); + } +}; + +// Safely convert a tourist FieldStr to std::string, stripping trailing NULs +// and returning an empty string on failure. +std::string SafeFieldStr(class FieldStr&& Field); + +struct SessionInfo +{ + std::string Platform; + std::string AppName; + std::string ProjectName; + std::string CommandLine; + std::string Branch; + std::string BuildVersion; + uint32_t Changelist = 0; + uint8_t ConfigurationType = 0; + bool HasSession = false; +}; + +struct ThreadInfoEntry +{ + uint32_t ThreadId = 0; + std::string Name; + std::string GroupName; // from $Trace.ThreadGroupBegin/End bracketing, or synthesized by stripping a numeric suffix from Name + uint32_t SystemId = 0; + int32_t SortHint = 0; +}; + +struct ChannelInfo +{ + std::string Name; + bool Enabled = false; + bool ReadOnly = false; +}; + +// A DLL / shared library that was loaded (or seen already loaded) during the +// capture. Populated from the Diagnostics.Module{Init,Load,Unload} events +// which are all marked NoSync|Important, so they survive reconnects and our +// own trim filter. Load/unload timestamps aren't available because the events +// don't carry a Cycle field. +struct ModuleInfo +{ + std::string Name; // basename of FullPath + std::string FullPath; // full path as reported by the engine + uint64_t Base = 0; + uint32_t Size = 0; + bool Unloaded = false; // set when we see a matching ModuleUnload + eastl::vector<uint8_t> ImageId; // PDB GUID + Age, opaque -- for later symbol lookup +}; + +// UE verbosity values mirror ELogVerbosity::Type. We expose the raw integer +// so the frontend can map it to a label / color. +struct LogCategoryInfo +{ + std::string Name; + uint8_t DefaultVerbosity = 0; +}; + +struct LogEntry +{ + uint32_t TimeUs; // microseconds from the start of the trace + uint32_t CategoryIndex; // index into TraceModel::LogCategories (or ~0u) + uint8_t Verbosity; + int32_t Line; + std::string File; + std::string Message; +}; + +// Point-in-time marker emitted via TRACE_BOOKMARK / UE_TRACE_BOOKMARK. +// Each entry's Text has already been formatted (FormatString + FormatArgs +// substituted) during parsing. +struct Bookmark +{ + uint32_t TimeUs; + int32_t Line; + std::string File; + std::string Text; +}; + +// A named time range announced via Misc.RegionBegin / Misc.RegionEnd +// (or the newer *WithId variants). Depth is the lane index assigned by +// the analyzer's greedy overlap-avoidance pass. +struct RegionEntry +{ + uint32_t BeginUs; + uint32_t EndUs; // == TraceEndUs if still open at trace end + uint16_t Depth; + uint16_t Reserved; + std::string Name; + std::string Category; +}; + +// A group of regions sharing the same category label. Each category has its +// own lane namespace so depths are assigned independently. +struct RegionCategory +{ + std::string Name; // display name; empty categories get "Uncategorized" + uint32_t LaneCount = 0; + eastl::vector<RegionEntry> Regions; // sorted by BeginUs, Depth is per-category +}; + +struct CpuScopeStat +{ + std::string Name; + uint64_t Count = 0; + uint32_t MinUs = 0; + uint32_t MaxUs = 0; + double MeanUs = 0.0; + double StdDevUs = 0.0; +}; + +// Single CPU scope interval captured by TimelineAnalyzer. Packed for size: +// timelines can easily contain millions of entries. +struct TimelineScope +{ + uint32_t BeginUs; // microseconds from the start of the trace + uint32_t DurationUs; // scope duration in microseconds + uint32_t NameId; // index into TraceModel::ScopeNames + uint16_t Depth; // call-stack depth (0 == outermost) + uint16_t MergeCount; // 0 = raw (LOD 0), N>0 = N scopes merged (LOD 1+) +}; + +// Pre-computed detail level for a thread timeline. Each level merges scopes +// shorter than ResolutionUs into "macro scopes" carrying the dominant name +// (the name of the longest contributing scope). The merge count is stored in +// TimelineScope::MergeCount. +struct TimelineDetailLevel +{ + uint32_t ResolutionUs = 0; + eastl::vector<TimelineScope> Scopes; // sorted by BeginUs +}; + +// LOD resolutions in microseconds (geometric spacing inspired by Unreal Insights). +// LOD 0 is the raw ThreadTimeline::Scopes; these are LOD 1-5. +inline constexpr uint32_t kTimelineLodResolutions[] = {100, 1000, 8000, 40000, 200000}; +inline constexpr size_t kTimelineLodCount = sizeof(kTimelineLodResolutions) / sizeof(kTimelineLodResolutions[0]); + +struct ThreadTimeline +{ + uint32_t ThreadId = 0; + std::string Name; + int32_t SortHint = 0; + eastl::vector<TimelineScope> Scopes; // LOD 0 -- full resolution, sorted by BeginUs + + TimelineDetailLevel DetailLevels[kTimelineLodCount]; // LOD 1-5 +}; + +// Build pre-computed LOD levels for a ThreadTimeline whose Scopes vector is +// already sorted by BeginUs. Called from BuildTraceModel after populating the +// raw scopes. +void BuildTimelineLods(ThreadTimeline& Timeline); + +// Complete in-memory view of a parsed .utrace file, produced by BuildTraceModel +// and consumed by the `zen trace serve` subcommand. +struct TraceModel +{ + std::filesystem::path FilePath; + uint64_t FileSize = 0; + uint64_t TotalEvents = 0; + uint64_t ParseTimeMs = 0; + uint32_t TraceStartUs = 0; + uint32_t TraceEndUs = 0; + + SessionInfo Session; + eastl::vector<ThreadInfoEntry> Threads; // sorted by SortHint + eastl::vector<ChannelInfo> Channels; // sorted by name + eastl::vector<ModuleInfo> Modules; // sorted by Name + + eastl::vector<std::string> ScopeNames; // referenced by TimelineScope::NameId + eastl::vector<CpuScopeStat> ScopeStats; // sorted by Count descending + eastl::vector<ThreadTimeline> Timelines; // one entry per thread that produced scopes + + eastl::vector<LogCategoryInfo> LogCategories; // referenced by LogEntry::CategoryIndex + eastl::vector<LogEntry> LogEntries; // sorted by TimeUs + + eastl::vector<Bookmark> Bookmarks; // sorted by TimeUs + eastl::vector<RegionCategory> RegionCategories; // sorted: uncategorized first, then alpha + + // -- CsvProfiler -- + struct CsvCategory + { + int32_t Index = 0; + std::string Name; + }; + + struct CsvStatDef + { + uint64_t StatId = 0; + int32_t CategoryIndex = 0; + std::string Name; + }; + + struct CsvSample + { + uint32_t TimeUs; + float Value; + }; + + // Time series for one stat on one thread. + struct CsvSeries + { + uint64_t StatId = 0; + uint32_t ThreadId = 0; + eastl::vector<CsvSample> Samples; // sorted by TimeUs + }; + + struct CsvEvent + { + uint32_t TimeUs; + int32_t CategoryIndex; + std::string Text; + }; + + struct CsvMeta + { + std::string Key; + std::string Value; + }; + + eastl::vector<CsvCategory> CsvCategories; + eastl::vector<CsvStatDef> CsvStatDefs; + eastl::vector<CsvSeries> CsvTimeSeries; // per stat+thread + eastl::vector<CsvEvent> CsvEvents; // sorted by TimeUs + eastl::vector<CsvMeta> CsvMetadata; + + // -- Event type counts (sorted by count descending) -- + struct EventTypeCount + { + std::string Name; + uint64_t Count = 0; + }; + eastl::vector<EventTypeCount> EventTypeCounts; + + // -- Memory allocations -- + AllocationSummary AllocSummary; + eastl::vector<HeapInfo> Heaps; // sorted by Id + eastl::vector<TagInfo> Tags; // sorted by Tag + eastl::vector<MemoryTimelineSample> MemoryTimeline; // sorted by TimeUs + eastl::vector<HeapStat> HeapStats; // sorted by HeapId + eastl::vector<CallstackEntry> Callstacks; // sorted by Id + eastl::vector<CallstackAllocStat> CallstackStats; // sorted by LiveBytes desc + eastl::vector<CallstackChurnStat> ChurnStats; // sorted by TotalAllocs desc + eastl::vector<AllocSizeBucket> AllocSizeHistogram; // sorted by MinSize asc, populated buckets only +}; + +// Resolve and validate a .utrace file path. Throws OptionParseException when +// the path is empty and runtime_error when the file does not exist. +std::filesystem::path ResolveTraceFile(const std::filesystem::path& Input, cxxopts::Options& HelpOptions); + +// Parse a .utrace file and print the event-schema inspect report to the console. +void RunInspect(const std::filesystem::path& FilePath); + +// Progress callback invoked once per bundle during trace iteration. +// Arguments: BytesProcessed (estimated), TotalFileBytes, EventsSoFar. +using ProgressCallback = std::function<void(uint64_t, uint64_t, uint64_t)>; + +// Parse a .utrace file into an in-memory TraceModel suitable for serving via +// the trace viewer. A single pass runs the session, CPU-stats and timeline +// analyzers. The optional progress callback is invoked once per bundle. +TraceModel BuildTraceModel(const std::filesystem::path& FilePath, WorkerThreadPool& ThreadPool, const ProgressCallback& OnProgress = {}); + +struct TraceTrimArgs +{ + std::filesystem::path InputPath; + std::filesystem::path OutputPath; + double StartSec = 0.0; + double EndSec = 0.0; +}; + +// Produce a trimmed .utrace file containing all type-definition and "important" +// packets from the input, plus any regular thread packets whose events overlap +// the [StartSec, EndSec] window. The output remains a valid .utrace that can be +// read by Unreal Insights and zen's own trace tooling. Trimming is coarse at +// the packet level: a packet that straddles the window boundary is kept in full. +void RunTraceTrim(const TraceTrimArgs& Args); + +} // namespace zen::trace_detail diff --git a/src/zen/trace/trace_viewer_service.cpp b/src/zen/trace/trace_viewer_service.cpp new file mode 100644 index 000000000..7d8301ae2 --- /dev/null +++ b/src/zen/trace/trace_viewer_service.cpp @@ -0,0 +1,1225 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_viewer_service.h" + +#include "timeline_query.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/iobuffer.h> +#include <zencore/logging.h> +#include <zencore/string.h> +#include <zenhttp/httpcommon.h> + +#include <algorithm> +#include <charconv> +#include <cstdio> +#include <cstdlib> +#include <string> +#include <type_traits> + +#if !defined(ZEN_EMBED_ZEN_HTML_ZIP) +# define ZEN_EMBED_ZEN_HTML_ZIP 0 +#endif + +#if ZEN_EMBED_ZEN_HTML_ZIP +static unsigned char gZenHtmlZipData[] = { +# include <zen-html.zip.h> +}; +#endif + +namespace zen { + +namespace { + + // Parse a uint32 query parameter; returns the fallback on error / absent. + // The entire string must be a valid base-10 unsigned integer. + uint32_t ParseUintParam(std::string_view Value, uint32_t Fallback) + { + if (Value.empty()) + { + return Fallback; + } + + uint32_t Number = 0; + auto [Ptr, Ec] = std::from_chars(Value.data(), Value.data() + Value.size(), Number, 10); + if (Ec != std::errc() || Ptr != Value.data() + Value.size()) + { + return Fallback; + } + + return Number; + } + + void WriteNotFound(HttpServerRequest& Request, std::string_view Message = "Not found") + { + Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, Message); + } + + struct CallstackSummaryInfo + { + std::string Summary; + std::string TopFrame; + std::string SecondaryFrame; + std::string GroupKey; + uint32_t HiddenPrefixCount = 0; + bool IncludedThirdPartyBoundary = false; + }; + + CallstackSummaryInfo BuildCallstackSummary(const trace_detail::FilteredCallstackView& View) + { + CallstackSummaryInfo Result; + Result.HiddenPrefixCount = View.HiddenPrefixCount; + Result.IncludedThirdPartyBoundary = View.IncludedThirdPartyBoundary; + if (View.Frames.empty()) + { + Result.Summary = "No frames"; + Result.GroupKey = "No frames"; + return Result; + } + + Result.TopFrame = View.Frames[0].Display; + Result.GroupKey = View.Frames[0].Display; + if (View.Frames.size() > 1) + { + Result.SecondaryFrame = View.Frames[1].Display; + Result.Summary = fmt::format("{} \xE2\x86\x90 {}", Result.TopFrame, Result.SecondaryFrame); + Result.GroupKey = fmt::format("{} | {}", Result.TopFrame, Result.SecondaryFrame); + } + else + { + Result.Summary = Result.TopFrame; + } + return Result; + } + + // Append a base-10 unsigned integer to a string builder via std::to_chars. + // Reserves the worst-case digit count up front, writes directly into the + // builder's buffer, then trims the unused suffix. About 5–10× faster than + // going through StringBuilder::operator<<(uint32_t), which routes integer + // formatting through snprintf via IntNum. + template<typename T> + inline void AppendUintFast(StringBuilderBase& Sb, T Value) + { + static_assert(std::is_unsigned_v<T> && std::is_integral_v<T>); + // digits10 is the largest K such that 10^K fits — the longest + // printable representation is digits10 + 1 digits. +1 more for safety. + constexpr size_t MaxDigits = std::numeric_limits<T>::digits10 + 2; + + const size_t Off = Sb.AddUninitialized(MaxDigits); + char* const Begin = Sb.Data() + Off; + const auto Result = std::to_chars(Begin, Begin + MaxDigits, Value); + const size_t Written = size_t(Result.ptr - Begin); + Sb.RemoveSuffix(uint32_t(MaxDigits - Written)); + } + + // Render a span of TimelineScopeView records directly into a string + // builder using the wire format consumed by the trace viewer front-end: + // [[beginUs, durationUs, nameId, depth, mergeCount?], ...] + // The trailing mergeCount element is only emitted for LOD-merged scopes. + // Output is compact (no whitespace) — the viewer parses both forms but + // dropping the spaces shaves ~10% off the response size. + void AppendScopesJsonArray(StringBuilderBase& Sb, const trace_detail::TimelineScopeView* Scopes, size_t Count) + { + Sb << '['; + for (size_t I = 0; I < Count; ++I) + { + const trace_detail::TimelineScopeView& S = Scopes[I]; + if (I > 0) + { + Sb << ','; + } + Sb << '['; + AppendUintFast(Sb, S.BeginUs); + Sb << ','; + AppendUintFast(Sb, S.DurationUs); + Sb << ','; + AppendUintFast(Sb, S.NameId); + Sb << ','; + AppendUintFast(Sb, S.Depth); + if (S.MergeCount > 1) + { + Sb << ','; + AppendUintFast(Sb, S.MergeCount); + } + Sb << ']'; + } + Sb << ']'; + } + +} // namespace + +////////////////////////////////////////////////////////////////////////////// + +TraceViewerService::TraceViewerService(const trace_detail::TraceModel& Model, + std::unique_ptr<trace_detail::SymbolResolver> Symbols, + std::filesystem::path DevHtmlDir) +: m_Model(Model) +, m_DevHtmlDir(std::move(DevHtmlDir)) +, m_Symbols(std::move(Symbols)) +, m_CallstackFormatter(m_Model, m_Symbols.get()) +{ +#if ZEN_EMBED_ZEN_HTML_ZIP + IoBuffer ZipBuffer(IoBuffer::Wrap, gZenHtmlZipData, sizeof(gZenHtmlZipData) - 1); + m_ZipFs = std::make_unique<ZipFs>(std::move(ZipBuffer)); +#endif + + m_TimelineQuery = trace_detail::MakeInMemoryTimelineQuery(m_Model); + + if (m_DevHtmlDir.empty()) + { + // Probe for development layout: walk up from the running executable + // until we find a directory named xmake.lua, then look for the html + // tree under src/zen/frontend/html. + std::filesystem::path Path = GetRunningExecutablePath(); + std::error_code Ec; + while (Path.has_parent_path()) + { + std::filesystem::path Parent = Path.parent_path(); + if (Parent == Path) + { + break; + } + if (IsFile(Parent / "xmake.lua", Ec)) + { + std::filesystem::path Candidate = Parent / "src" / "zen" / "frontend" / "html"; + if (IsDir(Candidate, Ec)) + { + m_DevHtmlDir = Candidate; + } + break; + } + Path = Parent; + } + } + + if (m_ZipFs) + { + ZEN_INFO("trace viewer front-end is served from embedded zip"); + } + else if (!m_DevHtmlDir.empty()) + { + ZEN_INFO("trace viewer front-end is served from '{}'", m_DevHtmlDir); + } + else + { + ZEN_WARN("trace viewer front-end is NOT AVAILABLE — only /api/* endpoints will respond"); + } +} + +TraceViewerService::~TraceViewerService() = default; + +const char* +TraceViewerService::BaseUri() const +{ + // Mounted at a sub-path so we don't collide with the http.sys server's + // own root handler on Windows. + return "/trace/"; +} + +void +TraceViewerService::HandleRequest(HttpServerRequest& Request) +{ + using namespace std::literals; + + std::string_view Uri = Request.RelativeUriWithExtension(); + for (; !Uri.empty() && Uri[0] == '/'; Uri = Uri.substr(1)) + { + } + + if (Uri.starts_with("api/"sv)) + { + HandleApiRequest(Request, Uri.substr(4)); + return; + } + + HandleStaticAsset(Request, Uri); +} + +////////////////////////////////////////////////////////////////////////////// +// Static asset handling + +void +TraceViewerService::HandleStaticAsset(HttpServerRequest& Request, std::string_view Uri) +{ + using namespace std::literals; + + ExtendableStringBuilder<256> UriBuilder; + if (Uri.empty()) + { + Uri = "index.html"sv; + } + else if (Uri.back() == '/') + { + UriBuilder << Uri << "index.html"sv; + Uri = UriBuilder; + } + + // Path traversal guard: reject parent refs, Windows-style separators, and absolute + // paths. `std::filesystem::path::operator/=` replaces the base when the RHS is + // absolute, so without this check a URI like `C:/Windows/...` would escape m_DevHtmlDir. + if (Uri.find("..") != Uri.npos || Uri.find('\\') != Uri.npos || std::filesystem::path(Uri).is_absolute()) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + return; + } + + HttpContentType ContentType = HttpContentType::kUnknownContentType; + if (const size_t DotIndex = Uri.rfind("."); DotIndex != Uri.npos) + { + const std::string_view DotExt = Uri.substr(DotIndex + 1); + ContentType = ParseContentType(DotExt); + if (ContentType == HttpContentType::kUnknownContentType) + { + if (DotExt == "txt"sv || DotExt == "md"sv) + { + ContentType = HttpContentType::kText; + } + } + } + + if (ContentType == HttpContentType::kUnknownContentType) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + return; + } + + // Dev mode: serve from disk first so HTML/JS edits show up without a rebuild + if (!m_DevHtmlDir.empty()) + { + std::filesystem::path FullPath = m_DevHtmlDir / std::filesystem::path(Uri).make_preferred(); + FileContents File = ReadFile(FullPath); + if (!File.ErrorCode) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]); + return; + } + } + + // Fallback: embedded zip + if (m_ZipFs) + { + if (IoBuffer File = m_ZipFs->GetFile(Uri)) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, File); + return; + } + } + + WriteNotFound(Request); +} + +////////////////////////////////////////////////////////////////////////////// +// REST endpoints + +void +TraceViewerService::HandleApiRequest(HttpServerRequest& Request, std::string_view Path) +{ + using namespace std::literals; + + if (Path == "session"sv) + { + HandleSessionApi(Request); + } + else if (Path == "threads"sv) + { + HandleThreadsApi(Request); + } + else if (Path == "channels"sv) + { + HandleChannelsApi(Request); + } + else if (Path == "scope-stats"sv) + { + HandleScopeStatsApi(Request); + } + else if (Path == "scope-names"sv) + { + HandleScopeNamesApi(Request); + } + else if (Path == "timeline"sv) + { + HandleTimelineApi(Request); + } + else if (Path == "timeline-batch"sv) + { + HandleTimelineBatchApi(Request); + } + else if (Path == "log-categories"sv) + { + HandleLogCategoriesApi(Request); + } + else if (Path == "logs"sv) + { + HandleLogsApi(Request); + } + else if (Path == "bookmarks"sv) + { + HandleBookmarksApi(Request); + } + else if (Path == "regions"sv) + { + HandleRegionsApi(Request); + } + else if (Path == "csv-categories"sv) + { + HandleCsvCategoriesApi(Request); + } + else if (Path == "csv-stats"sv) + { + HandleCsvStatsApi(Request); + } + else if (Path == "csv-series"sv) + { + HandleCsvSeriesApi(Request); + } + else if (Path == "csv-events"sv) + { + HandleCsvEventsApi(Request); + } + else if (Path == "csv-metadata"sv) + { + HandleCsvMetadataApi(Request); + } + else if (Path == "alloc-summary"sv) + { + HandleAllocSummaryApi(Request); + } + else if (Path == "heaps"sv) + { + HandleHeapsApi(Request); + } + else if (Path == "alloc-tags"sv) + { + HandleAllocTagsApi(Request); + } + else if (Path == "memory-timeline"sv) + { + HandleMemoryTimelineApi(Request); + } + else if (Path == "heap-stats"sv) + { + HandleHeapStatsApi(Request); + } + else if (Path == "callstacks"sv) + { + HandleCallstacksApi(Request); + } + else if (Path == "callstack-stats"sv) + { + HandleCallstackStatsApi(Request); + } + else if (Path == "churn-stats"sv) + { + HandleChurnStatsApi(Request); + } + else if (Path == "alloc-size-histogram"sv) + { + HandleAllocSizeHistogramApi(Request); + } + else + { + WriteNotFound(Request, "Unknown API endpoint"); + } +} + +void +TraceViewerService::HandleSessionApi(HttpServerRequest& Request) +{ + const trace_detail::SessionInfo& Session = m_Model.Session; + + CbObjectWriter Obj; + Obj << "file_path" << m_Model.FilePath.string(); + Obj << "file_size" << m_Model.FileSize; + Obj << "total_events" << m_Model.TotalEvents; + Obj << "parse_time_ms" << m_Model.ParseTimeMs; + Obj << "trace_start_us" << m_Model.TraceStartUs; + Obj << "trace_end_us" << m_Model.TraceEndUs; + Obj << "has_session" << Session.HasSession; + Obj << "platform" << Session.Platform; + Obj << "app_name" << Session.AppName; + Obj << "project_name" << Session.ProjectName; + Obj << "command_line" << Session.CommandLine; + Obj << "branch" << Session.Branch; + Obj << "build_version" << Session.BuildVersion; + Obj << "changelist" << Session.Changelist; + Obj << "has_memory_data" << m_Model.AllocSummary.HasMemoryData; + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleThreadsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::ThreadInfoEntry& Thread : m_Model.Threads) + { + Writer.BeginObject(); + Writer << "thread_id" << Thread.ThreadId; + Writer << "name" << Thread.Name; + Writer << "group" << Thread.GroupName; + Writer << "system_id" << Thread.SystemId; + Writer << "sort_hint" << Thread.SortHint; + // Lane threads use synthetic IDs starting at 2048 (see lane_trace.inl). + // SystemId==0 alone is insufficient — the main/trace threads also lack + // a system ID in some traces. + Writer << "is_lane" << (Thread.SystemId == 0 && Thread.ThreadId >= 2048); + + // Per-thread timeline summary: whether we captured scopes and their span. + auto It = std::find_if(m_Model.Timelines.begin(), + m_Model.Timelines.end(), + [Tid = Thread.ThreadId](const trace_detail::ThreadTimeline& T) { return T.ThreadId == Tid; }); + if (It != m_Model.Timelines.end()) + { + Writer << "scope_count" << uint64_t(It->Scopes.size()); + } + else + { + Writer << "scope_count" << uint64_t(0); + } + + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleChannelsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::ChannelInfo& Channel : m_Model.Channels) + { + Writer.BeginObject(); + Writer << "name" << Channel.Name; + Writer << "enabled" << Channel.Enabled; + Writer << "readonly" << Channel.ReadOnly; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleScopeStatsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::CpuScopeStat& Stat : m_Model.ScopeStats) + { + Writer.BeginObject(); + Writer << "name" << Stat.Name; + Writer << "count" << Stat.Count; + Writer << "min_us" << Stat.MinUs; + Writer << "max_us" << Stat.MaxUs; + Writer.AddFloat("mean_us", Stat.MeanUs); + Writer.AddFloat("stdev_us", Stat.StdDevUs); + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleScopeNamesApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const std::string& Name : m_Model.ScopeNames) + { + Writer.AddString(Name); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleTimelineApi(HttpServerRequest& Request) +{ + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string_view ThreadStr = Params.GetValue("thread"); + if (ThreadStr.empty()) + { + Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "thread parameter required"); + return; + } + + uint32_t ThreadId = ParseUintParam(ThreadStr, ~uint32_t(0)); + + trace_detail::TimelineQueryRequest Req; + Req.StartUs = ParseUintParam(Params.GetValue("start"), 0u); + Req.EndUs = ParseUintParam(Params.GetValue("end"), ~uint32_t(0)); + Req.MinDurUs = ParseUintParam(Params.GetValue("mindur"), 0u); + Req.ResolutionUs = ParseUintParam(Params.GetValue("resolution"), 0u); + + std::vector<trace_detail::TimelineScopeView> Scopes; + m_TimelineQuery->QueryThread(ThreadId, Req, Scopes); + + // Direct string formatting for the timeline wire format — the compact + // [[beginUs,durationUs,nameId,depth,mergeCount?],...] arrays are faster + // to serialize directly than via CbWriter. The 64 KB inline buffer + // covers small viewport queries without heap traffic. + ExtendableStringBuilder<65536> Sb; + Sb << R"({"thread_id":)"; + AppendUintFast(Sb, ThreadId); + Sb << R"(,"scopes":)"; + AppendScopesJsonArray(Sb, Scopes.data(), Scopes.size()); + Sb << '}'; + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Sb.ToView()); +} + +void +TraceViewerService::HandleTimelineBatchApi(HttpServerRequest& Request) +{ + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string_view ThreadsStr = Params.GetValue("threads"); + if (ThreadsStr.empty()) + { + Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "threads parameter required"); + return; + } + + trace_detail::TimelineQueryRequest Req; + Req.StartUs = ParseUintParam(Params.GetValue("start"), 0u); + Req.EndUs = ParseUintParam(Params.GetValue("end"), ~uint32_t(0)); + Req.MinDurUs = ParseUintParam(Params.GetValue("mindur"), 0u); + Req.ResolutionUs = ParseUintParam(Params.GetValue("resolution"), 0u); + + // Parse comma-separated thread IDs. Tokens with invalid IDs (~0u) are + // skipped — same behaviour as the previous handler. + std::vector<uint32_t> ThreadIds; + { + std::string_view Remaining = ThreadsStr; + while (!Remaining.empty()) + { + size_t Comma = Remaining.find(','); + std::string_view Token = (Comma != std::string_view::npos) ? Remaining.substr(0, Comma) : Remaining; + Remaining = (Comma != std::string_view::npos) ? Remaining.substr(Comma + 1) : std::string_view{}; + + uint32_t ThreadId = ParseUintParam(Token, ~uint32_t(0)); + if (ThreadId == ~uint32_t(0)) + { + continue; + } + ThreadIds.push_back(ThreadId); + } + } + + trace_detail::TimelineQuery::BatchResult Batch; + m_TimelineQuery->QueryBatch(ThreadIds, Req, Batch); + + // Multi-chunk response: one IoBuffer per thread plus the surrounding + // "{" / "}" braces. Avoids materialising the entire JSON in a single + // contiguous allocation. The transport gathers the chunks at write time. + static constexpr char kOpenBrace[] = "{"; + static constexpr char kCloseBrace[] = "}"; + + std::vector<IoBuffer> Chunks; + Chunks.reserve(2 + ThreadIds.size()); + Chunks.emplace_back(IoBuffer::Wrap, kOpenBrace, 1); + + for (size_t I = 0; I < ThreadIds.size(); ++I) + { + const trace_detail::TimelineQuery::BatchResult::Range R = Batch.Ranges[I]; + + // Per-thread chunk: optional leading comma, "<threadId>":{"scopes":[...]} + // 32 KB inline covers most threads at typical viewport zoom levels. + ExtendableStringBuilder<32768> Sb; + if (I > 0) + { + Sb << ','; + } + Sb << '"'; + AppendUintFast(Sb, ThreadIds[I]); + Sb << R"(":{"scopes":)"; + AppendScopesJsonArray(Sb, Batch.Scopes.data() + R.Begin, R.End - R.Begin); + Sb << '}'; + + // Clone into an IoBuffer so the chunk owns its bytes — the builder + // dies at the end of this iteration. + const std::string_view View = Sb.ToView(); + Chunks.emplace_back(IoBuffer::Clone, View.data(), View.size()); + } + + Chunks.emplace_back(IoBuffer::Wrap, kCloseBrace, 1); + + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, std::span<IoBuffer>{Chunks}); +} + +void +TraceViewerService::HandleLogCategoriesApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::LogCategoryInfo& Cat : m_Model.LogCategories) + { + Writer.BeginObject(); + Writer << "name" << Cat.Name; + Writer << "default_verbosity" << uint32_t(Cat.DefaultVerbosity); + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleLogsApi(HttpServerRequest& Request) +{ + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + uint32_t StartUs = ParseUintParam(Params.GetValue("start"), 0u); + uint32_t EndUs = ParseUintParam(Params.GetValue("end"), ~uint32_t(0)); + uint32_t MinVerb = ParseUintParam(Params.GetValue("min_verbosity"), 0u); + uint32_t CategoryId = ParseUintParam(Params.GetValue("category"), ~uint32_t(0)); + uint32_t Limit = ParseUintParam(Params.GetValue("limit"), 5000u); + + // Binary-search lower bound by TimeUs. + const eastl::vector<trace_detail::LogEntry>& Entries = m_Model.LogEntries; + auto FirstIt = + std::lower_bound(Entries.begin(), Entries.end(), StartUs, [](const trace_detail::LogEntry& E, uint32_t V) { return E.TimeUs < V; }); + + CbObjectWriter Obj; + Obj << "total" << uint64_t(Entries.size()); + + uint32_t Emitted = 0; + Obj.BeginArray("entries"); + for (auto It = FirstIt; It != Entries.end() && Emitted < Limit; ++It) + { + if (It->TimeUs > EndUs) + { + break; + } + if (MinVerb != 0 && It->Verbosity > MinVerb) + { + // Lower verbosity value = higher severity in UE's ELogVerbosity. + // Skip entries less severe than the requested floor. + continue; + } + if (CategoryId != ~uint32_t(0) && It->CategoryIndex != CategoryId) + { + continue; + } + + Obj.BeginObject(); + Obj << "time_us" << It->TimeUs; + Obj << "category_index" << It->CategoryIndex; + Obj << "verbosity" << uint32_t(It->Verbosity); + Obj << "line" << It->Line; + Obj << "file" << It->File; + Obj << "message" << It->Message; + Obj.EndObject(); + ++Emitted; + } + Obj.EndArray(); + + Obj << "returned" << Emitted; + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleBookmarksApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::Bookmark& B : m_Model.Bookmarks) + { + Writer.BeginObject(); + Writer << "time_us" << B.TimeUs; + Writer << "line" << B.Line; + Writer << "file" << B.File; + Writer << "text" << B.Text; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleRegionsApi(HttpServerRequest& Request) +{ + CbObjectWriter Obj; + Obj.BeginArray("categories"); + for (const trace_detail::RegionCategory& Cat : m_Model.RegionCategories) + { + Obj.BeginObject(); + Obj << "name" << std::string_view(Cat.Name.empty() ? "Uncategorized" : Cat.Name); + Obj << "lane_count" << Cat.LaneCount; + Obj.BeginArray("regions"); + for (const trace_detail::RegionEntry& R : Cat.Regions) + { + Obj.BeginObject(); + Obj << "begin_us" << R.BeginUs; + Obj << "end_us" << R.EndUs; + Obj << "depth" << uint32_t(R.Depth); + Obj << "name" << R.Name; + Obj.EndObject(); + } + Obj.EndArray(); + Obj.EndObject(); + } + Obj.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleCsvCategoriesApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const auto& Cat : m_Model.CsvCategories) + { + Writer.BeginObject(); + Writer << "index" << Cat.Index; + Writer << "name" << Cat.Name; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleCsvStatsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const auto& Def : m_Model.CsvStatDefs) + { + Writer.BeginObject(); + Writer << "stat_id" << Def.StatId; + Writer << "category_index" << Def.CategoryIndex; + Writer << "name" << Def.Name; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleCsvSeriesApi(HttpServerRequest& Request) +{ + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + // Accept either a single series index or iterate all for the requested stat+thread. + std::string_view StatStr = Params.GetValue("stat"); + std::string_view ThreadStr = Params.GetValue("thread"); + + uint64_t StatId = StatStr.empty() ? 0 : ParseUintParam(StatStr, 0); + uint32_t ThreadId = ThreadStr.empty() ? ~uint32_t(0) : ParseUintParam(ThreadStr, ~uint32_t(0)); + + CbWriter Writer; + Writer.BeginArray(); + for (const auto& S : m_Model.CsvTimeSeries) + { + if (StatId != 0 && S.StatId != StatId) + { + continue; + } + if (ThreadId != ~uint32_t(0) && S.ThreadId != ThreadId) + { + continue; + } + Writer.BeginObject(); + Writer << "stat_id" << S.StatId; + Writer << "thread_id" << S.ThreadId; + Writer.BeginArray("samples"); + for (const auto& Sample : S.Samples) + { + Writer.BeginArray(); + Writer.AddInteger(uint32_t(Sample.TimeUs)); + Writer.AddFloat(double(Sample.Value)); + Writer.EndArray(); + } + Writer.EndArray(); + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleCsvEventsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const auto& E : m_Model.CsvEvents) + { + Writer.BeginObject(); + Writer << "time_us" << E.TimeUs; + Writer << "category_index" << E.CategoryIndex; + Writer << "text" << E.Text; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleCsvMetadataApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const auto& M : m_Model.CsvMetadata) + { + Writer.BeginObject(); + Writer << "key" << M.Key; + Writer << "value" << M.Value; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +////////////////////////////////////////////////////////////////////////////// +// Memory allocation endpoints + +void +TraceViewerService::HandleAllocSummaryApi(HttpServerRequest& Request) +{ + const trace_detail::AllocationSummary& S = m_Model.AllocSummary; + + CbObjectWriter Obj; + Obj << "has_memory_data" << S.HasMemoryData; + Obj << "total_allocs" << S.TotalAllocs; + Obj << "total_frees" << S.TotalFrees; + Obj << "total_realloc_allocs" << S.TotalReallocAllocs; + Obj << "total_realloc_frees" << S.TotalReallocFrees; + Obj << "peak_bytes" << S.PeakBytes; + Obj << "peak_time_us" << S.PeakTimeUs; + Obj << "end_bytes" << S.EndBytes; + Obj << "live_allocations" << S.LiveAllocations; + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleHeapsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::HeapInfo& H : m_Model.Heaps) + { + Writer.BeginObject(); + Writer << "id" << H.Id; + Writer << "parent_id" << H.ParentId; + Writer << "flags" << H.Flags; + Writer << "name" << H.Name; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleAllocTagsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::TagInfo& T : m_Model.Tags) + { + Writer.BeginObject(); + Writer << "tag" << T.Tag; + Writer << "parent" << T.Parent; + Writer << "display" << T.Display; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleMemoryTimelineApi(HttpServerRequest& Request) +{ + const auto& Timeline = m_Model.MemoryTimeline; + + // Parse optional query parameters for range filtering and downsampling. + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + uint32_t StartUs = ParseUintParam(Params.GetValue("start"), 0); + uint32_t EndUs = ParseUintParam(Params.GetValue("end"), ~uint32_t(0)); + uint32_t MaxSamples = ParseUintParam(Params.GetValue("max_samples"), 2000); + if (MaxSamples == 0) + { + MaxSamples = 2000; + } + + // Binary-search for the start offset. + size_t Begin = 0; + { + size_t Lo = 0; + size_t Hi = Timeline.size(); + while (Lo < Hi) + { + size_t Mid = Lo + (Hi - Lo) / 2; + if (Timeline[Mid].TimeUs < StartUs) + { + Lo = Mid + 1; + } + else + { + Hi = Mid; + } + } + Begin = Lo; + } + + // Find end offset. + size_t End = Timeline.size(); + { + size_t Lo = Begin; + size_t Hi = Timeline.size(); + while (Lo < Hi) + { + size_t Mid = Lo + (Hi - Lo) / 2; + if (Timeline[Mid].TimeUs <= EndUs) + { + Lo = Mid + 1; + } + else + { + Hi = Mid; + } + } + End = Lo; + } + + size_t Count = (End > Begin) ? (End - Begin) : 0; + size_t Stride = (Count > MaxSamples) ? (Count / MaxSamples) : 1; + + CbObjectWriter Obj; + + uint64_t SampleCount = 0; + Obj.BeginArray("samples"); + for (size_t I = Begin; I < End; I += Stride) + { + const trace_detail::MemoryTimelineSample& S = Timeline[I]; + Obj.BeginArray(); + Obj.AddInteger(S.TimeUs); + Obj.AddInteger(S.TotalAllocatedBytes); + Obj.AddInteger(S.SystemBytes); + Obj.AddInteger(S.VideoBytes); + Obj.EndArray(); + ++SampleCount; + } + Obj.EndArray(); + + Obj << "sample_count" << SampleCount; + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleHeapStatsApi(HttpServerRequest& Request) +{ + CbWriter Writer; + Writer.BeginArray(); + for (const trace_detail::HeapStat& S : m_Model.HeapStats) + { + Writer.BeginObject(); + Writer << "heap_id" << S.HeapId; + Writer << "current_bytes" << S.CurrentBytes; + Writer << "peak_bytes" << S.PeakBytes; + Writer << "alloc_count" << S.AllocCount; + Writer << "free_count" << S.FreeCount; + Writer.EndObject(); + } + Writer.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Writer.Save().AsArray()); +} + +void +TraceViewerService::HandleCallstacksApi(HttpServerRequest& Request) +{ + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + uint32_t Id = ParseUintParam(Params.GetValue("id"), 0); + + if (Id == 0) + { + WriteNotFound(Request, "Missing or invalid 'id' parameter"); + return; + } + + const trace_detail::CallstackEntry* Entry = m_CallstackFormatter.FindCallstackEntry(Id); + if (Entry == nullptr) + { + WriteNotFound(Request, "Callstack not found"); + return; + } + + trace_detail::FilteredCallstackView Filtered = m_CallstackFormatter.BuildView(*Entry, m_CallstackFilterOptions); + + CbObjectWriter Obj; + CallstackSummaryInfo Summary = BuildCallstackSummary(Filtered); + Obj << "id" << Entry->Id; + Obj << "summary" << Summary.Summary; + Obj << "top_frame" << Summary.TopFrame; + Obj << "secondary_frame" << Summary.SecondaryFrame; + Obj << "group_key" << Summary.GroupKey; + Obj << "hidden_prefix_count" << Filtered.HiddenPrefixCount; + Obj << "included_third_party_boundary" << Filtered.IncludedThirdPartyBoundary; + Obj.BeginArray("frames"); + for (const trace_detail::FilteredCallstackFrame& FrameView : Filtered.Frames) + { + const trace_detail::ResolvedFrame& F = *FrameView.Frame; + Obj.BeginObject(); + Obj << "index" << uint64_t(FrameView.OriginalIndex); + Obj.AddString("address", fmt::format("0x{:X}", F.Address)); + Obj << "display" << FrameView.Display; + if (F.ModuleIndex != ~0u && F.ModuleIndex < m_Model.Modules.size()) + { + const trace_detail::ModuleInfo& Module = m_Model.Modules[F.ModuleIndex]; + Obj << "module" << std::string_view(Module.Name); + Obj << "module_path" << std::string_view(Module.FullPath); + Obj.AddString("offset", fmt::format("0x{:X}", F.Offset)); + } + Obj.EndObject(); + } + Obj.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleCallstackStatsApi(HttpServerRequest& Request) +{ + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + uint32_t Limit = ParseUintParam(Params.GetValue("limit"), 100); + if (Limit == 0) + { + Limit = 100; + } + + size_t Count = std::min(size_t(Limit), m_Model.CallstackStats.size()); + + CbObjectWriter Obj; + Obj << "total_unique_callstacks" << uint64_t(m_Model.Callstacks.size()); + Obj.BeginArray("stats"); + for (size_t I = 0; I < Count; ++I) + { + const trace_detail::CallstackAllocStat& S = m_Model.CallstackStats[I]; + Obj.BeginObject(); + Obj << "callstack_id" << S.CallstackId; + Obj << "live_bytes" << S.LiveBytes; + Obj << "live_count" << S.LiveCount; + if (const trace_detail::CallstackEntry* Entry = m_CallstackFormatter.FindCallstackEntry(S.CallstackId)) + { + CallstackSummaryInfo Summary = BuildCallstackSummary(m_CallstackFormatter.BuildView(*Entry, m_CallstackFilterOptions)); + Obj << "summary" << Summary.Summary; + Obj << "top_frame" << Summary.TopFrame; + Obj << "secondary_frame" << Summary.SecondaryFrame; + Obj << "group_key" << Summary.GroupKey; + Obj << "hidden_prefix_count" << Summary.HiddenPrefixCount; + Obj << "included_third_party_boundary" << Summary.IncludedThirdPartyBoundary; + } + Obj.EndObject(); + } + Obj.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleChurnStatsApi(HttpServerRequest& Request) +{ + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + uint32_t Limit = ParseUintParam(Params.GetValue("limit"), 100); + if (Limit == 0) + { + Limit = 100; + } + + size_t Count = std::min(size_t(Limit), m_Model.ChurnStats.size()); + + CbObjectWriter Obj; + Obj << "total_unique_callstacks" << uint64_t(m_Model.Callstacks.size()); + Obj.BeginArray("stats"); + for (size_t I = 0; I < Count; ++I) + { + const trace_detail::CallstackChurnStat& S = m_Model.ChurnStats[I]; + Obj.BeginObject(); + Obj << "callstack_id" << S.CallstackId; + Obj << "churn_allocs" << S.ChurnAllocs; + Obj << "churn_bytes" << S.ChurnBytes; + Obj << "total_allocs" << S.TotalAllocs; + Obj << "total_bytes" << S.TotalBytes; + Obj.AddFloat("mean_distance", S.MeanDistance); + if (const trace_detail::CallstackEntry* Entry = m_CallstackFormatter.FindCallstackEntry(S.CallstackId)) + { + CallstackSummaryInfo Summary = BuildCallstackSummary(m_CallstackFormatter.BuildView(*Entry, m_CallstackFilterOptions)); + Obj << "summary" << Summary.Summary; + Obj << "top_frame" << Summary.TopFrame; + Obj << "secondary_frame" << Summary.SecondaryFrame; + Obj << "group_key" << Summary.GroupKey; + Obj << "hidden_prefix_count" << Summary.HiddenPrefixCount; + Obj << "included_third_party_boundary" << Summary.IncludedThirdPartyBoundary; + } + Obj.EndObject(); + } + Obj.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +TraceViewerService::HandleAllocSizeHistogramApi(HttpServerRequest& Request) +{ + const auto& Buckets = m_Model.AllocSizeHistogram; + + uint64_t TotalCount = 0; + uint64_t TotalBytes = 0; + uint64_t MaxCount = 0; + uint64_t MaxBytes = 0; + for (const trace_detail::AllocSizeBucket& B : Buckets) + { + TotalCount += B.Count; + TotalBytes += B.Bytes; + if (B.Count > MaxCount) + { + MaxCount = B.Count; + } + if (B.Bytes > MaxBytes) + { + MaxBytes = B.Bytes; + } + } + + CbObjectWriter Obj; + Obj << "total_count" << TotalCount; + Obj << "total_bytes" << TotalBytes; + Obj << "max_count" << MaxCount; + Obj << "max_bytes" << MaxBytes; + Obj.BeginArray("buckets"); + for (const trace_detail::AllocSizeBucket& B : Buckets) + { + Obj.BeginObject(); + Obj << "min_size" << B.MinSize; + Obj << "max_size" << B.MaxSize; + Obj << "count" << B.Count; + Obj << "bytes" << B.Bytes; + Obj.EndObject(); + } + Obj.EndArray(); + + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +} // namespace zen diff --git a/src/zen/trace/trace_viewer_service.h b/src/zen/trace/trace_viewer_service.h new file mode 100644 index 000000000..f7bc51499 --- /dev/null +++ b/src/zen/trace/trace_viewer_service.h @@ -0,0 +1,71 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "callstack_formatter.h" +#include "timeline_query.h" +#include "trace_model.h" + +#include <zenhttp/httpserver.h> +#include <zenhttp/zipfs.h> + +#include <filesystem> +#include <memory> + +namespace zen { + +// HttpService that serves an interactive flame-graph viewer for a parsed +// TraceModel. Mounts at the server root; URIs beginning with "api/" return +// JSON describing the model, everything else is resolved to a static asset +// (first from the optional dev-mode directory, then from the embedded zip). +class TraceViewerService final : public HttpService +{ +public: + TraceViewerService(const trace_detail::TraceModel& Model, + std::unique_ptr<trace_detail::SymbolResolver> Symbols = {}, + std::filesystem::path DevHtmlDir = {}); + ~TraceViewerService() override; + + [[nodiscard]] const char* BaseUri() const override; + void HandleRequest(HttpServerRequest& Request) override; + +private: + void HandleStaticAsset(HttpServerRequest& Request, std::string_view Uri); + void HandleApiRequest(HttpServerRequest& Request, std::string_view Path); + + void HandleSessionApi(HttpServerRequest& Request); + void HandleThreadsApi(HttpServerRequest& Request); + void HandleChannelsApi(HttpServerRequest& Request); + void HandleScopeStatsApi(HttpServerRequest& Request); + void HandleScopeNamesApi(HttpServerRequest& Request); + void HandleTimelineApi(HttpServerRequest& Request); + void HandleTimelineBatchApi(HttpServerRequest& Request); + void HandleLogCategoriesApi(HttpServerRequest& Request); + void HandleLogsApi(HttpServerRequest& Request); + void HandleBookmarksApi(HttpServerRequest& Request); + void HandleRegionsApi(HttpServerRequest& Request); + void HandleCsvCategoriesApi(HttpServerRequest& Request); + void HandleCsvStatsApi(HttpServerRequest& Request); + void HandleCsvSeriesApi(HttpServerRequest& Request); + void HandleCsvEventsApi(HttpServerRequest& Request); + void HandleCsvMetadataApi(HttpServerRequest& Request); + void HandleAllocSummaryApi(HttpServerRequest& Request); + void HandleHeapsApi(HttpServerRequest& Request); + void HandleAllocTagsApi(HttpServerRequest& Request); + void HandleMemoryTimelineApi(HttpServerRequest& Request); + void HandleHeapStatsApi(HttpServerRequest& Request); + void HandleCallstacksApi(HttpServerRequest& Request); + void HandleCallstackStatsApi(HttpServerRequest& Request); + void HandleChurnStatsApi(HttpServerRequest& Request); + void HandleAllocSizeHistogramApi(HttpServerRequest& Request); + + const trace_detail::TraceModel& m_Model; + std::filesystem::path m_DevHtmlDir; + std::unique_ptr<ZipFs> m_ZipFs; + std::unique_ptr<trace_detail::TimelineQuery> m_TimelineQuery; + std::unique_ptr<trace_detail::SymbolResolver> m_Symbols; + trace_detail::CallstackFilterOptions m_CallstackFilterOptions; + trace_detail::CallstackFormatter m_CallstackFormatter; +}; + +} // namespace zen diff --git a/src/zen/xmake.lua b/src/zen/xmake.lua index df249ade4..c4084231d 100644 --- a/src/zen/xmake.lua +++ b/src/zen/xmake.lua @@ -5,13 +5,57 @@ target("zen") add_headerfiles("**.h") add_files("**.cpp") add_files("zen.cpp", {unity_ignored = true }) + add_rules("utils.bin2c", {extensions = {".zip"}}) + add_files(path.join(os.projectdir(), get_config("builddir") or get_config("buildir") or "build", "zen-frontend/zen-html.zip")) add_deps("zencore", "zenhttp", "zenremotestore", "zenstore", "zenutil") add_deps("zencompute", "zennet", "zentelemetry") - add_deps("cxxopts", "fmt") + add_deps("cxxopts", "fmt", "raw_pdb", "tourist") add_packages("json11") add_includedirs(".") + add_defines("ZEN_EMBED_ZEN_HTML_ZIP=1") set_symbols("debug") + on_load(function(target) + local html_dir = path.join(os.projectdir(), "src/zen/frontend/html") + local zip_dir = path.join(os.projectdir(), get_config("builddir") or get_config("buildir") or "build", "zen-frontend") + local zip_path = path.join(zip_dir, "zen-html.zip") + + -- Check if zip needs regeneration + local need_update = not os.isfile(zip_path) + if not need_update then + local zip_mtime = os.mtime(zip_path) + for _, file in ipairs(os.files(path.join(html_dir, "**"))) do + if os.mtime(file) > zip_mtime then + need_update = true + break + end + end + end + + if need_update then + print("Regenerating zen trace viewer frontend zip...") + + os.tryrm(zip_path) + os.mkdir(zip_dir) + + import("detect.tools.find_7z") + local cmd_7z = find_7z() + if cmd_7z then + os.execv(cmd_7z, {"a", "-bso0", zip_path, path.join(html_dir, "*")}) + else + import("detect.tools.find_zip") + local zip_cmd = find_zip() + if zip_cmd then + local oldir = os.cd(html_dir) + os.execv(zip_cmd, {"-r", "-q", zip_path, "."}) + os.cd(oldir) + else + raise("Unable to find a suitable zip tool (need 7z or zip)") + end + end + end + end) + if is_plat("windows") then add_files("zen.rc") add_ldflags("/subsystem:console,5.02", {force = true}) diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 87d83cc77..3cc52f53b 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -9,27 +9,26 @@ #include "cmds/bench_cmd.h" #include "cmds/builds_cmd.h" #include "cmds/cache_cmd.h" -#include "cmds/copy_cmd.h" +#include "cmds/compute_cmd.h" #include "cmds/dedup_cmd.h" #include "cmds/exec_cmd.h" #include "cmds/help_cmd.h" +#include "cmds/history_cmd.h" #include "cmds/hub_cmd.h" #include "cmds/info_cmd.h" #include "cmds/print_cmd.h" #include "cmds/projectstore_cmd.h" -#include "cmds/rpcreplay_cmd.h" -#include "cmds/run_cmd.h" #include "cmds/serve_cmd.h" #include "cmds/service_cmd.h" #include "cmds/status_cmd.h" #include "cmds/top_cmd.h" -#include "cmds/trace_cmd.h" #include "cmds/ui_cmd.h" #include "cmds/up_cmd.h" #include "cmds/version_cmd.h" #include "cmds/vfs_cmd.h" #include "cmds/wipe_cmd.h" #include "cmds/workspaces_cmd.h" +#include "trace/trace_cmd.h" #include <zencore/callstack.h> #include <zencore/config.h> @@ -44,9 +43,12 @@ #include <zencore/trace.h> #include <zencore/windows.h> #include <zenhttp/httpcommon.h> +#include <zenutil/config/commandlineoptions.h> #include <zenutil/config/environmentoptions.h> #include <zenutil/consoletui.h> +#include <zenutil/invocationhistory.h> #include <zenutil/logging.h> +#include <zenutil/suggest.h> #include <zenutil/workerpools.h> #include <zenutil/zenserverprocess.h> @@ -56,7 +58,7 @@ #include <zencore/memory/memorytrace.h> #include <zencore/memory/newdelete.h> -#include "progressbar.h" +#include "consoleprogress.h" #if ZEN_WITH_TESTS # include <zencore/testing.h> @@ -189,6 +191,9 @@ ZenCmdBase::ParseOptionsPermissive(cxxopts::Options& CmdOptions, int argc, char* { CmdOptions.set_width(TuiConsoleColumns(80)); CmdOptions.allow_unrecognised_options(); + // Revert the flag on scope exit so re-parsing the same Options later is strict. + // cxxopts has no getter for the previous state, so we unconditionally clear it. + auto _ = MakeGuard([&]() { CmdOptions.disallow_unrecognised_options(); }); cxxopts::ParseResult Result; @@ -271,20 +276,34 @@ ZenCmdWithSubCommands::PrintHelp() Options().set_width(TuiConsoleColumns(80)); printf("%s\n", Options().help(Groups).c_str()); - // Append subcommand listing. + // Append subcommand listing. When a subcommand has aliases, display them as + // "name|alias1|alias2" in the left column so callers discover the alternate spellings. + auto FormatSubCmdName = [](ZenSubCmdBase& SubCmd) { + std::string Name(SubCmd.SubOptions().program()); + for (const std::string& Alias : SubCmd.Aliases()) + { + Name.push_back('|'); + Name.append(Alias); + } + return Name; + }; + + std::vector<std::string> FormattedNames; + FormattedNames.reserve(m_SubCommands.size()); size_t MaxNameLen = 0; for (ZenSubCmdBase* SubCmd : m_SubCommands) { - MaxNameLen = std::max(MaxNameLen, SubCmd->SubOptions().program().size()); + FormattedNames.push_back(FormatSubCmdName(*SubCmd)); + MaxNameLen = std::max(MaxNameLen, FormattedNames.back().size()); } printf("subcommands:\n"); - for (ZenSubCmdBase* SubCmd : m_SubCommands) + for (size_t i = 0; i < m_SubCommands.size(); ++i) { printf(" %-*s %s\n", static_cast<int>(MaxNameLen), - SubCmd->SubOptions().program().c_str(), - std::string(SubCmd->Description()).c_str()); + FormattedNames[i].c_str(), + std::string(m_SubCommands[i]->Description()).c_str()); } printf("\nFor global options run: zen --help\n"); } @@ -309,16 +328,33 @@ ZenCmdWithSubCommands::PrintSubCommandHelp(cxxopts::Options& SubCmdOptions) void ZenCmdWithSubCommands::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { - std::vector<cxxopts::Options*> SubOptionPtrs; - SubOptionPtrs.reserve(m_SubCommands.size()); - for (ZenSubCmdBase* SubCmd : m_SubCommands) - { - SubOptionPtrs.push_back(&SubCmd->SubOptions()); - } - + ZenSubCmdBase* MatchedSubCmd = nullptr; cxxopts::Options* MatchedSubOption = nullptr; std::vector<char*> SubCommandArguments; - int ParentArgc = GetSubCommand(Options(), argc, argv, SubOptionPtrs, MatchedSubOption, SubCommandArguments); + int ParentArgc = argc; + for (int i = 1; i < argc; ++i) + { + std::string_view Arg(argv[i]); + for (ZenSubCmdBase* SubCmd : m_SubCommands) + { + const bool NameMatch = SubCmd->SubOptions().program() == Arg; + const bool AliasMatch = + !NameMatch && std::find(SubCmd->Aliases().begin(), SubCmd->Aliases().end(), Arg) != SubCmd->Aliases().end(); + if (NameMatch || AliasMatch) + { + MatchedSubCmd = SubCmd; + MatchedSubOption = &SubCmd->SubOptions(); + break; + } + } + if (MatchedSubCmd != nullptr) + { + SubCommandArguments.push_back(argv[0]); + std::copy(&argv[i + 1], &argv[argc], std::back_inserter(SubCommandArguments)); + ParentArgc = i + 1; + break; + } + } // Intercept --help/-h in the parent arg range before calling ParseOptions so // we can append subcommand information to the output. When a subcommand was @@ -336,6 +372,46 @@ ZenCmdWithSubCommands::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (MatchedSubOption == nullptr) { + // If the user typed what looks like a subcommand name (a non-option arg) but nothing + // matched, surface "did you mean" suggestions before falling through to full help. + std::string_view UnknownAttempt; + for (int i = 1; i < argc; ++i) + { + std::string_view Arg(argv[i]); + if (!Arg.empty() && Arg[0] != '-') + { + UnknownAttempt = Arg; + break; + } + } + + if (!UnknownAttempt.empty() && !m_SubCommands.empty()) + { + std::vector<std::string_view> SubNames; + SubNames.reserve(m_SubCommands.size() * 2); + for (ZenSubCmdBase* SubCmd : m_SubCommands) + { + SubNames.emplace_back(SubCmd->SubOptions().program()); + for (const std::string& Alias : SubCmd->Aliases()) + { + SubNames.emplace_back(Alias); + } + } + std::vector<std::string_view> Suggestions = SuggestSimilarCommands(UnknownAttempt, SubNames); + if (!Suggestions.empty()) + { + printf("Unknown subcommand: '%.*s'\n\n", static_cast<int>(UnknownAttempt.size()), UnknownAttempt.data()); + printf("The most similar subcommands are:\n"); + for (std::string_view Name : Suggestions) + { + printf(" %.*s\n", static_cast<int>(Name.size()), Name.data()); + } + printf("\n"); + fflush(stdout); + throw OptionParseException("Unknown subcommand", {}); + } + } + if (!ParseOptions(Options(), ParentArgc, argv)) { return; @@ -345,15 +421,6 @@ ZenCmdWithSubCommands::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw OptionParseException("No subcommand specified", {}); } - ZenSubCmdBase* MatchedSubCmd = nullptr; - for (ZenSubCmdBase* SubCmd : m_SubCommands) - { - if (&SubCmd->SubOptions() == MatchedSubOption) - { - MatchedSubCmd = SubCmd; - break; - } - } ZEN_ASSERT(MatchedSubCmd != nullptr); // Intercept --help/-h in the subcommand args so we can show combined help @@ -369,7 +436,7 @@ ZenCmdWithSubCommands::Run(const ZenCliOptions& GlobalOptions, int argc, char** } } - // Parse subcommand args permissively — unrecognised options are collected + // Parse subcommand args permissively - unrecognised options are collected // and forwarded to the parent parser so that parent options (e.g. --path) // can appear after the subcommand name on the command line. std::vector<std::string> SubUnmatched; @@ -552,6 +619,8 @@ main(int argc, char** argv) { zen::InstallCrashHandler(); + zen::LogInvocation("zen", /*Mode*/ "", argc, argv, {zen::HistoryCommand::Name}); + #if ZEN_PLATFORM_WINDOWS setlocale(LC_ALL, "en_us.UTF8"); #endif // ZEN_PLATFORM_WINDOWS @@ -575,12 +644,12 @@ main(int argc, char** argv) AttachCommand AttachCmd; BenchCommand BenchCmd; BuildsCommand BuildsCmd; + CacheCommand CacheCmd; CacheDetailsCommand CacheDetailsCmd; CacheGetCommand CacheGetCmd; CacheGenerateCommand CacheGenerateCmd; CacheInfoCommand CacheInfoCmd; CacheStatsCommand CacheStatsCmd; - CopyCommand CopyCmd; CopyStateCommand CopyStateCmd; CreateOplogCommand CreateOplogCmd; CreateProjectCommand CreateProjectCmd; @@ -589,7 +658,8 @@ main(int argc, char** argv) DropCommand DropCmd; DropProjectCommand ProjectDropCmd; #if ZEN_WITH_COMPUTE_SERVICES - ExecCommand ExecCmd; + ComputeCommand ComputeCmd; + ExecCommand ExecCmd; #endif // ZEN_WITH_COMPUTE_SERVICES ExportOplogCommand ExportOplogCmd; FlushCommand FlushCmd; @@ -597,6 +667,7 @@ main(int argc, char** argv) GcStatusCommand GcStatusCmd; GcStopCommand GcStopCmd; HelpCommand HelpCmd; + HistoryCommand HistoryCmd; HubCommand HubCmd; ImportOplogCommand ImportOplogCmd; InfoCommand InfoCmd; @@ -614,7 +685,6 @@ main(int argc, char** argv) RpcReplayCommand RpcReplayCmd; RpcStartRecordingCommand RpcStartRecordingCmd; RpcStopRecordingCommand RpcStopRecordingCmd; - RunCommand RunCmd; ScrubCommand ScrubCmd; ServeCommand ServeCmd; StatusCommand StatusCmd; @@ -635,23 +705,25 @@ main(int argc, char** argv) {AttachCommand::Name, &AttachCmd, AttachCommand::Description}, {BenchCommand::Name, &BenchCmd, BenchCommand::Description}, {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description}, + {CacheCommand::Name, &CacheCmd, CacheCommand::Description}, {CacheDetailsCommand::Name, &CacheDetailsCmd, CacheDetailsCommand::Description}, {CacheInfoCommand::Name, &CacheInfoCmd, CacheInfoCommand::Description}, {CacheGetCommand::Name, &CacheGetCmd, CacheGetCommand::Description}, {CacheGenerateCommand::Name, &CacheGenerateCmd, CacheGenerateCommand::Description}, {CacheStatsCommand::Name, &CacheStatsCmd, CacheStatsCommand::Description}, - {CopyCommand::Name, &CopyCmd, CopyCommand::Description}, {CopyStateCommand::Name, &CopyStateCmd, CopyStateCommand::Description}, {DedupCommand::Name, &DedupCmd, DedupCommand::Description}, {DownCommand::Name, &DownCmd, DownCommand::Description}, {DropCommand::Name, &DropCmd, DropCommand::Description}, #if ZEN_WITH_COMPUTE_SERVICES + {ComputeCommand::Name, &ComputeCmd, ComputeCommand::Description}, {ExecCommand::Name, &ExecCmd, ExecCommand::Description}, #endif {GcStatusCommand::Name, &GcStatusCmd, GcStatusCommand::Description}, {GcStopCommand::Name, &GcStopCmd, GcStopCommand::Description}, {GcCommand::Name, &GcCmd, GcCommand::Description}, - {HelpCommand::Name, &HelpCmd, HelpCommand::Description}, + {HelpCommand::Name, &HelpCmd, HelpCommand::Description}, + {HistoryCommand::Name, &HistoryCmd, HistoryCommand::Description}, {HubCommand::Name, &HubCmd, HubCommand::Description}, {InfoCommand::Name, &InfoCmd, InfoCommand::Description}, {JobCommand::Name, &JobCmd, JobCommand::Description}, @@ -674,7 +746,6 @@ main(int argc, char** argv) {RpcReplayCommand::Name, &RpcReplayCmd, RpcReplayCommand::Description}, {RpcStartRecordingCommand::Name, &RpcStartRecordingCmd, RpcStartRecordingCommand::Description}, {RpcStopRecordingCommand::Name, &RpcStopRecordingCmd, RpcStopRecordingCommand::Description}, - {RunCommand::Name, &RunCmd, RunCommand::Description}, {ScrubCommand::Name, &ScrubCmd, ScrubCommand::Description}, {ServeCommand::Name, &ServeCmd, ServeCommand::Description}, {StatusCommand::Name, &StatusCmd, StatusCommand::Description}, @@ -795,6 +866,9 @@ main(int argc, char** argv) Options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug)); Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value<bool>(GlobalOptions.IsVerbose)); + Options.add_options()("enable-execution-history", + "Record this invocation in the per-user execution history (use --enable-execution-history=false to suppress)", + cxxopts::value<bool>(GlobalOptions.EnableExecutionHistory)->default_value("true")->implicit_value("true")); Options.add_options()("malloc", "Configure memory allocator subsystem", cxxopts::value(MemoryOptions)->default_value("mimalloc")); Options.add_options()("help", "Show command line help"); Options.add_options()("c, command", "Sub command", cxxopts::value<std::string>(SubCommand)); @@ -875,6 +949,11 @@ main(int argc, char** argv) for (const CommandInfo& CmdInfo : Commands) { + if (CmdInfo.Cmd->IsHidden()) + { + continue; + } + ZenCmdCategory& Category = CmdInfo.Cmd->CommandCategory(); Categories[Category.Name] = &Category; @@ -932,13 +1011,19 @@ main(int argc, char** argv) { SB.Append(' '); } - - SB.Append(argv[i]); + SB.Append(std::string_view(argv[i])); } SentryConfig.DatabasePath = SentryDatabasePath; - Sentry.Initialize(SentryConfig, SB.ToString()); + if (std::filesystem::path HistoryPath = GetInvocationHistoryPath(); !HistoryPath.empty()) + { + SentryConfig.AttachmentPaths.push_back(std::move(HistoryPath)); + } + + std::string ScrubbedCmdLine = SB.ToString(); + ScrubSensitiveValues(ScrubbedCmdLine); + Sentry.Initialize(SentryConfig, ScrubbedCmdLine); SentryIntegration::ClearCaches(); } @@ -1036,7 +1121,28 @@ main(int argc, char** argv) } } - printf("Unknown command specified: '%s', exiting\n", SubCommand.c_str()); + printf("Unknown command specified: '%s'\n", SubCommand.c_str()); + + std::vector<std::string_view> VisibleNames; + VisibleNames.reserve(std::size(Commands)); + for (const CommandInfo& CmdInfo : Commands) + { + if (!CmdInfo.Cmd->IsHidden()) + { + VisibleNames.emplace_back(CmdInfo.CmdName); + } + } + + std::vector<std::string_view> Suggestions = zen::SuggestSimilarCommands(SubCommand, VisibleNames); + if (!Suggestions.empty()) + { + printf("\nThe most similar commands are:\n"); + for (std::string_view Name : Suggestions) + { + printf(" %.*s\n", static_cast<int>(Name.size()), Name.data()); + } + } + printf("\nRun 'zen --help' for the full list of commands.\n"); return (int)ReturnCode::kBadInput; } catch (const OptionParseException& Ex) diff --git a/src/zen/zen.h b/src/zen/zen.h index 80178910a..5cf3b99ed 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -9,12 +9,15 @@ #include <zenutil/config/commandlineoptions.h> #include <zenutil/config/loggingconfig.h> +#include <csignal> + namespace zen { struct ZenCliOptions { - bool IsDebug = false; - bool IsVerbose = false; + bool IsDebug = false; + bool IsVerbose = false; + bool EnableExecutionHistory = true; ZenLoggingConfig LoggingConfig; @@ -37,6 +40,33 @@ extern ZenCmdCategory g_ProjectStoreCategory; extern ZenCmdCategory g_CacheStoreCategory; extern ZenCmdCategory g_StorageCategory; +// RAII wrapper around `signal(2)` that restores the previous handler on scope +// exit. Use in command Run() methods so an exception during option parsing or +// execution doesn't leave a handler installed whose flag is scoped to a +// now-dead lifetime. +class ScopedSignalHandler +{ +public: + using Handler = void (*)(int); + + ScopedSignalHandler(int SigNum, Handler NewHandler) : m_SigNum(SigNum), m_PrevHandler(std::signal(SigNum, NewHandler)) {} + + ~ScopedSignalHandler() + { + if (m_PrevHandler != SIG_ERR) + { + std::signal(m_SigNum, m_PrevHandler); + } + } + + ScopedSignalHandler(const ScopedSignalHandler&) = delete; + ScopedSignalHandler& operator=(const ScopedSignalHandler&) = delete; + +private: + int m_SigNum; + Handler m_PrevHandler; +}; + class ErrorWithReturnCode : public std::runtime_error { public: @@ -58,6 +88,11 @@ public: virtual cxxopts::Options& Options() = 0; virtual ZenCmdCategory& CommandCategory() const; + // Hidden commands are dispatched normally but omitted from the top-level + // `zen --help` listing. Used for deprecated aliases that remain functional + // but should not be advertised. + virtual bool IsHidden() const { return false; } + bool ParseOptions(int argc, char** argv); static bool ParseOptions(cxxopts::Options& Options, int argc, char** argv); static bool ParseOptionsPermissive(cxxopts::Options& Options, int argc, char** argv, std::vector<std::string>& OutUnmatched); @@ -103,15 +138,19 @@ class ZenSubCmdBase public: ZenSubCmdBase(std::string_view Name, std::string_view Description); virtual ~ZenSubCmdBase() = default; - cxxopts::Options& SubOptions() { return m_SubOptions; } - std::string_view Description() const { return m_Description; } - virtual void Run(const ZenCliOptions& GlobalOptions) = 0; + cxxopts::Options& SubOptions() { return m_SubOptions; } + std::string_view Description() const { return m_Description; } + const std::vector<std::string>& Aliases() const { return m_Aliases; } + virtual void Run(const ZenCliOptions& GlobalOptions) = 0; protected: + void AddAlias(std::string Alias) { m_Aliases.push_back(std::move(Alias)); } + cxxopts::Options m_SubOptions; private: - std::string m_Description; + std::string m_Description; + std::vector<std::string> m_Aliases; }; // Base for commands that host subcommands - handles all dispatch boilerplate diff --git a/src/zen/zenserviceclient.cpp b/src/zen/zenserviceclient.cpp new file mode 100644 index 000000000..87d0a6c26 --- /dev/null +++ b/src/zen/zenserviceclient.cpp @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenserviceclient.h" + +#include "zen.h" + +#include <zencore/logging.h> +#include <zencore/logging/broadcastsink.h> +#include <zencore/session.h> +#include <zenutil/logging.h> + +namespace zen { + +ZenServiceClient::ZenServiceClient(Options Opts) +: m_HostSpec(ZenCmdBase::ResolveTargetHostSpec(Opts.HostSpec)) +, m_Http(ZenCmdBase::CreateHttpClient(m_HostSpec, Opts.HttpSettings)) +{ + if (m_HostSpec.empty()) + { + throw OptionParseException("Unable to resolve server specification", {}); + } + + SessionsServiceClient::Options SessionOpts{ + .TargetUrl = m_HostSpec, + .AppName = "zen", + .Mode = std::move(Opts.CommandName), + .SessionId = GetSessionId(), + }; + + // For unix socket connections, forward the socket path to the sessions client + if (ZenCmdBase::IsUnixSocketSpec(m_HostSpec)) + { + SessionOpts.TargetUrl = "http://localhost"; + SessionOpts.ClientSettings.UnixSocketPath = m_HostSpec.substr(7); // strip "unix://" + } + + m_Sessions = std::make_unique<SessionsServiceClient>(std::move(SessionOpts)); + m_Sessions->Announce(); + + m_LogSink = m_Sessions->CreateLogSink(); + GetDefaultBroadcastSink()->AddSink(m_LogSink); +} + +ZenServiceClient::~ZenServiceClient() +{ + if (m_LogSink) + { + if (Ref<logging::BroadcastSink> Broadcast = GetDefaultBroadcastSink()) + { + Broadcast->RemoveSink(m_LogSink); + } + } +} + +} // namespace zen diff --git a/src/zen/zenserviceclient.h b/src/zen/zenserviceclient.h new file mode 100644 index 000000000..00178b455 --- /dev/null +++ b/src/zen/zenserviceclient.h @@ -0,0 +1,43 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/sink.h> +#include <zenhttp/httpclient.h> +#include <zenutil/sessionsclient.h> + +#include <memory> +#include <string> + +namespace zen { + +/// RAII wrapper that combines host resolution, HTTP client creation, and session lifecycle. +/// On construction, resolves the host, creates an HttpClient, and announces a session. +/// On destruction, removes the session (best-effort). +class ZenServiceClient +{ +public: + struct Options + { + std::string HostSpec; // Raw host spec (empty = auto-resolve) + std::string CommandName; // e.g. "info", "gc" — used as session Mode + HttpClientSettings HttpSettings; // Forwarded to HttpClient + }; + + explicit ZenServiceClient(Options Opts); + ~ZenServiceClient(); + + ZenServiceClient(const ZenServiceClient&) = delete; + ZenServiceClient& operator=(const ZenServiceClient&) = delete; + + HttpClient& Http() { return m_Http; } + const std::string& HostSpec() const { return m_HostSpec; } + +private: + std::string m_HostSpec; + HttpClient m_Http; + std::unique_ptr<SessionsServiceClient> m_Sessions; + logging::SinkPtr m_LogSink; +}; + +} // namespace zen |