From 58e1e1ef2deedc49b3e88db57c110b88a39e21da Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 13 Feb 2026 14:47:24 +0100 Subject: spelling fixes (#755) --- src/zen/cmds/builds_cmd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index f4edb65ab..59b209384 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3016,7 +3016,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BlobHash.length() != IoHash::StringLength) { throw OptionParseException( - fmt::format("'--blob-hash' ('{}') is malfomed, it must be {} characters long", m_BlobHash, IoHash::StringLength), + fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", m_BlobHash, IoHash::StringLength), SubOption->help()); } @@ -3033,7 +3033,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BuildId.length() != Oid::StringLength) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malfomed, it must be {} characters long", m_BuildId, Oid::StringLength), + fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildId, Oid::StringLength), SubOption->help()); } else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero) -- cgit v1.2.3 From 5789739f042791d38f0d7f4219e5b0b823d8b3d6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 20 Feb 2026 09:07:00 +0100 Subject: fix MakeSafeAbsolutePathInPlace mis-spelling (#765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (was MakeSafeAbsolutePathÍnPlace - note accent) Also fixed misleading comments on multiple functions in filesystem.h --- src/zen/cmds/builds_cmd.cpp | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 59b209384..8dfe1093f 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2680,7 +2680,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + MakeSafeAbsolutePathInPlace(m_SystemRootDir); }; ParseSystemOptions(); @@ -2729,7 +2729,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(m_StoragePath); }; auto ParseOutputOptions = [&]() { @@ -2947,7 +2947,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--local-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_Path); + MakeSafeAbsolutePathInPlace(m_Path); }; auto ParseFileFilters = [&](std::vector& OutIncludeWildcards, std::vector& OutExcludeWildcards) { @@ -3004,7 +3004,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--compare-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_DiffPath); + MakeSafeAbsolutePathInPlace(m_DiffPath); }; auto ParseBlobHash = [&]() -> IoHash { @@ -3105,7 +3105,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_BuildMetadataPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath); + MakeSafeAbsolutePathInPlace(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -3202,8 +3202,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ListOptions) { - MakeSafeAbsolutePathÍnPlace(m_ListQueryPath); - MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + MakeSafeAbsolutePathInPlace(m_ListQueryPath); + MakeSafeAbsolutePathInPlace(m_ListResultPath); if (!m_ListResultPath.empty()) { @@ -3255,7 +3255,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); @@ -3294,7 +3294,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ListBlocksOptions) { - MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + MakeSafeAbsolutePathInPlace(m_ListResultPath); if (!m_ListResultPath.empty()) { @@ -3316,7 +3316,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); @@ -3387,8 +3387,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3532,7 +3532,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -3632,7 +3632,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -3652,7 +3652,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::unique_ptr StructuredOutput; if (!m_LsResultPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_LsResultPath); + MakeSafeAbsolutePathInPlace(m_LsResultPath); StructuredOutput = std::make_unique(); } @@ -3696,7 +3696,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParsePath(); ParseDiffPath(); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); std::vector ExcludeFolders = DefaultExcludeFolders; std::vector ExcludeExtensions = DefaultExcludeExtensions; @@ -3745,7 +3745,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3828,7 +3828,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3883,7 +3883,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3933,7 +3933,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(); @@ -4083,8 +4083,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, -- cgit v1.2.3 From 80bc5a53fe9077bc20d287b912f6476db233110c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 20 Feb 2026 10:31:31 +0100 Subject: fix builds download indexing timer (#769) * fix build download indexing timer log --- src/zen/cmds/builds_cmd.cpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 8dfe1093f..849259013 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1467,9 +1467,16 @@ namespace { ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); } + Stopwatch IndexTimer; + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + } + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); BuildsOperationUpdateFolder Updater( -- cgit v1.2.3 From eb3079e2ec2969829cbc5b6921575d53df351f0f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 Feb 2026 16:10:36 +0100 Subject: use partial blocks for oplog import (#780) Feature: Add --allow-partial-block-requests to zen oplog-import Improvement: zen oplog-import now uses partial block requests to reduce download size Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests --- src/zen/cmds/builds_cmd.cpp | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 849259013..5254ef3cf 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2842,13 +2842,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) TempPath / "storage"); Result.StorageName = ResolveRes.HostName; - StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - ResolveRes.HostName, - (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket); - ; + uint64_t HostLatencyNs = ResolveRes.HostLatencySec >= 0 ? uint64_t(ResolveRes.HostLatencySec * 1000000000.0) : 0; + + StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", + ResolveRes.HostName, + (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket, + NiceLatencyNs(HostLatencyNs)); + Result.BuildStorageLatencySec = ResolveRes.HostLatencySec; if (!ResolveRes.CacheUrl.empty()) { @@ -2874,12 +2877,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) : GetTinyWorkerPool(EWorkloadType::Background)); Result.CacheName = ResolveRes.CacheName; + uint64_t CacheLatencyNs = ResolveRes.CacheLatencySec >= 0 ? uint64_t(ResolveRes.CacheLatencySec * 1000000000.0) : 0; + CacheDescription = - fmt::format("Zen {}{}. SessionId: '{}'", + fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}", ResolveRes.CacheName, (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), - Result.CacheHttp->GetSessionId()); - ; + Result.CacheHttp->GetSessionId(), + NiceLatencyNs(CacheLatencyNs)); + + Result.CacheLatencySec = ResolveRes.CacheLatencySec; + if (!m_Namespace.empty()) { CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); -- cgit v1.2.3 From b67dac7c093cc82b7e8f12f9eb57bfa34dfe26d8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 08:35:32 +0100 Subject: unity build fixes (#802) Various fixes to make cpp files build in unity build mode as an aside using Unity build doesn't really seem to work on Linux, unsure why but it leads to link-time issues --- src/zen/cmds/builds_cmd.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 5254ef3cf..ffdc5fe48 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -67,13 +67,11 @@ ZEN_THIRD_PARTY_INCLUDES_END static const bool DoExtraContentVerify = false; -#define ZEN_CLOUD_STORAGE "Cloud Storage" - namespace zen { using namespace std::literals; -namespace { +namespace builds_impl { static std::atomic AbortFlag = false; static std::atomic PauseFlag = false; @@ -270,6 +268,7 @@ namespace { static bool IsQuiet = false; static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; +#undef ZEN_CONSOLE_VERBOSE #define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ if (IsVerbose) \ { \ @@ -2009,12 +2008,13 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } -} // namespace +} // namespace builds_impl ////////////////////////////////////////////////////////////////////////////////////////////////////// BuildsCommand::BuildsCommand() { + using namespace builds_impl; m_Options.add_options()("h,help", "Print help"); auto AddSystemOptions = [this](cxxopts::Options& Ops) { @@ -2655,6 +2655,7 @@ BuildsCommand::~BuildsCommand() = default; void BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace builds_impl; ZEN_UNUSED(GlobalOptions); signal(SIGINT, SignalCallbackHandler); -- cgit v1.2.3 From 6926c04dc4d7c5c0f0310b66c17c9a4e94d2e341 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Mar 2026 16:07:14 +0100 Subject: more feedback during auth option parsing (#806) * remove stray std::unique_ptr Auth; causing crashes * add more feedback during parsing of auth options --- src/zen/cmds/builds_cmd.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index ffdc5fe48..0722e9714 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2808,8 +2808,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .Verbose = m_VerboseHttp, .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; - std::unique_ptr Auth; - std::string StorageDescription; std::string CacheDescription; -- cgit v1.2.3 From 19a117889c2db6b817af9458c04c04f324162e75 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 9 Mar 2026 10:50:47 +0100 Subject: Eliminate spdlog dependency (#773) Removes the vendored spdlog library (~12,000 lines) and replaces it with a purpose-built logging system in zencore (~1,800 lines). The new implementation provides the same functionality with fewer abstractions, no shared_ptr overhead, and full control over the logging pipeline. ### What changed **New logging core in zencore/logging/:** - LogMessage, Formatter, Sink, Logger, Registry - core abstractions matching spdlog's model but simplified - AnsiColorStdoutSink - ANSI color console output (replaces spdlog stdout_color_sink) - MsvcSink - OutputDebugString on Windows (replaces spdlog msvc_sink) - AsyncSink - async logging via BlockingQueue worker thread (replaces spdlog async_logger) - NullSink, MessageOnlyFormatter - utility types - Thread-safe timestamp caching in formatters using RwLock **Moved to zenutil/logging/:** - FullFormatter - full log formatting with timestamp, logger name, level, source location, multiline alignment - JsonFormatter - structured JSON log output - RotatingFileSink - rotating file sink with atomic size tracking **API changes:** - Log levels are now an enum (LogLevel) instead of int, eliminating the zen::logging::level namespace - LoggerRef no longer wraps shared_ptr - it holds a raw pointer with the registry owning lifetime - Logger error handler is wired through Registry and propagated to all loggers on registration - Logger::Log() now populates ThreadId on every message **Cleanup:** - Deleted thirdparty/spdlog/ entirely (110+ files) - Deleted full_test_formatter (was ~80% duplicate of FullFormatter) - Renamed snake_case classes to PascalCase (full_formatter -> FullFormatter, json_formatter -> JsonFormatter, sentry_sink -> SentrySink) - Removed spdlog from xmake dependency graph ### Build / test impact - zencore no longer depends on spdlog - zenutil and zenvfs xmake.lua updated to drop spdlog dep - zentelemetry xmake.lua updated to drop spdlog dep - All existing tests pass, no test changes required beyond formatter class renames --- src/zen/cmds/builds_cmd.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 0722e9714..e5cbafbea 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -269,10 +269,10 @@ namespace builds_impl { static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; #undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ } const std::string DefaultAccessTokenEnvVariableName( -- cgit v1.2.3 From f9d8cbcb3573b47b639b7bd73d3a4eed17653d71 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Mar 2026 13:08:00 +0100 Subject: add fallback for zencache multirange (#816) * clean up BuildStorageResolveResult to allow capabilities * add check for multirange request capability * add MaxRangeCountPerRequest capabilities * project export tests * add InMemoryBuildStorageCache * progress and logging improvements * fix ElapsedSeconds calculations in fileremoteprojectstore.cpp * oplogs/builds test script --- src/zen/cmds/builds_cmd.cpp | 139 ++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 63 deletions(-) (limited to 'src/zen/cmds/builds_cmd.cpp') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index e5cbafbea..b4b4df7c9 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1594,7 +1594,7 @@ namespace builds_impl { } } } - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { if (SB.Size() > 0) { @@ -1649,9 +1649,9 @@ namespace builds_impl { } if (Options.PrimeCacheOnly) { - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { - Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { + Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) { if (!IsQuiet) { if (Remaining == 0) @@ -2826,47 +2826,47 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); - if (!ResolveRes.HostUrl.empty()) + if (!ResolveRes.Cloud.Address.empty()) { - ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2; + ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique(ResolveRes.HostUrl, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); - Result.BuildStorage = CreateJupiterBuildStorage(Log(), + Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, TempPath / "storage"); - Result.StorageName = ResolveRes.HostName; + Result.BuildStorageHost = ResolveRes.Cloud; - uint64_t HostLatencyNs = ResolveRes.HostLatencySec >= 0 ? uint64_t(ResolveRes.HostLatencySec * 1000000000.0) : 0; + uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; - StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", - ResolveRes.HostName, - (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, - NiceLatencyNs(HostLatencyNs)); - Result.BuildStorageLatencySec = ResolveRes.HostLatencySec; + StorageDescription = + fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", + ResolveRes.Cloud.Name, + (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket, + NiceLatencyNs(HostLatencyNs)); - if (!ResolveRes.CacheUrl.empty()) + if (!ResolveRes.Cache.Address.empty()) { Result.CacheHttp = std::make_unique( - ResolveRes.CacheUrl, + ResolveRes.Cache.Address, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, + .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, .Verbose = m_VerboseHttp, .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, []() { return AbortFlag.load(); }); - Result.BuildCacheStorage = + Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, @@ -2874,19 +2874,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) TempPath / "zencache", BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheName = ResolveRes.CacheName; + Result.CacheHost = ResolveRes.Cache; - uint64_t CacheLatencyNs = ResolveRes.CacheLatencySec >= 0 ? uint64_t(ResolveRes.CacheLatencySec * 1000000000.0) : 0; + uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}", - ResolveRes.CacheName, - (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), + ResolveRes.Cache.Name, + (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address), Result.CacheHttp->GetSessionId(), NiceLatencyNs(CacheLatencyNs)); - Result.CacheLatencySec = ResolveRes.CacheLatencySec; - if (!m_Namespace.empty()) { CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); @@ -2902,41 +2900,56 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { StorageDescription = fmt::format("folder {}", m_StoragePath); Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + .Name = "Disk", + .LatencySec = 1.0 / 100000, // 1 us + .Caps = {.MaxRangeCountPerRequest = 2048u}}; if (!m_ZenCacheHost.empty()) { - Result.CacheHttp = std::make_unique( - m_ZenCacheHost, - HttpClientSettings{ - .LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); - Result.BuildCacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheName = m_ZenCacheHost; - - CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", Result.CacheName, "", Result.CacheHttp->GetSessionId()); - ; - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); - } - if (!m_Bucket.empty()) + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + + if (TestResult.Success) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + Result.CacheHttp = std::make_unique( + m_ZenCacheHost, + HttpClientSettings{ + .LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 0, + .Verbose = m_VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, + []() { return AbortFlag.load(); }); + + Result.CacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = + BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, + .Name = m_ZenCacheHost, + .AssumeHttp2 = m_AssumeHttp2, + .LatencySec = TestResult.LatencySeconds, + .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; + + CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); + + if (!m_Namespace.empty()) + { + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + } + if (!m_Bucket.empty()) + { + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + } } } } @@ -2948,7 +2961,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsQuiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); - if (!Result.CacheName.empty()) + if (!Result.CacheHost.Name.empty()) { ZEN_CONSOLE("Cache : {}", CacheDescription); } @@ -3489,7 +3502,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - Storage.StorageName, + Storage.BuildStorageHost.Name, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -3810,12 +3823,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsQuiet) { - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { - ZEN_CONSOLE("Uploaded {} ({}) blobs", + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", StorageCacheStats.PutBlobCount.load(), NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheName); + Storage.CacheHost.Name); } } -- cgit v1.2.3