aboutsummaryrefslogtreecommitdiff
path: root/src/zen/cmds/builds_cmd.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-08-11 12:58:27 +0200
committerGitHub Enterprise <[email protected]>2025-08-11 12:58:27 +0200
commiteeac0654ab8f9e3438f2331916261a0440286fbd (patch)
tree56bd48ef8dedf91eeb25a7c26471742ab189874c /src/zen/cmds/builds_cmd.cpp
parentMerge pull request #434 from ue-foundation/zs/put-overwrite-policy (diff)
downloadarchived-zen-eeac0654ab8f9e3438f2331916261a0440286fbd.tar.xz
archived-zen-eeac0654ab8f9e3438f2331916261a0440286fbd.zip
list build part content (#462)
- Feature: Added `zen build ls` option to list the content of a build part(s) - Build source is specified using one of the following options - `--cloud-url` cloud artifact URL to build - `--host` or `--override-host`, `--namespace`, `--bucket` and `--buildid` - `--filestorage`, `--namespace`, `--bucket` and `--buildid` - `--build-part-name` to specify a particular build part(s) in the build - `--wildcard` windows style wildcard (using * and ?) to match file paths to include - `--exclude-wildcard` windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter - Improvement: Added `--quiet` option to zen `builds` commands to suppress non-essential output
Diffstat (limited to 'src/zen/cmds/builds_cmd.cpp')
-rw-r--r--src/zen/cmds/builds_cmd.cpp957
1 files changed, 666 insertions, 291 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 7cb12cd4c..8e17ba97c 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -33,6 +33,7 @@
#include <zenutil/jupiter/jupiterbuildstorage.h>
#include <zenutil/jupiter/jupitersession.h>
#include <zenutil/parallelwork.h>
+#include <zenutil/wildcard.h>
#include <zenutil/workerpools.h>
#include <zenutil/zenserverprocess.h>
@@ -65,6 +66,7 @@ using namespace std::literals;
namespace {
namespace zenutil {
+
#if ZEN_PLATFORM_WINDOWS
class SecurityAttributes
{
@@ -368,6 +370,7 @@ namespace {
const std::vector<std::string_view> DefaultExcludeExtensions({});
static bool IsVerbose = false;
+ static bool IsQuiet = false;
static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode)
@@ -402,6 +405,20 @@ namespace {
);
+ bool IncludePath(const std::string_view IncludeWildcard, const std::string_view ExcludeWildcard, const std::filesystem::path& Path)
+ {
+ const std::string PathString = Path.generic_string();
+ if (!IncludeWildcard.empty() && !MatchWildcard(IncludeWildcard, PathString, /*CaseSensitive*/ false))
+ {
+ return false;
+ }
+ if (!ExcludeWildcard.empty() && MatchWildcard(ExcludeWildcard, PathString, /*CaseSensitive*/ false))
+ {
+ return false;
+ }
+ return true;
+ }
+
bool IsFileWithRetry(const std::filesystem::path& Path)
{
std::error_code Ec;
@@ -795,7 +812,7 @@ namespace {
}
uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
- if (ElapsedTimeMs >= 200)
+ if (ElapsedTimeMs >= 200 && !IsQuiet)
{
ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}",
Path,
@@ -2134,11 +2151,14 @@ namespace {
Stopwatch Timer;
auto _ = MakeGuard([&]() {
- ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
- BuildId,
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
+ BuildId,
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
});
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount);
@@ -2163,7 +2183,10 @@ namespace {
CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId);
ValidateStats.BuildPartSize = BuildPart.GetSize();
- ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize()));
+ }
std::vector<IoHash> ChunkAttachments;
for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv])
{
@@ -3770,14 +3793,17 @@ namespace {
}
}
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- LocalContent.Paths.size(),
- NiceBytes(TotalRawSize),
- ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(ChunkingStats.UniqueBytesFound.load()),
- Path,
- NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ LocalContent.Paths.size(),
+ NiceBytes(TotalRawSize),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()),
+ Path,
+ NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
}
const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
@@ -3790,15 +3816,18 @@ namespace {
PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get();
- ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}",
- NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
- CreateBuild ? "PutBuild" : "GetBuild",
- NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
- NiceBytes(PrepBuildResult.PayloadSize),
- IgnoreExistingBlocks ? ""
- : fmt::format(". Found {} blocks in {}",
- PrepBuildResult.KnownBlocks.size(),
- NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}",
+ NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
+ CreateBuild ? "PutBuild" : "GetBuild",
+ NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
+ NiceBytes(PrepBuildResult.PayloadSize),
+ IgnoreExistingBlocks ? ""
+ : fmt::format(". Found {} blocks in {}",
+ PrepBuildResult.KnownBlocks.size(),
+ NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ }
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount);
@@ -3829,7 +3858,10 @@ namespace {
if (IgnoreExistingBlocks)
{
- ZEN_CONSOLE("Ignoring any existing blocks in store");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Ignoring any existing blocks in store");
+ }
NewBlockChunkIndexes = std::move(BlockChunkIndexes);
}
else
@@ -3872,40 +3904,43 @@ namespace {
FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) /
(FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount)
: 0.0;
- ZEN_CONSOLE(
- "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
- " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
- " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
- " Rejected {} ({}) chunks in {} blocks\n"
- " Arranged {} ({}) chunks in {} new blocks\n"
- " Keeping {} ({}) chunks as loose chunks\n"
- " Discovery completed in {}",
- FindBlocksStats.FoundBlockChunkCount,
- FindBlocksStats.FoundBlockCount,
- NiceBytes(FindBlocksStats.FoundBlockByteCount),
- NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
+ " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
+ " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
+ " Rejected {} ({}) chunks in {} blocks\n"
+ " Arranged {} ({}) chunks in {} new blocks\n"
+ " Keeping {} ({}) chunks as loose chunks\n"
+ " Discovery completed in {}",
+ FindBlocksStats.FoundBlockChunkCount,
+ FindBlocksStats.FoundBlockCount,
+ NiceBytes(FindBlocksStats.FoundBlockByteCount),
+ NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
- FindBlocksStats.AcceptedChunkCount,
- NiceBytes(FindBlocksStats.AcceptedRawByteCount),
- FindBlocksStats.AcceptedBlockCount,
- AcceptedByteCountPercent,
+ FindBlocksStats.AcceptedChunkCount,
+ NiceBytes(FindBlocksStats.AcceptedRawByteCount),
+ FindBlocksStats.AcceptedBlockCount,
+ AcceptedByteCountPercent,
- FindBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(FindBlocksStats.AcceptedReduntantByteCount),
- AcceptedReduntantByteCountPercent,
+ FindBlocksStats.AcceptedReduntantChunkCount,
+ NiceBytes(FindBlocksStats.AcceptedReduntantByteCount),
+ AcceptedReduntantByteCountPercent,
- FindBlocksStats.RejectedChunkCount,
- NiceBytes(FindBlocksStats.RejectedByteCount),
- FindBlocksStats.RejectedBlockCount,
+ FindBlocksStats.RejectedChunkCount,
+ NiceBytes(FindBlocksStats.RejectedByteCount),
+ FindBlocksStats.RejectedBlockCount,
- FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
- FindBlocksStats.NewBlocksCount,
+ FindBlocksStats.NewBlocksChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ FindBlocksStats.NewBlocksCount,
- LooseChunksStats.ChunkCount,
- NiceBytes(LooseChunksStats.ChunkByteCount),
+ LooseChunksStats.ChunkCount,
+ NiceBytes(LooseChunksStats.ChunkByteCount),
- NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ }
DiskStatistics DiskStats;
UploadStatistics UploadStats;
@@ -3918,15 +3953,18 @@ namespace {
Stopwatch GenerateBuildBlocksTimer;
auto __ = MakeGuard([&]() {
uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs();
- ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
- UploadStats.BlockCount.load(),
- NiceBytes(UploadStats.BlocksBytes.load()),
- NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
- NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+ NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
+ NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
+ GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ }
});
GenerateBuildBlocks(Path,
LocalContent,
@@ -3945,9 +3983,12 @@ namespace {
CbObjectWriter PartManifestWriter;
Stopwatch ManifestGenerationTimer;
auto __ = MakeGuard([&]() {
- ZEN_CONSOLE("Generated build part manifest in {} ({})",
- NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifestWriter.GetSaveSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Generated build part manifest in {} ({})",
+ NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifestWriter.GetSaveSize()));
+ }
});
PartManifestWriter.AddObject("chunker"sv, ChunkerParameters);
@@ -4096,10 +4137,13 @@ namespace {
Stopwatch PutBuildPartResultTimer;
std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest);
- ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.",
- NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifest.GetSize()),
- PutBuildPartResult.second.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.",
+ NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifest.GetSize()),
+ PutBuildPartResult.second.size());
+ }
IoHash PartHash = PutBuildPartResult.first;
auto UploadAttachments = [&Storage,
@@ -4122,24 +4166,27 @@ namespace {
Stopwatch TempUploadTimer;
auto __ = MakeGuard([&]() {
- uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
- ZEN_CONSOLE(
- "Uploaded {} ({}) blocks. "
- "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
- "Transferred {} ({}bits/s) in {}",
- TempUploadStats.BlockCount.load(),
- NiceBytes(TempUploadStats.BlocksBytes),
-
- TempLooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(
- GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)),
- TempUploadStats.ChunkCount.load(),
- NiceBytes(TempUploadStats.ChunksBytes),
-
- NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
- NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ if (!IsQuiet)
+ {
+ uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
+ ZEN_CONSOLE(
+ "Uploaded {} ({}) blocks. "
+ "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
+ "Transferred {} ({}bits/s) in {}",
+ TempUploadStats.BlockCount.load(),
+ NiceBytes(TempUploadStats.BlocksBytes),
+
+ TempLooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
+ TempLooseChunksStats.ChunkByteCount)),
+ TempUploadStats.ChunkCount.load(),
+ NiceBytes(TempUploadStats.ChunksBytes),
+
+ NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
+ NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ }
});
UploadPartBlobs(Storage,
BuildId,
@@ -4193,9 +4240,12 @@ namespace {
{
Stopwatch FinalizeBuildPartTimer;
std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash);
- ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.",
- NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
- Needs.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.",
+ NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
+ Needs.size());
+ }
if (Needs.empty())
{
break;
@@ -4208,7 +4258,10 @@ namespace {
{
Stopwatch FinalizeBuildTimer;
Storage.BuildStorage->FinalizeBuild(BuildId);
- ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
+ }
}
if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag)
@@ -4256,7 +4309,10 @@ namespace {
{
uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs();
UploadStats.ElapsedWallTimeUS += ElapsedUS;
- ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000));
+ }
}
}
@@ -4452,55 +4508,59 @@ namespace {
NiceTimeSpanMs(ValidateStats.ElapsedWallTimeUS / 1000));
}
- ZEN_CONSOLE(
- "Uploaded part {} ('{}') to build {}, {}\n"
- " Scanned files: {:>8} ({}), {}B/sec, {}\n"
- " New data: {:>8} ({}) {:.1f}%\n"
- " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
- " Blocks: {:>8} ({})\n"
- " Chunks: {:>8} ({}){}"
- "{}",
- BuildPartId,
- BuildPartName,
- BuildId,
- NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()),
-
- LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)),
- NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000),
-
- FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount,
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
- DeltaByteCountPercent,
-
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
- NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
-
- LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(LooseChunksStats.ChunkByteCount),
- NiceBytes(LooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)),
- NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
-
- UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(),
- NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
- NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000),
-
- UploadStats.BlockCount.load(),
- NiceBytes(UploadStats.BlocksBytes.load()),
-
- UploadStats.ChunkCount.load(),
- NiceBytes(UploadStats.ChunksBytes.load()),
- MultipartAttachmentStats,
-
- ValidateInfo);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Uploaded part {} ('{}') to build {}, {}\n"
+ " Scanned files: {:>8} ({}), {}B/sec, {}\n"
+ " New data: {:>8} ({}) {:.1f}%\n"
+ " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
+ " Blocks: {:>8} ({})\n"
+ " Chunks: {:>8} ({}){}"
+ "{}",
+ BuildPartId,
+ BuildPartName,
+ BuildId,
+ NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()),
+
+ LocalFolderScanStats.FoundFileCount.load(),
+ NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)),
+ NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000),
+
+ FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
+ DeltaByteCountPercent,
+
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceNum(
+ GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
+
+ LooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(LooseChunksStats.ChunkByteCount),
+ NiceBytes(LooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)),
+ NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
+
+ UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(),
+ NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
+ NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000),
+
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+
+ UploadStats.ChunkCount.load(),
+ NiceBytes(UploadStats.ChunksBytes.load()),
+ MultipartAttachmentStats,
+
+ ValidateInfo);
+ }
Storage.BuildStorage->PutBuildPartStats(
BuildId,
@@ -5728,7 +5788,10 @@ namespace {
const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
- ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ }
const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath);
@@ -6340,37 +6403,39 @@ namespace {
}
CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
}
-
- if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
- CachedSequenceHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
- CachedChunkHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheChunkByteCount),
- CachedBlocksFound.size(),
- NiceBytes(CacheMappingStats.CacheBlocksByteCount),
- NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
- }
+ if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
+ {
+ ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
+ CachedSequenceHashesFound.size(),
+ NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
+ CachedChunkHashesFound.size(),
+ NiceBytes(CacheMappingStats.CacheChunkByteCount),
+ CachedBlocksFound.size(),
+ NiceBytes(CacheMappingStats.CacheBlocksByteCount),
+ NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
+ }
- if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
- {
- ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
- LocalPathIndexesMatchingSequenceIndexes.size(),
- NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
- CacheMappingStats.LocalChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
- }
- if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
- {
- ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
- ScavengedPathsCount,
- CacheMappingStats.ScavengedPathsMatchingSequencesCount,
- NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
- CacheMappingStats.ScavengedChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
+ {
+ ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ LocalPathIndexesMatchingSequenceIndexes.size(),
+ NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
+ CacheMappingStats.LocalChunkMatchingRemoteCount,
+ NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
+ }
+ if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
+ {
+ ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ ScavengedPathsCount,
+ CacheMappingStats.ScavengedPathsMatchingSequencesCount,
+ NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
+ CacheMappingStats.ScavengedChunkMatchingRemoteCount,
+ NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ }
}
uint64_t BytesToWrite = 0;
@@ -6758,7 +6823,7 @@ namespace {
}
}
ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs();
- if (!ExistsResult.ExistingBlobs.empty())
+ if (!ExistsResult.ExistingBlobs.empty() && !IsQuiet)
{
ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}",
ExistsResult.ExistingBlobs.size(),
@@ -7920,10 +7985,13 @@ namespace {
const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex];
ZEN_ASSERT(!IncompletePath.empty());
const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
- ZEN_CONSOLE("{}: Max count {}, Current count {}",
- IncompletePath,
- ExpectedSequenceCount,
- SequenceIndexChunksLeftToWriteCounter.load());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}: Max count {}, Current count {}",
+ IncompletePath,
+ ExpectedSequenceCount,
+ SequenceIndexChunksLeftToWriteCounter.load());
+ }
ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount);
}
}
@@ -7932,14 +8000,17 @@ namespace {
const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() +
+DownloadStats.DownloadedPartialBlockByteCount.load();
- ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
- NiceBytes(DownloadedBytes),
- NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
- NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
- NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
+ NiceBytes(DownloadedBytes),
+ NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
+ NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceBytes(DiskStats.WriteByteCount.load()),
+ NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
+ NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
+ }
WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
@@ -8521,12 +8592,15 @@ namespace {
{
Stopwatch GetBuildTimer;
CbObject BuildObject = Storage.GetBuild(BuildId);
- ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
- NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
- BuildObject["name"sv].AsString(),
- NiceBytes(BuildObject.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
+ NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
+ BuildObject["name"sv].AsString(),
+ NiceBytes(BuildObject.GetSize()));
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
+ ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
+ }
CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
if (!PartsObject)
@@ -8613,12 +8687,15 @@ namespace {
const Oid BuildPartId = BuildParts[0].first;
const std::string_view BuildPartName = BuildParts[0].second;
CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(BuildPartManifest.GetSize()));
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ }
{
CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView();
@@ -8745,10 +8822,13 @@ namespace {
}
}
- ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
- BuildPartId,
- NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
- OutBlockDescriptions.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
+ BuildPartId,
+ NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
+ OutBlockDescriptions.size());
+ }
}
if (OutBlockDescriptions.size() != BlockRawHashes.size())
@@ -8853,11 +8933,14 @@ namespace {
const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
Stopwatch GetOverlayBuildPartTimer;
CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- OverlayBuildPartId,
- OverlayBuildPartName,
- NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(OverlayBuildPartManifest.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ OverlayBuildPartId,
+ OverlayBuildPartName,
+ NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(OverlayBuildPartManifest.GetSize()));
+ }
ChunkedFolderContent OverlayPartContent;
std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions;
@@ -8928,7 +9011,7 @@ namespace {
Stopwatch ReadStateTimer;
const bool HasLocalState = IsFile(StateFilePath) && ReadStateFile(StateFilePath, LocalFolderState, LocalContent);
- if (HasLocalState)
+ if (HasLocalState && !IsQuiet)
{
ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
}
@@ -8992,10 +9075,13 @@ namespace {
LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
}
- ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
- DeletedPaths.size(),
- UpdatedContent.Paths.size(),
- LocaStatePathCount);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
+ DeletedPaths.size(),
+ UpdatedContent.Paths.size(),
+ LocaStatePathCount);
+ }
if (UpdatedContent.Paths.size() > 0)
{
uint64_t ByteCountToScan = 0;
@@ -9153,14 +9239,17 @@ namespace {
const uint64_t ChunkedRawSize =
std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- Result.Paths.size(),
- NiceBytes(TotalRawSize),
- Result.ChunkedContent.ChunkHashes.size(),
- NiceBytes(ChunkedRawSize),
- Path,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ Result.Paths.size(),
+ NiceBytes(TotalRawSize),
+ Result.ChunkedContent.ChunkHashes.size(),
+ NiceBytes(ChunkedRawSize),
+ Path,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
return Result;
};
@@ -9236,7 +9325,7 @@ namespace {
{
if (!WipeTargetFolder)
{
- if (!ChunkController)
+ if (!ChunkController && !IsQuiet)
{
ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default");
ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
@@ -9305,14 +9394,20 @@ namespace {
if (CompareContent(RemoteContent, LocalContent))
{
- ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
- NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
+ NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+ }
Stopwatch WriteStateTimer;
CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path);
CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
}
@@ -9326,7 +9421,10 @@ namespace {
uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));
- ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
+ }
FolderContent LocalFolderState;
DiskStatistics DiskStats;
@@ -9373,7 +9471,10 @@ namespace {
CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
@@ -9390,30 +9491,33 @@ namespace {
DownloadStats.DownloadedPartialBlockByteCount.load();
const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();
- ZEN_CONSOLE(
- "Downloaded build {}, parts:{} in {}\n"
- " Download: {} ({}) {}bits/s\n"
- " Write: {} ({}) {}B/s\n"
- " Clean: {}\n"
- " Finalize: {}\n"
- " Verify: {}",
- BuildId,
- BuildPartString.ToView(),
- NiceTimeSpanMs(DownloadTimeMs),
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Downloaded build {}, parts:{} in {}\n"
+ " Download: {} ({}) {}bits/s\n"
+ " Write: {} ({}) {}B/s\n"
+ " Clean: {}\n"
+ " Finalize: {}\n"
+ " Verify: {}",
+ BuildId,
+ BuildPartString.ToView(),
+ NiceTimeSpanMs(DownloadTimeMs),
- DownloadCount,
- NiceBytes(DownloadByteCount),
- NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
+ DownloadCount,
+ NiceBytes(DownloadByteCount),
+ NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
- DiskStats.WriteCount.load(),
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())),
+ DiskStats.WriteCount.load(),
+ NiceBytes(DiskStats.WriteByteCount.load()),
+ NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())),
- NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+ NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+ }
}
}
if (PrimeCacheOnly)
@@ -9421,13 +9525,16 @@ namespace {
if (Storage.BuildCacheStorage)
{
Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) {
- if (Remaining == 0)
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Build cache upload complete");
- }
- else
- {
- ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
+ if (Remaining == 0)
+ {
+ ZEN_CONSOLE("Build cache upload complete");
+ }
+ else
+ {
+ ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
+ }
}
return !AbortFlag;
});
@@ -9442,6 +9549,87 @@ namespace {
}
}
+ void ListBuild(StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ std::string_view IncludeWildcard,
+ std::string_view ExcludeWildcard)
+ {
+ std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+
+ Stopwatch GetBuildPartTimer;
+
+ for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
+ {
+ const Oid BuildPartId = AllBuildParts[BuildPartIndex].first;
+ const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second;
+ CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
+
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}Part: {} ('{}'):\n",
+ BuildPartIndex > 0 ? "\n" : "",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ }
+
+ std::vector<std::filesystem::path> Paths;
+ std::vector<IoHash> RawHashes;
+ std::vector<uint64_t> RawSizes;
+ std::vector<uint32_t> Attributes;
+
+ SourcePlatform Platform;
+ std::vector<IoHash> SequenceRawHashes;
+ std::vector<uint32_t> ChunkCounts;
+ std::vector<uint32_t> AbsoluteChunkOrders;
+ std::vector<IoHash> LooseChunkHashes;
+ std::vector<uint64_t> LooseChunkRawSizes;
+ std::vector<IoHash> BlockRawHashes;
+
+ ReadBuildContentFromCompactBinary(BuildPartManifest,
+ Platform,
+ Paths,
+ RawHashes,
+ RawSizes,
+ Attributes,
+ SequenceRawHashes,
+ ChunkCounts,
+ AbsoluteChunkOrders,
+ LooseChunkHashes,
+ LooseChunkRawSizes,
+ BlockRawHashes);
+
+ std::vector<size_t> Order(Paths.size());
+ std::iota(Order.begin(), Order.end(), 0);
+
+ std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
+ const std::filesystem::path& LhsPath = Paths[Lhs];
+ const std::filesystem::path& RhsPath = Paths[Rhs];
+ return LhsPath < RhsPath;
+ });
+
+ for (size_t Index : Order)
+ {
+ const std::filesystem::path& Path = Paths[Index];
+ if (IncludePath(IncludeWildcard, ExcludeWildcard, Path))
+ {
+ const IoHash& RawHash = RawHashes[Index];
+ const uint64_t RawSize = RawSizes[Index];
+ const uint32_t Attribute = Attributes[Index];
+ ZEN_UNUSED(Attribute);
+
+ ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
+ }
+ }
+ }
+ }
+
void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked)
{
ZEN_TRACE_CPU("DiffFolders");
@@ -9774,6 +9962,7 @@ BuildsCommand::BuildsCommand()
cxxopts::value(m_LogProgress),
"<logprogress>");
Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
+ Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
};
auto AddWorkerOptions = [this](cxxopts::Options& Ops) {
@@ -9793,6 +9982,23 @@ BuildsCommand::BuildsCommand()
cxxopts::value(m_ZenFolderPath),
"<boostworkers>");
};
+
+ auto AddWildcardOptions = [this](cxxopts::Options& Ops) {
+ Ops.add_option("",
+ "",
+ "wildcard",
+ "Windows style wildcard (using * and ?) to match file paths to include",
+ cxxopts::value(m_IncludeWildcard),
+ "<wildcard>");
+
+ Ops.add_option("",
+ "",
+ "exclude-wildcard",
+ "Windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter",
+ cxxopts::value(m_ExcludeWildcard),
+ "<excludewildcard>");
+ };
+
m_Options.add_option("",
"v",
"verb",
@@ -9927,6 +10133,8 @@ BuildsCommand::BuildsCommand()
AddCacheOptions(m_DownloadOptions);
AddZenFolderOptions(m_DownloadOptions);
AddWorkerOptions(m_DownloadOptions);
+ // TODO: AddWildcardOptions(m_DownloadOptions);
+
m_DownloadOptions.add_option("cache",
"",
"cache-prime-only",
@@ -9977,6 +10185,37 @@ BuildsCommand::BuildsCommand()
m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"});
m_DownloadOptions.positional_help("local-path build-id build-part-name");
+ // ls
+ AddSystemOptions(m_LsOptions);
+ AddCloudOptions(m_LsOptions);
+ AddFileOptions(m_LsOptions);
+ AddOutputOptions(m_LsOptions);
+ AddCacheOptions(m_LsOptions);
+ AddZenFolderOptions(m_LsOptions);
+ AddWorkerOptions(m_LsOptions);
+ AddWildcardOptions(m_LsOptions);
+
+ m_LsOptions.add_options()("h,help", "Print help");
+ m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ m_LsOptions.add_option(
+ "",
+ "",
+ "build-part-id",
+ "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
+ cxxopts::value(m_BuildPartIds),
+ "<id>");
+ m_LsOptions.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
+ "all parts will be downloaded",
+ cxxopts::value(m_BuildPartNames),
+ "<name>");
+
+ m_LsOptions.parse_positional({"build-id", "wildcard"});
+ m_LsOptions.positional_help("build-id wildcard");
+
+ // diff
AddOutputOptions(m_DiffOptions);
AddWorkerOptions(m_DiffOptions);
m_DiffOptions.add_options()("h,help", "Print help");
@@ -9991,6 +10230,7 @@ BuildsCommand::BuildsCommand()
m_DiffOptions.parse_positional({"local-path", "compare-path"});
m_DiffOptions.positional_help("local-path compare-path");
+ // test
AddSystemOptions(m_TestOptions);
AddCloudOptions(m_TestOptions);
AddFileOptions(m_TestOptions);
@@ -10020,6 +10260,7 @@ BuildsCommand::BuildsCommand()
m_TestOptions.parse_positional({"local-path"});
m_TestOptions.positional_help("local-path");
+ // fetch-blob
AddSystemOptions(m_FetchBlobOptions);
AddCloudOptions(m_FetchBlobOptions);
AddFileOptions(m_FetchBlobOptions);
@@ -10047,6 +10288,7 @@ BuildsCommand::BuildsCommand()
m_AbortOptions.parse_positional({"process-id"});
m_AbortOptions.positional_help("process-id");
+ // validate-part
AddSystemOptions(m_ValidateBuildPartOptions);
AddCloudOptions(m_ValidateBuildPartOptions);
AddFileOptions(m_ValidateBuildPartOptions);
@@ -10070,6 +10312,7 @@ BuildsCommand::BuildsCommand()
m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"});
m_ValidateBuildPartOptions.positional_help("build-id build-part-id");
+ // multi-test-download
AddSystemOptions(m_MultiTestDownloadOptions);
AddCloudOptions(m_MultiTestDownloadOptions);
AddFileOptions(m_MultiTestDownloadOptions);
@@ -10193,13 +10436,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_EncryptionKey.empty())
{
m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456";
- ZEN_CONSOLE("Warning: Using default encryption key");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Warning: Using default encryption key");
+ }
}
if (m_EncryptionIV.empty())
{
m_EncryptionIV = "0123456789abcdef";
- ZEN_CONSOLE("Warning: Using default encryption initialization vector");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Warning: Using default encryption initialization vector");
+ }
}
AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth",
@@ -10301,7 +10550,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty())
{
const std::string& CloudHost = m_OverrideHost.empty() ? m_Host : m_OverrideHost;
- ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost);
+ ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost, IsQuiet);
}
if (!ClientSettings.AccessTokenProvider)
@@ -10312,15 +10561,36 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
};
auto ParseOutputOptions = [&]() {
+ if (m_Verbose && m_Quiet)
+ {
+ throw std::runtime_error("--verbose option is not compatible with --quiet option");
+ }
IsVerbose = m_Verbose;
+ IsQuiet = m_Quiet;
if (m_LogProgress)
{
+ if (IsQuiet)
+ {
+ throw std::runtime_error("--quiet option is not compatible with --log-progress option");
+ }
ProgressMode = ProgressBar::Mode::Log;
}
- else if (m_Verbose || m_PlainProgress)
+ if (m_PlainProgress)
{
+ if (IsQuiet)
+ {
+ throw std::runtime_error("--quiet option is not compatible with --plain-progress option");
+ }
ProgressMode = ProgressBar::Mode::Plain;
}
+ else if (m_Verbose)
+ {
+ ProgressMode = ProgressBar::Mode::Plain;
+ }
+ else if (IsQuiet)
+ {
+ ProgressMode = ProgressBar::Mode::Quiet;
+ }
else
{
ProgressMode = ProgressBar::Mode::Pretty;
@@ -10489,7 +10759,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
});
}
- if (m_ZenCacheHost.empty())
+ if (m_ZenCacheHost.empty() && !IsQuiet)
{
ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
}
@@ -10578,10 +10848,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName;
}
- ZEN_CONSOLE("Remote: {}", StorageDescription);
- if (!Result.CacheName.empty())
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Cache : {}", CacheDescription);
+ ZEN_CONSOLE("Remote: {}", StorageDescription);
+ if (!Result.CacheName.empty())
+ {
+ ZEN_CONSOLE("Cache : {}", CacheDescription);
+ }
}
return Result;
};
@@ -10594,6 +10867,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
MakeSafeAbsolutePathÍnPlace(m_Path);
};
+ auto ParseFileFilters = [&]() {
+ for (auto It = begin(m_IncludeWildcard); It != end(m_IncludeWildcard); It++)
+ {
+ if (*It == '\\')
+ {
+ *It = '/';
+ }
+ }
+
+ for (auto It = begin(m_ExcludeWildcard); It != end(m_ExcludeWildcard); It++)
+ {
+ if (*It == '\\')
+ {
+ *It = '/';
+ }
+ }
+ };
+
auto ParseDiffPath = [&]() {
if (m_DiffPath.empty())
{
@@ -10742,10 +11033,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!m_ListResultPath.empty())
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
}
BuildStorage::Statistics StorageStats;
@@ -10803,10 +11097,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!m_ListResultPath.empty())
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
}
CbObject QueryObject;
if (m_ListQueryPath.empty())
@@ -10888,7 +11185,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_UploadOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
@@ -10957,34 +11260,44 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (false)
{
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.StorageName,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "{}:\n"
+ "Read: {}\n"
+ "Write: {}\n"
+ "Requests: {}\n"
+ "Avg Request Time: {}\n"
+ "Avg I/O Time: {}",
+ Storage.StorageName,
+ NiceBytes(StorageStats.TotalBytesRead.load()),
+ NiceBytes(StorageStats.TotalBytesWritten.load()),
+ StorageStats.TotalRequestCount.load(),
+ StorageStats.TotalExecutionTimeUs.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0,
+ StorageStats.TotalRequestCount.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0);
+ }
}
return AbortFlag ? 11 : 0;
}
if (SubOption == &m_DownloadOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
ParsePath();
+ ParseFileFilters();
if (m_ZenFolderPath.empty())
{
@@ -11022,6 +11335,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::vector<Oid> BuildPartIds = ParseBuildPartIds();
std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+ // TODO: Add file filters
DownloadFolder(Storage,
BuildId,
BuildPartIds,
@@ -11038,6 +11352,46 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return AbortFlag ? 11 : 0;
}
+
+ if (SubOption == &m_LsOptions)
+ {
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
+
+ ZenState InstanceState;
+
+ ParseFileFilters();
+
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
+
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
+
+ const Oid BuildId = ParseBuildId();
+
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds();
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+
+ ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard);
+
+ return AbortFlag ? 11 : 0;
+ }
+
if (SubOption == &m_DiffOptions)
{
ParsePath();
@@ -11049,7 +11403,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_FetchBlobOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
@@ -11086,16 +11446,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
return 11;
}
- ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
- BlobHash,
- CompressedSize,
- DecompressedSize);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
+ BlobHash,
+ CompressedSize,
+ DecompressedSize);
+ }
return 0;
}
if (SubOption == &m_ValidateBuildPartOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
@@ -11189,9 +11558,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("Download cancelled");
return 11;
}
- ZEN_CONSOLE("\n");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("\n");
+ }
+ }
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
- ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
return 0;
}