aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-08-11 12:58:27 +0200
committerGitHub Enterprise <[email protected]>2025-08-11 12:58:27 +0200
commiteeac0654ab8f9e3438f2331916261a0440286fbd (patch)
tree56bd48ef8dedf91eeb25a7c26471742ab189874c /src
parentMerge pull request #434 from ue-foundation/zs/put-overwrite-policy (diff)
downloadzen-eeac0654ab8f9e3438f2331916261a0440286fbd.tar.xz
zen-eeac0654ab8f9e3438f2331916261a0440286fbd.zip
list build part content (#462)
- Feature: Added `zen build ls` option to list the content of a build part(s) - Build source is specified using one of the following options - `--cloud-url` cloud artifact URL to build - `--host` or `--override-host`, `--namespace`, `--bucket` and `--buildid` - `--filestorage`, `--namespace`, `--bucket` and `--buildid` - `--build-part-name` to specify a particular build part(s) in the build - `--wildcard` windows style wildcard (using * and ?) to match file paths to include - `--exclude-wildcard` windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter - Improvement: Added `--quiet` option to zen `builds` commands to suppress non-essential output
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/builds_cmd.cpp957
-rw-r--r--src/zen/cmds/builds_cmd.h8
-rw-r--r--src/zen/zen.h3
-rw-r--r--src/zenhttp/httpclientauth.cpp23
-rw-r--r--src/zenhttp/include/zenhttp/httpclientauth.h3
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp4
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.h3
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.cpp4
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.h3
-rw-r--r--src/zenserver/projectstore/projectstore.cpp4
-rw-r--r--src/zenutil/include/zenutil/wildcard.h13
-rw-r--r--src/zenutil/wildcard.cpp112
-rw-r--r--src/zenutil/zenutil.cpp2
13 files changed, 832 insertions, 307 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 7cb12cd4c..8e17ba97c 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -33,6 +33,7 @@
#include <zenutil/jupiter/jupiterbuildstorage.h>
#include <zenutil/jupiter/jupitersession.h>
#include <zenutil/parallelwork.h>
+#include <zenutil/wildcard.h>
#include <zenutil/workerpools.h>
#include <zenutil/zenserverprocess.h>
@@ -65,6 +66,7 @@ using namespace std::literals;
namespace {
namespace zenutil {
+
#if ZEN_PLATFORM_WINDOWS
class SecurityAttributes
{
@@ -368,6 +370,7 @@ namespace {
const std::vector<std::string_view> DefaultExcludeExtensions({});
static bool IsVerbose = false;
+ static bool IsQuiet = false;
static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode)
@@ -402,6 +405,20 @@ namespace {
);
+ bool IncludePath(const std::string_view IncludeWildcard, const std::string_view ExcludeWildcard, const std::filesystem::path& Path)
+ {
+ const std::string PathString = Path.generic_string();
+ if (!IncludeWildcard.empty() && !MatchWildcard(IncludeWildcard, PathString, /*CaseSensitive*/ false))
+ {
+ return false;
+ }
+ if (!ExcludeWildcard.empty() && MatchWildcard(ExcludeWildcard, PathString, /*CaseSensitive*/ false))
+ {
+ return false;
+ }
+ return true;
+ }
+
bool IsFileWithRetry(const std::filesystem::path& Path)
{
std::error_code Ec;
@@ -795,7 +812,7 @@ namespace {
}
uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
- if (ElapsedTimeMs >= 200)
+ if (ElapsedTimeMs >= 200 && !IsQuiet)
{
ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}",
Path,
@@ -2134,11 +2151,14 @@ namespace {
Stopwatch Timer;
auto _ = MakeGuard([&]() {
- ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
- BuildId,
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
+ BuildId,
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
});
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount);
@@ -2163,7 +2183,10 @@ namespace {
CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId);
ValidateStats.BuildPartSize = BuildPart.GetSize();
- ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize()));
+ }
std::vector<IoHash> ChunkAttachments;
for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv])
{
@@ -3770,14 +3793,17 @@ namespace {
}
}
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- LocalContent.Paths.size(),
- NiceBytes(TotalRawSize),
- ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(ChunkingStats.UniqueBytesFound.load()),
- Path,
- NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ LocalContent.Paths.size(),
+ NiceBytes(TotalRawSize),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()),
+ Path,
+ NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
}
const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
@@ -3790,15 +3816,18 @@ namespace {
PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get();
- ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}",
- NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
- CreateBuild ? "PutBuild" : "GetBuild",
- NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
- NiceBytes(PrepBuildResult.PayloadSize),
- IgnoreExistingBlocks ? ""
- : fmt::format(". Found {} blocks in {}",
- PrepBuildResult.KnownBlocks.size(),
- NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}",
+ NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
+ CreateBuild ? "PutBuild" : "GetBuild",
+ NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
+ NiceBytes(PrepBuildResult.PayloadSize),
+ IgnoreExistingBlocks ? ""
+ : fmt::format(". Found {} blocks in {}",
+ PrepBuildResult.KnownBlocks.size(),
+ NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ }
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount);
@@ -3829,7 +3858,10 @@ namespace {
if (IgnoreExistingBlocks)
{
- ZEN_CONSOLE("Ignoring any existing blocks in store");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Ignoring any existing blocks in store");
+ }
NewBlockChunkIndexes = std::move(BlockChunkIndexes);
}
else
@@ -3872,40 +3904,43 @@ namespace {
FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) /
(FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount)
: 0.0;
- ZEN_CONSOLE(
- "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
- " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
- " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
- " Rejected {} ({}) chunks in {} blocks\n"
- " Arranged {} ({}) chunks in {} new blocks\n"
- " Keeping {} ({}) chunks as loose chunks\n"
- " Discovery completed in {}",
- FindBlocksStats.FoundBlockChunkCount,
- FindBlocksStats.FoundBlockCount,
- NiceBytes(FindBlocksStats.FoundBlockByteCount),
- NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
+ " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
+ " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
+ " Rejected {} ({}) chunks in {} blocks\n"
+ " Arranged {} ({}) chunks in {} new blocks\n"
+ " Keeping {} ({}) chunks as loose chunks\n"
+ " Discovery completed in {}",
+ FindBlocksStats.FoundBlockChunkCount,
+ FindBlocksStats.FoundBlockCount,
+ NiceBytes(FindBlocksStats.FoundBlockByteCount),
+ NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
- FindBlocksStats.AcceptedChunkCount,
- NiceBytes(FindBlocksStats.AcceptedRawByteCount),
- FindBlocksStats.AcceptedBlockCount,
- AcceptedByteCountPercent,
+ FindBlocksStats.AcceptedChunkCount,
+ NiceBytes(FindBlocksStats.AcceptedRawByteCount),
+ FindBlocksStats.AcceptedBlockCount,
+ AcceptedByteCountPercent,
- FindBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(FindBlocksStats.AcceptedReduntantByteCount),
- AcceptedReduntantByteCountPercent,
+ FindBlocksStats.AcceptedReduntantChunkCount,
+ NiceBytes(FindBlocksStats.AcceptedReduntantByteCount),
+ AcceptedReduntantByteCountPercent,
- FindBlocksStats.RejectedChunkCount,
- NiceBytes(FindBlocksStats.RejectedByteCount),
- FindBlocksStats.RejectedBlockCount,
+ FindBlocksStats.RejectedChunkCount,
+ NiceBytes(FindBlocksStats.RejectedByteCount),
+ FindBlocksStats.RejectedBlockCount,
- FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
- FindBlocksStats.NewBlocksCount,
+ FindBlocksStats.NewBlocksChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ FindBlocksStats.NewBlocksCount,
- LooseChunksStats.ChunkCount,
- NiceBytes(LooseChunksStats.ChunkByteCount),
+ LooseChunksStats.ChunkCount,
+ NiceBytes(LooseChunksStats.ChunkByteCount),
- NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ }
DiskStatistics DiskStats;
UploadStatistics UploadStats;
@@ -3918,15 +3953,18 @@ namespace {
Stopwatch GenerateBuildBlocksTimer;
auto __ = MakeGuard([&]() {
uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs();
- ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
- UploadStats.BlockCount.load(),
- NiceBytes(UploadStats.BlocksBytes.load()),
- NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
- NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+ NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
+ NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
+ GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ }
});
GenerateBuildBlocks(Path,
LocalContent,
@@ -3945,9 +3983,12 @@ namespace {
CbObjectWriter PartManifestWriter;
Stopwatch ManifestGenerationTimer;
auto __ = MakeGuard([&]() {
- ZEN_CONSOLE("Generated build part manifest in {} ({})",
- NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifestWriter.GetSaveSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Generated build part manifest in {} ({})",
+ NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifestWriter.GetSaveSize()));
+ }
});
PartManifestWriter.AddObject("chunker"sv, ChunkerParameters);
@@ -4096,10 +4137,13 @@ namespace {
Stopwatch PutBuildPartResultTimer;
std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest);
- ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.",
- NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifest.GetSize()),
- PutBuildPartResult.second.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.",
+ NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifest.GetSize()),
+ PutBuildPartResult.second.size());
+ }
IoHash PartHash = PutBuildPartResult.first;
auto UploadAttachments = [&Storage,
@@ -4122,24 +4166,27 @@ namespace {
Stopwatch TempUploadTimer;
auto __ = MakeGuard([&]() {
- uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
- ZEN_CONSOLE(
- "Uploaded {} ({}) blocks. "
- "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
- "Transferred {} ({}bits/s) in {}",
- TempUploadStats.BlockCount.load(),
- NiceBytes(TempUploadStats.BlocksBytes),
-
- TempLooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(
- GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)),
- TempUploadStats.ChunkCount.load(),
- NiceBytes(TempUploadStats.ChunksBytes),
-
- NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
- NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ if (!IsQuiet)
+ {
+ uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
+ ZEN_CONSOLE(
+ "Uploaded {} ({}) blocks. "
+ "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
+ "Transferred {} ({}bits/s) in {}",
+ TempUploadStats.BlockCount.load(),
+ NiceBytes(TempUploadStats.BlocksBytes),
+
+ TempLooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
+ TempLooseChunksStats.ChunkByteCount)),
+ TempUploadStats.ChunkCount.load(),
+ NiceBytes(TempUploadStats.ChunksBytes),
+
+ NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
+ NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ }
});
UploadPartBlobs(Storage,
BuildId,
@@ -4193,9 +4240,12 @@ namespace {
{
Stopwatch FinalizeBuildPartTimer;
std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash);
- ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.",
- NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
- Needs.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.",
+ NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
+ Needs.size());
+ }
if (Needs.empty())
{
break;
@@ -4208,7 +4258,10 @@ namespace {
{
Stopwatch FinalizeBuildTimer;
Storage.BuildStorage->FinalizeBuild(BuildId);
- ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
+ }
}
if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag)
@@ -4256,7 +4309,10 @@ namespace {
{
uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs();
UploadStats.ElapsedWallTimeUS += ElapsedUS;
- ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000));
+ }
}
}
@@ -4452,55 +4508,59 @@ namespace {
NiceTimeSpanMs(ValidateStats.ElapsedWallTimeUS / 1000));
}
- ZEN_CONSOLE(
- "Uploaded part {} ('{}') to build {}, {}\n"
- " Scanned files: {:>8} ({}), {}B/sec, {}\n"
- " New data: {:>8} ({}) {:.1f}%\n"
- " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
- " Blocks: {:>8} ({})\n"
- " Chunks: {:>8} ({}){}"
- "{}",
- BuildPartId,
- BuildPartName,
- BuildId,
- NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()),
-
- LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)),
- NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000),
-
- FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount,
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
- DeltaByteCountPercent,
-
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
- NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
-
- LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(LooseChunksStats.ChunkByteCount),
- NiceBytes(LooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)),
- NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
-
- UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(),
- NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
- NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000),
-
- UploadStats.BlockCount.load(),
- NiceBytes(UploadStats.BlocksBytes.load()),
-
- UploadStats.ChunkCount.load(),
- NiceBytes(UploadStats.ChunksBytes.load()),
- MultipartAttachmentStats,
-
- ValidateInfo);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Uploaded part {} ('{}') to build {}, {}\n"
+ " Scanned files: {:>8} ({}), {}B/sec, {}\n"
+ " New data: {:>8} ({}) {:.1f}%\n"
+ " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
+ " Blocks: {:>8} ({})\n"
+ " Chunks: {:>8} ({}){}"
+ "{}",
+ BuildPartId,
+ BuildPartName,
+ BuildId,
+ NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()),
+
+ LocalFolderScanStats.FoundFileCount.load(),
+ NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)),
+ NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000),
+
+ FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
+ DeltaByteCountPercent,
+
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceNum(
+ GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
+
+ LooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(LooseChunksStats.ChunkByteCount),
+ NiceBytes(LooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)),
+ NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
+
+ UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(),
+ NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
+ NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000),
+
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+
+ UploadStats.ChunkCount.load(),
+ NiceBytes(UploadStats.ChunksBytes.load()),
+ MultipartAttachmentStats,
+
+ ValidateInfo);
+ }
Storage.BuildStorage->PutBuildPartStats(
BuildId,
@@ -5728,7 +5788,10 @@ namespace {
const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
- ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ }
const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath);
@@ -6340,37 +6403,39 @@ namespace {
}
CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
}
-
- if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
- CachedSequenceHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
- CachedChunkHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheChunkByteCount),
- CachedBlocksFound.size(),
- NiceBytes(CacheMappingStats.CacheBlocksByteCount),
- NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
- }
+ if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
+ {
+ ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
+ CachedSequenceHashesFound.size(),
+ NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
+ CachedChunkHashesFound.size(),
+ NiceBytes(CacheMappingStats.CacheChunkByteCount),
+ CachedBlocksFound.size(),
+ NiceBytes(CacheMappingStats.CacheBlocksByteCount),
+ NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
+ }
- if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
- {
- ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
- LocalPathIndexesMatchingSequenceIndexes.size(),
- NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
- CacheMappingStats.LocalChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
- }
- if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
- {
- ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
- ScavengedPathsCount,
- CacheMappingStats.ScavengedPathsMatchingSequencesCount,
- NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
- CacheMappingStats.ScavengedChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
+ {
+ ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ LocalPathIndexesMatchingSequenceIndexes.size(),
+ NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
+ CacheMappingStats.LocalChunkMatchingRemoteCount,
+ NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
+ }
+ if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
+ {
+ ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ ScavengedPathsCount,
+ CacheMappingStats.ScavengedPathsMatchingSequencesCount,
+ NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
+ CacheMappingStats.ScavengedChunkMatchingRemoteCount,
+ NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ }
}
uint64_t BytesToWrite = 0;
@@ -6758,7 +6823,7 @@ namespace {
}
}
ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs();
- if (!ExistsResult.ExistingBlobs.empty())
+ if (!ExistsResult.ExistingBlobs.empty() && !IsQuiet)
{
ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}",
ExistsResult.ExistingBlobs.size(),
@@ -7920,10 +7985,13 @@ namespace {
const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex];
ZEN_ASSERT(!IncompletePath.empty());
const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
- ZEN_CONSOLE("{}: Max count {}, Current count {}",
- IncompletePath,
- ExpectedSequenceCount,
- SequenceIndexChunksLeftToWriteCounter.load());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}: Max count {}, Current count {}",
+ IncompletePath,
+ ExpectedSequenceCount,
+ SequenceIndexChunksLeftToWriteCounter.load());
+ }
ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount);
}
}
@@ -7932,14 +8000,17 @@ namespace {
const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() +
+DownloadStats.DownloadedPartialBlockByteCount.load();
- ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
- NiceBytes(DownloadedBytes),
- NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
- NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
- NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
+ NiceBytes(DownloadedBytes),
+ NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
+ NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceBytes(DiskStats.WriteByteCount.load()),
+ NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
+ NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
+ }
WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
@@ -8521,12 +8592,15 @@ namespace {
{
Stopwatch GetBuildTimer;
CbObject BuildObject = Storage.GetBuild(BuildId);
- ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
- NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
- BuildObject["name"sv].AsString(),
- NiceBytes(BuildObject.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
+ NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
+ BuildObject["name"sv].AsString(),
+ NiceBytes(BuildObject.GetSize()));
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
+ ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
+ }
CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
if (!PartsObject)
@@ -8613,12 +8687,15 @@ namespace {
const Oid BuildPartId = BuildParts[0].first;
const std::string_view BuildPartName = BuildParts[0].second;
CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(BuildPartManifest.GetSize()));
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ }
{
CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView();
@@ -8745,10 +8822,13 @@ namespace {
}
}
- ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
- BuildPartId,
- NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
- OutBlockDescriptions.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
+ BuildPartId,
+ NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
+ OutBlockDescriptions.size());
+ }
}
if (OutBlockDescriptions.size() != BlockRawHashes.size())
@@ -8853,11 +8933,14 @@ namespace {
const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
Stopwatch GetOverlayBuildPartTimer;
CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- OverlayBuildPartId,
- OverlayBuildPartName,
- NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(OverlayBuildPartManifest.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ OverlayBuildPartId,
+ OverlayBuildPartName,
+ NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(OverlayBuildPartManifest.GetSize()));
+ }
ChunkedFolderContent OverlayPartContent;
std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions;
@@ -8928,7 +9011,7 @@ namespace {
Stopwatch ReadStateTimer;
const bool HasLocalState = IsFile(StateFilePath) && ReadStateFile(StateFilePath, LocalFolderState, LocalContent);
- if (HasLocalState)
+ if (HasLocalState && !IsQuiet)
{
ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
}
@@ -8992,10 +9075,13 @@ namespace {
LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
}
- ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
- DeletedPaths.size(),
- UpdatedContent.Paths.size(),
- LocaStatePathCount);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
+ DeletedPaths.size(),
+ UpdatedContent.Paths.size(),
+ LocaStatePathCount);
+ }
if (UpdatedContent.Paths.size() > 0)
{
uint64_t ByteCountToScan = 0;
@@ -9153,14 +9239,17 @@ namespace {
const uint64_t ChunkedRawSize =
std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- Result.Paths.size(),
- NiceBytes(TotalRawSize),
- Result.ChunkedContent.ChunkHashes.size(),
- NiceBytes(ChunkedRawSize),
- Path,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ Result.Paths.size(),
+ NiceBytes(TotalRawSize),
+ Result.ChunkedContent.ChunkHashes.size(),
+ NiceBytes(ChunkedRawSize),
+ Path,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
return Result;
};
@@ -9236,7 +9325,7 @@ namespace {
{
if (!WipeTargetFolder)
{
- if (!ChunkController)
+ if (!ChunkController && !IsQuiet)
{
ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default");
ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
@@ -9305,14 +9394,20 @@ namespace {
if (CompareContent(RemoteContent, LocalContent))
{
- ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
- NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
+ NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+ }
Stopwatch WriteStateTimer;
CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path);
CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
}
@@ -9326,7 +9421,10 @@ namespace {
uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));
- ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
+ }
FolderContent LocalFolderState;
DiskStatistics DiskStats;
@@ -9373,7 +9471,10 @@ namespace {
CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
@@ -9390,30 +9491,33 @@ namespace {
DownloadStats.DownloadedPartialBlockByteCount.load();
const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();
- ZEN_CONSOLE(
- "Downloaded build {}, parts:{} in {}\n"
- " Download: {} ({}) {}bits/s\n"
- " Write: {} ({}) {}B/s\n"
- " Clean: {}\n"
- " Finalize: {}\n"
- " Verify: {}",
- BuildId,
- BuildPartString.ToView(),
- NiceTimeSpanMs(DownloadTimeMs),
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Downloaded build {}, parts:{} in {}\n"
+ " Download: {} ({}) {}bits/s\n"
+ " Write: {} ({}) {}B/s\n"
+ " Clean: {}\n"
+ " Finalize: {}\n"
+ " Verify: {}",
+ BuildId,
+ BuildPartString.ToView(),
+ NiceTimeSpanMs(DownloadTimeMs),
- DownloadCount,
- NiceBytes(DownloadByteCount),
- NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
+ DownloadCount,
+ NiceBytes(DownloadByteCount),
+ NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
- DiskStats.WriteCount.load(),
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())),
+ DiskStats.WriteCount.load(),
+ NiceBytes(DiskStats.WriteByteCount.load()),
+ NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())),
- NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+ NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+ }
}
}
if (PrimeCacheOnly)
@@ -9421,13 +9525,16 @@ namespace {
if (Storage.BuildCacheStorage)
{
Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) {
- if (Remaining == 0)
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Build cache upload complete");
- }
- else
- {
- ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
+ if (Remaining == 0)
+ {
+ ZEN_CONSOLE("Build cache upload complete");
+ }
+ else
+ {
+ ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
+ }
}
return !AbortFlag;
});
@@ -9442,6 +9549,87 @@ namespace {
}
}
+ void ListBuild(StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ std::string_view IncludeWildcard,
+ std::string_view ExcludeWildcard)
+ {
+ std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+
+ Stopwatch GetBuildPartTimer;
+
+ for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
+ {
+ const Oid BuildPartId = AllBuildParts[BuildPartIndex].first;
+ const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second;
+ CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
+
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}Part: {} ('{}'):\n",
+ BuildPartIndex > 0 ? "\n" : "",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ }
+
+ std::vector<std::filesystem::path> Paths;
+ std::vector<IoHash> RawHashes;
+ std::vector<uint64_t> RawSizes;
+ std::vector<uint32_t> Attributes;
+
+ SourcePlatform Platform;
+ std::vector<IoHash> SequenceRawHashes;
+ std::vector<uint32_t> ChunkCounts;
+ std::vector<uint32_t> AbsoluteChunkOrders;
+ std::vector<IoHash> LooseChunkHashes;
+ std::vector<uint64_t> LooseChunkRawSizes;
+ std::vector<IoHash> BlockRawHashes;
+
+ ReadBuildContentFromCompactBinary(BuildPartManifest,
+ Platform,
+ Paths,
+ RawHashes,
+ RawSizes,
+ Attributes,
+ SequenceRawHashes,
+ ChunkCounts,
+ AbsoluteChunkOrders,
+ LooseChunkHashes,
+ LooseChunkRawSizes,
+ BlockRawHashes);
+
+ std::vector<size_t> Order(Paths.size());
+ std::iota(Order.begin(), Order.end(), 0);
+
+ std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
+ const std::filesystem::path& LhsPath = Paths[Lhs];
+ const std::filesystem::path& RhsPath = Paths[Rhs];
+ return LhsPath < RhsPath;
+ });
+
+ for (size_t Index : Order)
+ {
+ const std::filesystem::path& Path = Paths[Index];
+ if (IncludePath(IncludeWildcard, ExcludeWildcard, Path))
+ {
+ const IoHash& RawHash = RawHashes[Index];
+ const uint64_t RawSize = RawSizes[Index];
+ const uint32_t Attribute = Attributes[Index];
+ ZEN_UNUSED(Attribute);
+
+ ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
+ }
+ }
+ }
+ }
+
void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked)
{
ZEN_TRACE_CPU("DiffFolders");
@@ -9774,6 +9962,7 @@ BuildsCommand::BuildsCommand()
cxxopts::value(m_LogProgress),
"<logprogress>");
Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
+ Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
};
auto AddWorkerOptions = [this](cxxopts::Options& Ops) {
@@ -9793,6 +9982,23 @@ BuildsCommand::BuildsCommand()
cxxopts::value(m_ZenFolderPath),
"<boostworkers>");
};
+
+ auto AddWildcardOptions = [this](cxxopts::Options& Ops) {
+ Ops.add_option("",
+ "",
+ "wildcard",
+ "Windows style wildcard (using * and ?) to match file paths to include",
+ cxxopts::value(m_IncludeWildcard),
+ "<wildcard>");
+
+ Ops.add_option("",
+ "",
+ "exclude-wildcard",
+ "Windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter",
+ cxxopts::value(m_ExcludeWildcard),
+ "<excludewildcard>");
+ };
+
m_Options.add_option("",
"v",
"verb",
@@ -9927,6 +10133,8 @@ BuildsCommand::BuildsCommand()
AddCacheOptions(m_DownloadOptions);
AddZenFolderOptions(m_DownloadOptions);
AddWorkerOptions(m_DownloadOptions);
+ // TODO: AddWildcardOptions(m_DownloadOptions);
+
m_DownloadOptions.add_option("cache",
"",
"cache-prime-only",
@@ -9977,6 +10185,37 @@ BuildsCommand::BuildsCommand()
m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"});
m_DownloadOptions.positional_help("local-path build-id build-part-name");
+ // ls
+ AddSystemOptions(m_LsOptions);
+ AddCloudOptions(m_LsOptions);
+ AddFileOptions(m_LsOptions);
+ AddOutputOptions(m_LsOptions);
+ AddCacheOptions(m_LsOptions);
+ AddZenFolderOptions(m_LsOptions);
+ AddWorkerOptions(m_LsOptions);
+ AddWildcardOptions(m_LsOptions);
+
+ m_LsOptions.add_options()("h,help", "Print help");
+ m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ m_LsOptions.add_option(
+ "",
+ "",
+ "build-part-id",
+ "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
+ cxxopts::value(m_BuildPartIds),
+ "<id>");
+ m_LsOptions.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
+ "all parts will be downloaded",
+ cxxopts::value(m_BuildPartNames),
+ "<name>");
+
+ m_LsOptions.parse_positional({"build-id", "wildcard"});
+ m_LsOptions.positional_help("build-id wildcard");
+
+ // diff
AddOutputOptions(m_DiffOptions);
AddWorkerOptions(m_DiffOptions);
m_DiffOptions.add_options()("h,help", "Print help");
@@ -9991,6 +10230,7 @@ BuildsCommand::BuildsCommand()
m_DiffOptions.parse_positional({"local-path", "compare-path"});
m_DiffOptions.positional_help("local-path compare-path");
+ // test
AddSystemOptions(m_TestOptions);
AddCloudOptions(m_TestOptions);
AddFileOptions(m_TestOptions);
@@ -10020,6 +10260,7 @@ BuildsCommand::BuildsCommand()
m_TestOptions.parse_positional({"local-path"});
m_TestOptions.positional_help("local-path");
+ // fetch-blob
AddSystemOptions(m_FetchBlobOptions);
AddCloudOptions(m_FetchBlobOptions);
AddFileOptions(m_FetchBlobOptions);
@@ -10047,6 +10288,7 @@ BuildsCommand::BuildsCommand()
m_AbortOptions.parse_positional({"process-id"});
m_AbortOptions.positional_help("process-id");
+ // validate-part
AddSystemOptions(m_ValidateBuildPartOptions);
AddCloudOptions(m_ValidateBuildPartOptions);
AddFileOptions(m_ValidateBuildPartOptions);
@@ -10070,6 +10312,7 @@ BuildsCommand::BuildsCommand()
m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"});
m_ValidateBuildPartOptions.positional_help("build-id build-part-id");
+ // multi-test-download
AddSystemOptions(m_MultiTestDownloadOptions);
AddCloudOptions(m_MultiTestDownloadOptions);
AddFileOptions(m_MultiTestDownloadOptions);
@@ -10193,13 +10436,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_EncryptionKey.empty())
{
m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456";
- ZEN_CONSOLE("Warning: Using default encryption key");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Warning: Using default encryption key");
+ }
}
if (m_EncryptionIV.empty())
{
m_EncryptionIV = "0123456789abcdef";
- ZEN_CONSOLE("Warning: Using default encryption initialization vector");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Warning: Using default encryption initialization vector");
+ }
}
AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth",
@@ -10301,7 +10550,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty())
{
const std::string& CloudHost = m_OverrideHost.empty() ? m_Host : m_OverrideHost;
- ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost);
+ ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost, IsQuiet);
}
if (!ClientSettings.AccessTokenProvider)
@@ -10312,15 +10561,36 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
};
auto ParseOutputOptions = [&]() {
+ if (m_Verbose && m_Quiet)
+ {
+ throw std::runtime_error("--verbose option is not compatible with --quiet option");
+ }
IsVerbose = m_Verbose;
+ IsQuiet = m_Quiet;
if (m_LogProgress)
{
+ if (IsQuiet)
+ {
+ throw std::runtime_error("--quiet option is not compatible with --log-progress option");
+ }
ProgressMode = ProgressBar::Mode::Log;
}
- else if (m_Verbose || m_PlainProgress)
+ if (m_PlainProgress)
{
+ if (IsQuiet)
+ {
+ throw std::runtime_error("--quiet option is not compatible with --plain-progress option");
+ }
ProgressMode = ProgressBar::Mode::Plain;
}
+ else if (m_Verbose)
+ {
+ ProgressMode = ProgressBar::Mode::Plain;
+ }
+ else if (IsQuiet)
+ {
+ ProgressMode = ProgressBar::Mode::Quiet;
+ }
else
{
ProgressMode = ProgressBar::Mode::Pretty;
@@ -10489,7 +10759,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
});
}
- if (m_ZenCacheHost.empty())
+ if (m_ZenCacheHost.empty() && !IsQuiet)
{
ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
}
@@ -10578,10 +10848,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName;
}
- ZEN_CONSOLE("Remote: {}", StorageDescription);
- if (!Result.CacheName.empty())
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Cache : {}", CacheDescription);
+ ZEN_CONSOLE("Remote: {}", StorageDescription);
+ if (!Result.CacheName.empty())
+ {
+ ZEN_CONSOLE("Cache : {}", CacheDescription);
+ }
}
return Result;
};
@@ -10594,6 +10867,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
MakeSafeAbsolutePathÍnPlace(m_Path);
};
+ auto ParseFileFilters = [&]() {
+ for (auto It = begin(m_IncludeWildcard); It != end(m_IncludeWildcard); It++)
+ {
+ if (*It == '\\')
+ {
+ *It = '/';
+ }
+ }
+
+ for (auto It = begin(m_ExcludeWildcard); It != end(m_ExcludeWildcard); It++)
+ {
+ if (*It == '\\')
+ {
+ *It = '/';
+ }
+ }
+ };
+
auto ParseDiffPath = [&]() {
if (m_DiffPath.empty())
{
@@ -10742,10 +11033,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!m_ListResultPath.empty())
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
}
BuildStorage::Statistics StorageStats;
@@ -10803,10 +11097,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!m_ListResultPath.empty())
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
}
CbObject QueryObject;
if (m_ListQueryPath.empty())
@@ -10888,7 +11185,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_UploadOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
@@ -10957,34 +11260,44 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (false)
{
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.StorageName,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "{}:\n"
+ "Read: {}\n"
+ "Write: {}\n"
+ "Requests: {}\n"
+ "Avg Request Time: {}\n"
+ "Avg I/O Time: {}",
+ Storage.StorageName,
+ NiceBytes(StorageStats.TotalBytesRead.load()),
+ NiceBytes(StorageStats.TotalBytesWritten.load()),
+ StorageStats.TotalRequestCount.load(),
+ StorageStats.TotalExecutionTimeUs.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0,
+ StorageStats.TotalRequestCount.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0);
+ }
}
return AbortFlag ? 11 : 0;
}
if (SubOption == &m_DownloadOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
ParsePath();
+ ParseFileFilters();
if (m_ZenFolderPath.empty())
{
@@ -11022,6 +11335,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::vector<Oid> BuildPartIds = ParseBuildPartIds();
std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+ // TODO: Add file filters
DownloadFolder(Storage,
BuildId,
BuildPartIds,
@@ -11038,6 +11352,46 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return AbortFlag ? 11 : 0;
}
+
+ if (SubOption == &m_LsOptions)
+ {
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
+
+ ZenState InstanceState;
+
+ ParseFileFilters();
+
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
+
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
+
+ const Oid BuildId = ParseBuildId();
+
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds();
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+
+ ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard);
+
+ return AbortFlag ? 11 : 0;
+ }
+
if (SubOption == &m_DiffOptions)
{
ParsePath();
@@ -11049,7 +11403,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_FetchBlobOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
@@ -11086,16 +11446,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
return 11;
}
- ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
- BlobHash,
- CompressedSize,
- DecompressedSize);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
+ BlobHash,
+ CompressedSize,
+ DecompressedSize);
+ }
return 0;
}
if (SubOption == &m_ValidateBuildPartOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
@@ -11189,9 +11558,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("Download cancelled");
return 11;
}
- ZEN_CONSOLE("\n");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("\n");
+ }
+ }
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
- ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
return 0;
}
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index b3466c0d3..d057d24ac 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -32,6 +32,7 @@ private:
bool m_Verbose = false;
bool m_BoostWorkerThreads = false;
bool m_UseSparseFiles = true;
+ bool m_Quiet = false;
std::filesystem::path m_ZenFolderPath;
@@ -107,6 +108,10 @@ private:
bool m_PostDownloadVerify = false;
bool m_EnableScavenging = true;
+ cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"};
+ std::string m_IncludeWildcard;
+ std::string m_ExcludeWildcard;
+
cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"};
std::filesystem::path m_DiffPath;
bool m_OnlyChunked = false;
@@ -127,7 +132,7 @@ private:
cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
std::vector<std::string> m_BuildIds;
- cxxopts::Options* m_SubCommands[12] = {&m_ListNamespacesOptions,
+ cxxopts::Options* m_SubCommands[13] = {&m_ListNamespacesOptions,
&m_ListOptions,
&m_UploadOptions,
&m_DownloadOptions,
@@ -135,6 +140,7 @@ private:
&m_ResumeOptions,
&m_AbortOptions,
&m_DiffOptions,
+ &m_LsOptions,
&m_FetchBlobOptions,
&m_ValidateBuildPartOptions,
&m_TestOptions,
diff --git a/src/zen/zen.h b/src/zen/zen.h
index 995bd13b6..40c745bc7 100644
--- a/src/zen/zen.h
+++ b/src/zen/zen.h
@@ -105,7 +105,8 @@ public:
{
Plain,
Pretty,
- Log
+ Log,
+ Quiet
};
static void SetLogOperationName(Mode InMode, std::string_view Name);
diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp
index 39efe1d0c..62e1b77bc 100644
--- a/src/zenhttp/httpclientauth.cpp
+++ b/src/zenhttp/httpclientauth.cpp
@@ -89,14 +89,25 @@ namespace zen { namespace httpclientauth {
static HttpClientAccessToken GetOidcTokenFromExe(const std::filesystem::path& OidcExecutablePath,
std::string_view CloudHost,
- bool Unattended)
+ bool Unattended,
+ bool Quiet)
{
Stopwatch Timer;
CreateProcOptions ProcOptions;
+ if (Quiet)
+ {
+ ProcOptions.StdoutFile = std::filesystem::temp_directory_path() / fmt::format(".zen-auth-output-{}", Oid::NewOid());
+ }
const std::filesystem::path AuthTokenPath(std::filesystem::temp_directory_path() / fmt::format(".zen-auth-{}", Oid::NewOid()));
- auto _ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); });
+ auto _ = MakeGuard([AuthTokenPath, &ProcOptions]() {
+ RemoveFile(AuthTokenPath);
+ if (!ProcOptions.StdoutFile.empty())
+ {
+ RemoveFile(ProcOptions.StdoutFile);
+ }
+ });
const std::string ProcArgs = fmt::format("{} --AuthConfigUrl {} --OutFile {} --Unattended={}",
OidcExecutablePath,
@@ -164,13 +175,15 @@ namespace zen { namespace httpclientauth {
}
std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath,
- std::string_view CloudHost)
+ std::string_view CloudHost,
+ bool Quiet)
{
- HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, false);
+ HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ false, Quiet);
if (InitialToken.IsValid())
{
return [OidcExecutablePath = std::filesystem::path(OidcExecutablePath),
CloudHost = std::string(CloudHost),
+ Quiet,
InitialToken]() mutable {
if (InitialToken.IsValid())
{
@@ -178,7 +191,7 @@ namespace zen { namespace httpclientauth {
InitialToken = {};
return Result;
}
- return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, true);
+ return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ true, Quiet);
};
}
return {};
diff --git a/src/zenhttp/include/zenhttp/httpclientauth.h b/src/zenhttp/include/zenhttp/httpclientauth.h
index 5b9b9d305..32d00f87f 100644
--- a/src/zenhttp/include/zenhttp/httpclientauth.h
+++ b/src/zenhttp/include/zenhttp/httpclientauth.h
@@ -27,7 +27,8 @@ namespace httpclientauth {
std::function<HttpClientAccessToken()> CreateFromDefaultOpenIdProvider(AuthMgr& AuthManager);
std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath,
- std::string_view CloudHost);
+ std::string_view CloudHost,
+ bool Quiet);
} // namespace httpclientauth
} // namespace zen
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index ab96ae92d..eab687db2 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -460,7 +460,7 @@ private:
};
std::shared_ptr<RemoteProjectStore>
-CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
+CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet)
{
std::string Url = Options.Url;
if (Url.find("://"sv) == std::string::npos)
@@ -491,7 +491,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
}
else if (!Options.OidcExePath.empty())
{
- if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe)
{
TokenProvider = TokenProviderMaybe.value();
}
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h
index c52b13886..6f33ac89b 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.h
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.h
@@ -25,6 +25,7 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions
};
std::shared_ptr<RemoteProjectStore> CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
- const std::filesystem::path& TempFilePath);
+ const std::filesystem::path& TempFilePath,
+ bool Quiet);
} // namespace zen
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 3728babb5..dba5cd4a7 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -337,7 +337,7 @@ private:
};
std::shared_ptr<RemoteProjectStore>
-CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
+CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet)
{
std::string Url = Options.Url;
if (Url.find("://"sv) == std::string::npos)
@@ -368,7 +368,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
}
else if (!Options.OidcExePath.empty())
{
- if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe)
{
TokenProvider = TokenProviderMaybe.value();
}
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h
index 8bf79d563..ac2d25b47 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.h
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h
@@ -25,6 +25,7 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions
};
std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
- const std::filesystem::path& TempFilePath);
+ const std::filesystem::path& TempFilePath,
+ bool Quiet);
} // namespace zen
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 53e687983..a8c970323 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -267,7 +267,7 @@ namespace {
ForceDisableBlocks,
ForceDisableTempBlocks,
AssumeHttp2};
- RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath);
+ RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false);
}
if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -364,7 +364,7 @@ namespace {
ForceDisableTempBlocks,
AssumeHttp2,
MetaData};
- RemoteStore = CreateBuildsRemoteStore(Options, TempFilePath);
+ RemoteStore = CreateBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false);
}
if (!RemoteStore)
diff --git a/src/zenutil/include/zenutil/wildcard.h b/src/zenutil/include/zenutil/wildcard.h
new file mode 100644
index 000000000..9f402e100
--- /dev/null
+++ b/src/zenutil/include/zenutil/wildcard.h
@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <string_view>
+
+namespace zen {
+
+bool MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive);
+
+void wildcard_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zenutil/wildcard.cpp b/src/zenutil/wildcard.cpp
new file mode 100644
index 000000000..df69f6a5e
--- /dev/null
+++ b/src/zenutil/wildcard.cpp
@@ -0,0 +1,112 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/string.h>
+#include <zenutil/wildcard.h>
+
+#if ZEN_WITH_TESTS
+# include <zencore/testing.h>
+#endif // ZEN_WITH_TESTS
+
+namespace zen {
+
+bool
+MatchWildcard(std::string_view::const_iterator WildcardIt,
+ std::string_view::const_iterator WildcardEnd,
+ std::string_view::const_iterator StringIt,
+ std::string_view::const_iterator StringEnd)
+{
+ for (; WildcardIt != WildcardEnd; WildcardIt++)
+ {
+ switch (*WildcardIt)
+ {
+ case '?':
+ if (StringIt == StringEnd)
+ {
+ return false;
+ }
+ StringIt++;
+ break;
+ case '*':
+ {
+ if ((WildcardIt + 1) == WildcardEnd)
+ {
+ return true;
+ }
+ size_t Max = std::distance(StringIt, StringEnd);
+ for (size_t i = 0; i < Max; i++)
+ {
+ if (MatchWildcard(WildcardIt + 1, WildcardEnd, StringIt + i, StringEnd))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ default:
+ if (*StringIt != *WildcardIt)
+ {
+ return false;
+ }
+ ++StringIt;
+ }
+ }
+ return StringIt == StringEnd;
+}
+
+bool
+MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive)
+{
+ if (CaseSensitive)
+ {
+ return MatchWildcard(begin(Wildcard), end(Wildcard), begin(String), end(String));
+ }
+ else
+ {
+ std::string LowercaseWildcard = ToLower(Wildcard);
+ std::string LowercaseString = ToLower(String);
+ std::string_view LowercaseWildcardView(LowercaseWildcard);
+ std::string_view LowercaseStringView(LowercaseString);
+ return MatchWildcard(begin(LowercaseWildcardView),
+ end(LowercaseWildcardView),
+ begin(LowercaseStringView),
+ end(LowercaseStringView));
+ }
+}
+
+#if ZEN_WITH_TESTS
+
+void
+wildcard_forcelink()
+{
+}
+
+TEST_CASE("Wildcard")
+{
+ CHECK(MatchWildcard("*.*", "normal.txt", true));
+ CHECK(MatchWildcard("*.*", "normal.txt", false));
+
+ CHECK(!MatchWildcard("*.*", "normal", true));
+ CHECK(!MatchWildcard("*.*", "normal", false));
+
+ CHECK(MatchWildcard("*", "hey/normal", true));
+ CHECK(MatchWildcard("*", "hey/normal", false));
+
+ CHECK(MatchWildcard("hey/*.txt", "hey/normal.txt", true));
+ CHECK(MatchWildcard("*/?ormal.txt", "hey/normal.txt", true));
+ CHECK(!MatchWildcard("*/?rmal.txt", "hey/normal.txt", true));
+ CHECK(MatchWildcard("*/?ormal.*", "hey/normal.txt", true));
+ CHECK(MatchWildcard("*/?ormal", "hey/normal", true));
+
+ CHECK(MatchWildcard("hey/*.txt", "hey/normaL.txt", false));
+ CHECK(MatchWildcard("*/?ormal.TXT", "hey/normal.txt", false));
+ CHECK(MatchWildcard("*/?ORMAL.*", "hey/normal.txt", false));
+ CHECK(MatchWildcard("*/?ormal", "HEY/normal", false));
+
+ CHECK(!MatchWildcard("hey/*.txt", "heY/normal.txt", true));
+ CHECK(!MatchWildcard("*/?ormal.TXT", "hey/normal.txt", true));
+ CHECK(!MatchWildcard("*/?ORMAL.*", "hey/normal.txt", true));
+ CHECK(!MatchWildcard("*/?ormal", "hey/normaL", true));
+}
+
+#endif
+} // namespace zen
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
index fe23b00c1..37b229c49 100644
--- a/src/zenutil/zenutil.cpp
+++ b/src/zenutil/zenutil.cpp
@@ -9,6 +9,7 @@
# include <zenutil/chunkedfile.h>
# include <zenutil/commandlineoptions.h>
# include <zenutil/parallelwork.h>
+# include <zenutil/wildcard.h>
namespace zen {
@@ -21,6 +22,7 @@ zenutil_forcelinktests()
chunkedfile_forcelink();
commandlineoptions_forcelink();
parallellwork_forcelink();
+ wildcard_forcelink();
}
} // namespace zen