diff options
| author | Dan Engelbrecht <[email protected]> | 2025-08-11 12:58:27 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-08-11 12:58:27 +0200 |
| commit | eeac0654ab8f9e3438f2331916261a0440286fbd (patch) | |
| tree | 56bd48ef8dedf91eeb25a7c26471742ab189874c /src | |
| parent | Merge pull request #434 from ue-foundation/zs/put-overwrite-policy (diff) | |
| download | zen-eeac0654ab8f9e3438f2331916261a0440286fbd.tar.xz zen-eeac0654ab8f9e3438f2331916261a0440286fbd.zip | |
list build part content (#462)
- Feature: Added `zen build ls` option to list the content of a build part(s)
- Build source is specified using one of the following options
- `--cloud-url` cloud artifact URL to build
- `--host` or `--override-host`, `--namespace`, `--bucket` and `--buildid`
- `--filestorage`, `--namespace`, `--bucket` and `--buildid`
- `--build-part-name` to specify a particular build part(s) in the build
- `--wildcard` windows style wildcard (using * and ?) to match file paths to include
- `--exclude-wildcard` windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter
- Improvement: Added `--quiet` option to zen `builds` commands to suppress non-essential output
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 957 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.h | 8 | ||||
| -rw-r--r-- | src/zen/zen.h | 3 | ||||
| -rw-r--r-- | src/zenhttp/httpclientauth.cpp | 23 | ||||
| -rw-r--r-- | src/zenhttp/include/zenhttp/httpclientauth.h | 3 | ||||
| -rw-r--r-- | src/zenserver/projectstore/buildsremoteprojectstore.cpp | 4 | ||||
| -rw-r--r-- | src/zenserver/projectstore/buildsremoteprojectstore.h | 3 | ||||
| -rw-r--r-- | src/zenserver/projectstore/jupiterremoteprojectstore.cpp | 4 | ||||
| -rw-r--r-- | src/zenserver/projectstore/jupiterremoteprojectstore.h | 3 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 4 | ||||
| -rw-r--r-- | src/zenutil/include/zenutil/wildcard.h | 13 | ||||
| -rw-r--r-- | src/zenutil/wildcard.cpp | 112 | ||||
| -rw-r--r-- | src/zenutil/zenutil.cpp | 2 |
13 files changed, 832 insertions, 307 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 7cb12cd4c..8e17ba97c 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -33,6 +33,7 @@ #include <zenutil/jupiter/jupiterbuildstorage.h> #include <zenutil/jupiter/jupitersession.h> #include <zenutil/parallelwork.h> +#include <zenutil/wildcard.h> #include <zenutil/workerpools.h> #include <zenutil/zenserverprocess.h> @@ -65,6 +66,7 @@ using namespace std::literals; namespace { namespace zenutil { + #if ZEN_PLATFORM_WINDOWS class SecurityAttributes { @@ -368,6 +370,7 @@ namespace { const std::vector<std::string_view> DefaultExcludeExtensions({}); static bool IsVerbose = false; + static bool IsQuiet = false; static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) @@ -402,6 +405,20 @@ namespace { ); + bool IncludePath(const std::string_view IncludeWildcard, const std::string_view ExcludeWildcard, const std::filesystem::path& Path) + { + const std::string PathString = Path.generic_string(); + if (!IncludeWildcard.empty() && !MatchWildcard(IncludeWildcard, PathString, /*CaseSensitive*/ false)) + { + return false; + } + if (!ExcludeWildcard.empty() && MatchWildcard(ExcludeWildcard, PathString, /*CaseSensitive*/ false)) + { + return false; + } + return true; + } + bool IsFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; @@ -795,7 +812,7 @@ namespace { } uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); - if (ElapsedTimeMs >= 200) + if (ElapsedTimeMs >= 200 && !IsQuiet) { ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}", Path, @@ -2134,11 +2151,14 @@ namespace { Stopwatch Timer; auto _ = MakeGuard([&]() { - ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", - BuildId, - BuildPartId, - BuildPartName, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", + BuildId, + BuildPartId, + BuildPartName, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } }); ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount); @@ -2163,7 +2183,10 @@ namespace { CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); ValidateStats.BuildPartSize = BuildPart.GetSize(); - ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); + } std::vector<IoHash> ChunkAttachments; for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) { @@ -3770,14 +3793,17 @@ namespace { } } - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - LocalContent.Paths.size(), - NiceBytes(TotalRawSize), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load()), - Path, - NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + if (!IsQuiet) + { + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + LocalContent.Paths.size(), + NiceBytes(TotalRawSize), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load()), + Path, + NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } } const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); @@ -3790,15 +3816,18 @@ namespace { PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get(); - ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}", - NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), - CreateBuild ? "PutBuild" : "GetBuild", - NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), - NiceBytes(PrepBuildResult.PayloadSize), - IgnoreExistingBlocks ? "" - : fmt::format(". Found {} blocks in {}", - PrepBuildResult.KnownBlocks.size(), - NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + if (!IsQuiet) + { + ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}", + NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), + CreateBuild ? "PutBuild" : "GetBuild", + NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), + NiceBytes(PrepBuildResult.PayloadSize), + IgnoreExistingBlocks ? "" + : fmt::format(". Found {} blocks in {}", + PrepBuildResult.KnownBlocks.size(), + NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + } ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount); @@ -3829,7 +3858,10 @@ namespace { if (IgnoreExistingBlocks) { - ZEN_CONSOLE("Ignoring any existing blocks in store"); + if (!IsQuiet) + { + ZEN_CONSOLE("Ignoring any existing blocks in store"); + } NewBlockChunkIndexes = std::move(BlockChunkIndexes); } else @@ -3872,40 +3904,43 @@ namespace { FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) / (FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount) : 0.0; - ZEN_CONSOLE( - "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" - " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" - " Accepting {} ({}) redundant chunks ({:.1f}%)\n" - " Rejected {} ({}) chunks in {} blocks\n" - " Arranged {} ({}) chunks in {} new blocks\n" - " Keeping {} ({}) chunks as loose chunks\n" - " Discovery completed in {}", - FindBlocksStats.FoundBlockChunkCount, - FindBlocksStats.FoundBlockCount, - NiceBytes(FindBlocksStats.FoundBlockByteCount), - NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), + if (!IsQuiet) + { + ZEN_CONSOLE( + "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" + " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" + " Accepting {} ({}) redundant chunks ({:.1f}%)\n" + " Rejected {} ({}) chunks in {} blocks\n" + " Arranged {} ({}) chunks in {} new blocks\n" + " Keeping {} ({}) chunks as loose chunks\n" + " Discovery completed in {}", + FindBlocksStats.FoundBlockChunkCount, + FindBlocksStats.FoundBlockCount, + NiceBytes(FindBlocksStats.FoundBlockByteCount), + NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), - FindBlocksStats.AcceptedChunkCount, - NiceBytes(FindBlocksStats.AcceptedRawByteCount), - FindBlocksStats.AcceptedBlockCount, - AcceptedByteCountPercent, + FindBlocksStats.AcceptedChunkCount, + NiceBytes(FindBlocksStats.AcceptedRawByteCount), + FindBlocksStats.AcceptedBlockCount, + AcceptedByteCountPercent, - FindBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(FindBlocksStats.AcceptedReduntantByteCount), - AcceptedReduntantByteCountPercent, + FindBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(FindBlocksStats.AcceptedReduntantByteCount), + AcceptedReduntantByteCountPercent, - FindBlocksStats.RejectedChunkCount, - NiceBytes(FindBlocksStats.RejectedByteCount), - FindBlocksStats.RejectedBlockCount, + FindBlocksStats.RejectedChunkCount, + NiceBytes(FindBlocksStats.RejectedByteCount), + FindBlocksStats.RejectedBlockCount, - FindBlocksStats.NewBlocksChunkCount, - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), - FindBlocksStats.NewBlocksCount, + FindBlocksStats.NewBlocksChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), + FindBlocksStats.NewBlocksCount, - LooseChunksStats.ChunkCount, - NiceBytes(LooseChunksStats.ChunkByteCount), + LooseChunksStats.ChunkCount, + NiceBytes(LooseChunksStats.ChunkByteCount), - NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); + NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); + } DiskStatistics DiskStats; UploadStatistics UploadStats; @@ -3918,15 +3953,18 @@ namespace { Stopwatch GenerateBuildBlocksTimer; auto __ = MakeGuard([&]() { uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs(); - ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", - GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), - UploadStats.BlockCount.load(), - NiceBytes(UploadStats.BlocksBytes.load()), - NiceTimeSpanMs(BlockGenerateTimeUs / 1000), - NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - GenerateBlocksStats.GeneratedBlockByteCount)), - NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); + if (!IsQuiet) + { + ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + NiceTimeSpanMs(BlockGenerateTimeUs / 1000), + NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + GenerateBlocksStats.GeneratedBlockByteCount)), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); + } }); GenerateBuildBlocks(Path, LocalContent, @@ -3945,9 +3983,12 @@ namespace { CbObjectWriter PartManifestWriter; Stopwatch ManifestGenerationTimer; auto __ = MakeGuard([&]() { - ZEN_CONSOLE("Generated build part manifest in {} ({})", - NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), - NiceBytes(PartManifestWriter.GetSaveSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("Generated build part manifest in {} ({})", + NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), + NiceBytes(PartManifestWriter.GetSaveSize())); + } }); PartManifestWriter.AddObject("chunker"sv, ChunkerParameters); @@ -4096,10 +4137,13 @@ namespace { Stopwatch PutBuildPartResultTimer; std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); - ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", - NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), - NiceBytes(PartManifest.GetSize()), - PutBuildPartResult.second.size()); + if (!IsQuiet) + { + ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", + NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), + NiceBytes(PartManifest.GetSize()), + PutBuildPartResult.second.size()); + } IoHash PartHash = PutBuildPartResult.first; auto UploadAttachments = [&Storage, @@ -4122,24 +4166,27 @@ namespace { Stopwatch TempUploadTimer; auto __ = MakeGuard([&]() { - uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); - ZEN_CONSOLE( - "Uploaded {} ({}) blocks. " - "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " - "Transferred {} ({}bits/s) in {}", - TempUploadStats.BlockCount.load(), - NiceBytes(TempUploadStats.BlocksBytes), - - TempLooseChunksStats.CompressedChunkCount.load(), - NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), - NiceNum( - GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)), - TempUploadStats.ChunkCount.load(), - NiceBytes(TempUploadStats.ChunksBytes), - - NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), - NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); + if (!IsQuiet) + { + uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); + ZEN_CONSOLE( + "Uploaded {} ({}) blocks. " + "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " + "Transferred {} ({}bits/s) in {}", + TempUploadStats.BlockCount.load(), + NiceBytes(TempUploadStats.BlocksBytes), + + TempLooseChunksStats.CompressedChunkCount.load(), + NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, + TempLooseChunksStats.ChunkByteCount)), + TempUploadStats.ChunkCount.load(), + NiceBytes(TempUploadStats.ChunksBytes), + + NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), + NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); + } }); UploadPartBlobs(Storage, BuildId, @@ -4193,9 +4240,12 @@ namespace { { Stopwatch FinalizeBuildPartTimer; std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash); - ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", - NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), - Needs.size()); + if (!IsQuiet) + { + ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", + NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), + Needs.size()); + } if (Needs.empty()) { break; @@ -4208,7 +4258,10 @@ namespace { { Stopwatch FinalizeBuildTimer; Storage.BuildStorage->FinalizeBuild(BuildId); - ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); + } } if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag) @@ -4256,7 +4309,10 @@ namespace { { uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs(); UploadStats.ElapsedWallTimeUS += ElapsedUS; - ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); + if (!IsQuiet) + { + ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); + } } } @@ -4452,55 +4508,59 @@ namespace { NiceTimeSpanMs(ValidateStats.ElapsedWallTimeUS / 1000)); } - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}" - "{}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()), - - LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)), - NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000), - - FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount, - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(LooseChunksStats.ChunkByteCount), - NiceBytes(LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)), - NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(), - NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000), - - UploadStats.BlockCount.load(), - NiceBytes(UploadStats.BlocksBytes.load()), - - UploadStats.ChunkCount.load(), - NiceBytes(UploadStats.ChunksBytes.load()), - MultipartAttachmentStats, - - ValidateInfo); + if (!IsQuiet) + { + ZEN_CONSOLE( + "Uploaded part {} ('{}') to build {}, {}\n" + " Scanned files: {:>8} ({}), {}B/sec, {}\n" + " New data: {:>8} ({}) {:.1f}%\n" + " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" + " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" + " Uploaded: {:>8} ({}), {}bits/sec, {}\n" + " Blocks: {:>8} ({})\n" + " Chunks: {:>8} ({}){}" + "{}", + BuildPartId, + BuildPartName, + BuildId, + NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()), + + LocalFolderScanStats.FoundFileCount.load(), + NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)), + NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000), + + FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), + DeltaByteCountPercent, + + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum( + GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)), + NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), + + LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(LooseChunksStats.ChunkByteCount), + NiceBytes(LooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)), + NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), + + UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(), + NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)), + NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000), + + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + + UploadStats.ChunkCount.load(), + NiceBytes(UploadStats.ChunksBytes.load()), + MultipartAttachmentStats, + + ValidateInfo); + } Storage.BuildStorage->PutBuildPartStats( BuildId, @@ -5728,7 +5788,10 @@ namespace { const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + } const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath); @@ -6340,37 +6403,39 @@ namespace { } CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); } - - if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) + if (!IsQuiet) { - ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", - CachedSequenceHashesFound.size(), - NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), - CachedChunkHashesFound.size(), - NiceBytes(CacheMappingStats.CacheChunkByteCount), - CachedBlocksFound.size(), - NiceBytes(CacheMappingStats.CacheBlocksByteCount), - NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); - } + if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) + { + ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", + CachedSequenceHashesFound.size(), + NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), + CachedChunkHashesFound.size(), + NiceBytes(CacheMappingStats.CacheChunkByteCount), + CachedBlocksFound.size(), + NiceBytes(CacheMappingStats.CacheBlocksByteCount), + NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); + } - if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) - { - ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), - CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount), - NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); - } - if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) - { - ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}", - ScavengedPathsCount, - CacheMappingStats.ScavengedPathsMatchingSequencesCount, - NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), - CacheMappingStats.ScavengedChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), - NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); + if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", + LocalPathIndexesMatchingSequenceIndexes.size(), + NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), + CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount), + NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); + } + if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}", + ScavengedPathsCount, + CacheMappingStats.ScavengedPathsMatchingSequencesCount, + NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), + CacheMappingStats.ScavengedChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), + NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); + } } uint64_t BytesToWrite = 0; @@ -6758,7 +6823,7 @@ namespace { } } ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - if (!ExistsResult.ExistingBlobs.empty()) + if (!ExistsResult.ExistingBlobs.empty() && !IsQuiet) { ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}", ExistsResult.ExistingBlobs.size(), @@ -7920,10 +7985,13 @@ namespace { const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; ZEN_ASSERT(!IncompletePath.empty()); const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; - ZEN_CONSOLE("{}: Max count {}, Current count {}", - IncompletePath, - ExpectedSequenceCount, - SequenceIndexChunksLeftToWriteCounter.load()); + if (!IsQuiet) + { + ZEN_CONSOLE("{}: Max count {}, Current count {}", + IncompletePath, + ExpectedSequenceCount, + SequenceIndexChunksLeftToWriteCounter.load()); + } ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); } } @@ -7932,14 +8000,17 @@ namespace { const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); - ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}", - NiceBytes(DownloadedBytes), - NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), - NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceBytes(DiskStats.WriteByteCount.load()), - NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())), - NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}", + NiceBytes(DownloadedBytes), + NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), + NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), + NiceBytes(DiskStats.WriteByteCount.load()), + NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())), + NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), + NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); + } WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs(); WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(); @@ -8521,12 +8592,15 @@ namespace { { Stopwatch GetBuildTimer; CbObject BuildObject = Storage.GetBuild(BuildId); - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", - NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["name"sv].AsString(), - NiceBytes(BuildObject.GetSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", + NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), + BuildObject["name"sv].AsString(), + NiceBytes(BuildObject.GetSize())); - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); + } CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) @@ -8613,12 +8687,15 @@ namespace { const Oid BuildPartId = BuildParts[0].first; const std::string_view BuildPartName = BuildParts[0].second; CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); + } { CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); @@ -8745,10 +8822,13 @@ namespace { } } - ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", - BuildPartId, - NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), - OutBlockDescriptions.size()); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), + OutBlockDescriptions.size()); + } } if (OutBlockDescriptions.size() != BlockRawHashes.size()) @@ -8853,11 +8933,14 @@ namespace { const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; Stopwatch GetOverlayBuildPartTimer; CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); - ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", - OverlayBuildPartId, - OverlayBuildPartName, - NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(OverlayBuildPartManifest.GetSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", + OverlayBuildPartId, + OverlayBuildPartName, + NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(OverlayBuildPartManifest.GetSize())); + } ChunkedFolderContent OverlayPartContent; std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions; @@ -8928,7 +9011,7 @@ namespace { Stopwatch ReadStateTimer; const bool HasLocalState = IsFile(StateFilePath) && ReadStateFile(StateFilePath, LocalFolderState, LocalContent); - if (HasLocalState) + if (HasLocalState && !IsQuiet) { ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); } @@ -8992,10 +9075,13 @@ namespace { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); } - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocaStatePathCount); + if (!IsQuiet) + { + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", + DeletedPaths.size(), + UpdatedContent.Paths.size(), + LocaStatePathCount); + } if (UpdatedContent.Paths.size() > 0) { uint64_t ByteCountToScan = 0; @@ -9153,14 +9239,17 @@ namespace { const uint64_t ChunkedRawSize = std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Result.Paths.size(), - NiceBytes(TotalRawSize), - Result.ChunkedContent.ChunkHashes.size(), - NiceBytes(ChunkedRawSize), - Path, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + if (!IsQuiet) + { + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + Result.Paths.size(), + NiceBytes(TotalRawSize), + Result.ChunkedContent.ChunkHashes.size(), + NiceBytes(ChunkedRawSize), + Path, + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } return Result; }; @@ -9236,7 +9325,7 @@ namespace { { if (!WipeTargetFolder) { - if (!ChunkController) + if (!ChunkController && !IsQuiet) { ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{}); @@ -9305,14 +9394,20 @@ namespace { if (CompareContent(RemoteContent, LocalContent)) { - ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", + NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + } Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path); CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + } AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); } @@ -9326,7 +9421,10 @@ namespace { uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); + if (!IsQuiet) + { + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); + } FolderContent LocalFolderState; DiskStatistics DiskStats; @@ -9373,7 +9471,10 @@ namespace { CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + } AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); @@ -9390,30 +9491,33 @@ namespace { DownloadStats.DownloadedPartialBlockByteCount.load(); const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); - ZEN_CONSOLE( - "Downloaded build {}, parts:{} in {}\n" - " Download: {} ({}) {}bits/s\n" - " Write: {} ({}) {}B/s\n" - " Clean: {}\n" - " Finalize: {}\n" - " Verify: {}", - BuildId, - BuildPartString.ToView(), - NiceTimeSpanMs(DownloadTimeMs), + if (!IsQuiet) + { + ZEN_CONSOLE( + "Downloaded build {}, parts:{} in {}\n" + " Download: {} ({}) {}bits/s\n" + " Write: {} ({}) {}B/s\n" + " Clean: {}\n" + " Finalize: {}\n" + " Verify: {}", + BuildId, + BuildPartString.ToView(), + NiceTimeSpanMs(DownloadTimeMs), - DownloadCount, - NiceBytes(DownloadByteCount), - NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), + DownloadCount, + NiceBytes(DownloadByteCount), + NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - DiskStats.WriteCount.load(), - NiceBytes(DiskStats.WriteByteCount.load()), - NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())), + DiskStats.WriteCount.load(), + NiceBytes(DiskStats.WriteByteCount.load()), + NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())), - NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), + NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), - NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), + NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), - NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); + NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); + } } } if (PrimeCacheOnly) @@ -9421,13 +9525,16 @@ namespace { if (Storage.BuildCacheStorage) { Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { - if (Remaining == 0) + if (!IsQuiet) { - ZEN_CONSOLE("Build cache upload complete"); - } - else - { - ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); + if (Remaining == 0) + { + ZEN_CONSOLE("Build cache upload complete"); + } + else + { + ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); + } } return !AbortFlag; }); @@ -9442,6 +9549,87 @@ namespace { } } + void ListBuild(StorageInstance& Storage, + const Oid& BuildId, + const std::vector<Oid>& BuildPartIds, + std::span<const std::string> BuildPartNames, + std::string_view IncludeWildcard, + std::string_view ExcludeWildcard) + { + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + std::vector<std::pair<Oid, std::string>> AllBuildParts = + ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + + Stopwatch GetBuildPartTimer; + + for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) + { + const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; + const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); + + if (!IsQuiet) + { + ZEN_CONSOLE("{}Part: {} ('{}'):\n", + BuildPartIndex > 0 ? "\n" : "", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + } + + std::vector<std::filesystem::path> Paths; + std::vector<IoHash> RawHashes; + std::vector<uint64_t> RawSizes; + std::vector<uint32_t> Attributes; + + SourcePlatform Platform; + std::vector<IoHash> SequenceRawHashes; + std::vector<uint32_t> ChunkCounts; + std::vector<uint32_t> AbsoluteChunkOrders; + std::vector<IoHash> LooseChunkHashes; + std::vector<uint64_t> LooseChunkRawSizes; + std::vector<IoHash> BlockRawHashes; + + ReadBuildContentFromCompactBinary(BuildPartManifest, + Platform, + Paths, + RawHashes, + RawSizes, + Attributes, + SequenceRawHashes, + ChunkCounts, + AbsoluteChunkOrders, + LooseChunkHashes, + LooseChunkRawSizes, + BlockRawHashes); + + std::vector<size_t> Order(Paths.size()); + std::iota(Order.begin(), Order.end(), 0); + + std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { + const std::filesystem::path& LhsPath = Paths[Lhs]; + const std::filesystem::path& RhsPath = Paths[Rhs]; + return LhsPath < RhsPath; + }); + + for (size_t Index : Order) + { + const std::filesystem::path& Path = Paths[Index]; + if (IncludePath(IncludeWildcard, ExcludeWildcard, Path)) + { + const IoHash& RawHash = RawHashes[Index]; + const uint64_t RawSize = RawSizes[Index]; + const uint32_t Attribute = Attributes[Index]; + ZEN_UNUSED(Attribute); + + ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); + } + } + } + } + void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) { ZEN_TRACE_CPU("DiffFolders"); @@ -9774,6 +9962,7 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_LogProgress), "<logprogress>"); Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); }; auto AddWorkerOptions = [this](cxxopts::Options& Ops) { @@ -9793,6 +9982,23 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_ZenFolderPath), "<boostworkers>"); }; + + auto AddWildcardOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "wildcard", + "Windows style wildcard (using * and ?) to match file paths to include", + cxxopts::value(m_IncludeWildcard), + "<wildcard>"); + + Ops.add_option("", + "", + "exclude-wildcard", + "Windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter", + cxxopts::value(m_ExcludeWildcard), + "<excludewildcard>"); + }; + m_Options.add_option("", "v", "verb", @@ -9927,6 +10133,8 @@ BuildsCommand::BuildsCommand() AddCacheOptions(m_DownloadOptions); AddZenFolderOptions(m_DownloadOptions); AddWorkerOptions(m_DownloadOptions); + // TODO: AddWildcardOptions(m_DownloadOptions); + m_DownloadOptions.add_option("cache", "", "cache-prime-only", @@ -9977,6 +10185,37 @@ BuildsCommand::BuildsCommand() m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); m_DownloadOptions.positional_help("local-path build-id build-part-name"); + // ls + AddSystemOptions(m_LsOptions); + AddCloudOptions(m_LsOptions); + AddFileOptions(m_LsOptions); + AddOutputOptions(m_LsOptions); + AddCacheOptions(m_LsOptions); + AddZenFolderOptions(m_LsOptions); + AddWorkerOptions(m_LsOptions); + AddWildcardOptions(m_LsOptions); + + m_LsOptions.add_options()("h,help", "Print help"); + m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + m_LsOptions.add_option( + "", + "", + "build-part-id", + "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", + cxxopts::value(m_BuildPartIds), + "<id>"); + m_LsOptions.add_option("", + "", + "build-part-name", + "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " + "all parts will be downloaded", + cxxopts::value(m_BuildPartNames), + "<name>"); + + m_LsOptions.parse_positional({"build-id", "wildcard"}); + m_LsOptions.positional_help("build-id wildcard"); + + // diff AddOutputOptions(m_DiffOptions); AddWorkerOptions(m_DiffOptions); m_DiffOptions.add_options()("h,help", "Print help"); @@ -9991,6 +10230,7 @@ BuildsCommand::BuildsCommand() m_DiffOptions.parse_positional({"local-path", "compare-path"}); m_DiffOptions.positional_help("local-path compare-path"); + // test AddSystemOptions(m_TestOptions); AddCloudOptions(m_TestOptions); AddFileOptions(m_TestOptions); @@ -10020,6 +10260,7 @@ BuildsCommand::BuildsCommand() m_TestOptions.parse_positional({"local-path"}); m_TestOptions.positional_help("local-path"); + // fetch-blob AddSystemOptions(m_FetchBlobOptions); AddCloudOptions(m_FetchBlobOptions); AddFileOptions(m_FetchBlobOptions); @@ -10047,6 +10288,7 @@ BuildsCommand::BuildsCommand() m_AbortOptions.parse_positional({"process-id"}); m_AbortOptions.positional_help("process-id"); + // validate-part AddSystemOptions(m_ValidateBuildPartOptions); AddCloudOptions(m_ValidateBuildPartOptions); AddFileOptions(m_ValidateBuildPartOptions); @@ -10070,6 +10312,7 @@ BuildsCommand::BuildsCommand() m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"}); m_ValidateBuildPartOptions.positional_help("build-id build-part-id"); + // multi-test-download AddSystemOptions(m_MultiTestDownloadOptions); AddCloudOptions(m_MultiTestDownloadOptions); AddFileOptions(m_MultiTestDownloadOptions); @@ -10193,13 +10436,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_EncryptionKey.empty()) { m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; - ZEN_CONSOLE("Warning: Using default encryption key"); + if (!IsQuiet) + { + ZEN_CONSOLE("Warning: Using default encryption key"); + } } if (m_EncryptionIV.empty()) { m_EncryptionIV = "0123456789abcdef"; - ZEN_CONSOLE("Warning: Using default encryption initialization vector"); + if (!IsQuiet) + { + ZEN_CONSOLE("Warning: Using default encryption initialization vector"); + } } AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth", @@ -10301,7 +10550,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) { const std::string& CloudHost = m_OverrideHost.empty() ? m_Host : m_OverrideHost; - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost); + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost, IsQuiet); } if (!ClientSettings.AccessTokenProvider) @@ -10312,15 +10561,36 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseOutputOptions = [&]() { + if (m_Verbose && m_Quiet) + { + throw std::runtime_error("--verbose option is not compatible with --quiet option"); + } IsVerbose = m_Verbose; + IsQuiet = m_Quiet; if (m_LogProgress) { + if (IsQuiet) + { + throw std::runtime_error("--quiet option is not compatible with --log-progress option"); + } ProgressMode = ProgressBar::Mode::Log; } - else if (m_Verbose || m_PlainProgress) + if (m_PlainProgress) { + if (IsQuiet) + { + throw std::runtime_error("--quiet option is not compatible with --plain-progress option"); + } ProgressMode = ProgressBar::Mode::Plain; } + else if (m_Verbose) + { + ProgressMode = ProgressBar::Mode::Plain; + } + else if (IsQuiet) + { + ProgressMode = ProgressBar::Mode::Quiet; + } else { ProgressMode = ProgressBar::Mode::Pretty; @@ -10489,7 +10759,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); } - if (m_ZenCacheHost.empty()) + if (m_ZenCacheHost.empty() && !IsQuiet) { ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); } @@ -10578,10 +10848,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName; } - ZEN_CONSOLE("Remote: {}", StorageDescription); - if (!Result.CacheName.empty()) + if (!IsQuiet) { - ZEN_CONSOLE("Cache : {}", CacheDescription); + ZEN_CONSOLE("Remote: {}", StorageDescription); + if (!Result.CacheName.empty()) + { + ZEN_CONSOLE("Cache : {}", CacheDescription); + } } return Result; }; @@ -10594,6 +10867,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) MakeSafeAbsolutePathÍnPlace(m_Path); }; + auto ParseFileFilters = [&]() { + for (auto It = begin(m_IncludeWildcard); It != end(m_IncludeWildcard); It++) + { + if (*It == '\\') + { + *It = '/'; + } + } + + for (auto It = begin(m_ExcludeWildcard); It != end(m_ExcludeWildcard); It++) + { + if (*It == '\\') + { + *It = '/'; + } + } + }; + auto ParseDiffPath = [&]() { if (m_DiffPath.empty()) { @@ -10742,10 +11033,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_ListResultPath.empty()) { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } } BuildStorage::Statistics StorageStats; @@ -10803,10 +11097,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_ListResultPath.empty()) { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } } CbObject QueryObject; if (m_ListQueryPath.empty()) @@ -10888,7 +11185,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } ZenState InstanceState; @@ -10957,34 +11260,44 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (false) { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.StorageName, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); + if (!IsQuiet) + { + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Storage.StorageName, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); + } } return AbortFlag ? 11 : 0; } if (SubOption == &m_DownloadOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } ZenState InstanceState; ParsePath(); + ParseFileFilters(); if (m_ZenFolderPath.empty()) { @@ -11022,6 +11335,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::vector<Oid> BuildPartIds = ParseBuildPartIds(); std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + // TODO: Add file filters DownloadFolder(Storage, BuildId, BuildPartIds, @@ -11038,6 +11352,46 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return AbortFlag ? 11 : 0; } + + if (SubOption == &m_LsOptions) + { + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } + + ZenState InstanceState; + + ParseFileFilters(); + + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); + + const Oid BuildId = ParseBuildId(); + + std::vector<Oid> BuildPartIds = ParseBuildPartIds(); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + + ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard); + + return AbortFlag ? 11 : 0; + } + if (SubOption == &m_DiffOptions) { ParsePath(); @@ -11049,7 +11403,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -11086,16 +11446,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { return 11; } - ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", - BlobHash, - CompressedSize, - DecompressedSize); + if (!IsQuiet) + { + ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", + BlobHash, + CompressedSize, + DecompressedSize); + } return 0; } if (SubOption == &m_ValidateBuildPartOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } ZenState InstanceState; @@ -11189,9 +11558,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Download cancelled"); return 11; } - ZEN_CONSOLE("\n"); + if (!IsQuiet) + { + ZEN_CONSOLE("\n"); + } + } + if (!IsQuiet) + { + ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } - ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); return 0; } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index b3466c0d3..d057d24ac 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -32,6 +32,7 @@ private: bool m_Verbose = false; bool m_BoostWorkerThreads = false; bool m_UseSparseFiles = true; + bool m_Quiet = false; std::filesystem::path m_ZenFolderPath; @@ -107,6 +108,10 @@ private: bool m_PostDownloadVerify = false; bool m_EnableScavenging = true; + cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"}; + std::string m_IncludeWildcard; + std::string m_ExcludeWildcard; + cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; std::filesystem::path m_DiffPath; bool m_OnlyChunked = false; @@ -127,7 +132,7 @@ private: cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; std::vector<std::string> m_BuildIds; - cxxopts::Options* m_SubCommands[12] = {&m_ListNamespacesOptions, + cxxopts::Options* m_SubCommands[13] = {&m_ListNamespacesOptions, &m_ListOptions, &m_UploadOptions, &m_DownloadOptions, @@ -135,6 +140,7 @@ private: &m_ResumeOptions, &m_AbortOptions, &m_DiffOptions, + &m_LsOptions, &m_FetchBlobOptions, &m_ValidateBuildPartOptions, &m_TestOptions, diff --git a/src/zen/zen.h b/src/zen/zen.h index 995bd13b6..40c745bc7 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -105,7 +105,8 @@ public: { Plain, Pretty, - Log + Log, + Quiet }; static void SetLogOperationName(Mode InMode, std::string_view Name); diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp index 39efe1d0c..62e1b77bc 100644 --- a/src/zenhttp/httpclientauth.cpp +++ b/src/zenhttp/httpclientauth.cpp @@ -89,14 +89,25 @@ namespace zen { namespace httpclientauth { static HttpClientAccessToken GetOidcTokenFromExe(const std::filesystem::path& OidcExecutablePath, std::string_view CloudHost, - bool Unattended) + bool Unattended, + bool Quiet) { Stopwatch Timer; CreateProcOptions ProcOptions; + if (Quiet) + { + ProcOptions.StdoutFile = std::filesystem::temp_directory_path() / fmt::format(".zen-auth-output-{}", Oid::NewOid()); + } const std::filesystem::path AuthTokenPath(std::filesystem::temp_directory_path() / fmt::format(".zen-auth-{}", Oid::NewOid())); - auto _ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); }); + auto _ = MakeGuard([AuthTokenPath, &ProcOptions]() { + RemoveFile(AuthTokenPath); + if (!ProcOptions.StdoutFile.empty()) + { + RemoveFile(ProcOptions.StdoutFile); + } + }); const std::string ProcArgs = fmt::format("{} --AuthConfigUrl {} --OutFile {} --Unattended={}", OidcExecutablePath, @@ -164,13 +175,15 @@ namespace zen { namespace httpclientauth { } std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath, - std::string_view CloudHost) + std::string_view CloudHost, + bool Quiet) { - HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, false); + HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ false, Quiet); if (InitialToken.IsValid()) { return [OidcExecutablePath = std::filesystem::path(OidcExecutablePath), CloudHost = std::string(CloudHost), + Quiet, InitialToken]() mutable { if (InitialToken.IsValid()) { @@ -178,7 +191,7 @@ namespace zen { namespace httpclientauth { InitialToken = {}; return Result; } - return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, true); + return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ true, Quiet); }; } return {}; diff --git a/src/zenhttp/include/zenhttp/httpclientauth.h b/src/zenhttp/include/zenhttp/httpclientauth.h index 5b9b9d305..32d00f87f 100644 --- a/src/zenhttp/include/zenhttp/httpclientauth.h +++ b/src/zenhttp/include/zenhttp/httpclientauth.h @@ -27,7 +27,8 @@ namespace httpclientauth { std::function<HttpClientAccessToken()> CreateFromDefaultOpenIdProvider(AuthMgr& AuthManager); std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath, - std::string_view CloudHost); + std::string_view CloudHost, + bool Quiet); } // namespace httpclientauth } // namespace zen diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index ab96ae92d..eab687db2 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -460,7 +460,7 @@ private: }; std::shared_ptr<RemoteProjectStore> -CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath) +CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet) { std::string Url = Options.Url; if (Url.find("://"sv) == std::string::npos) @@ -491,7 +491,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file } else if (!Options.OidcExePath.empty()) { - if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe) + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe) { TokenProvider = TokenProviderMaybe.value(); } diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h index c52b13886..6f33ac89b 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.h +++ b/src/zenserver/projectstore/buildsremoteprojectstore.h @@ -25,6 +25,7 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions }; std::shared_ptr<RemoteProjectStore> CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, - const std::filesystem::path& TempFilePath); + const std::filesystem::path& TempFilePath, + bool Quiet); } // namespace zen diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index 3728babb5..dba5cd4a7 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -337,7 +337,7 @@ private: }; std::shared_ptr<RemoteProjectStore> -CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath) +CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet) { std::string Url = Options.Url; if (Url.find("://"sv) == std::string::npos) @@ -368,7 +368,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi } else if (!Options.OidcExePath.empty()) { - if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe) + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe) { TokenProvider = TokenProviderMaybe.value(); } diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h index 8bf79d563..ac2d25b47 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.h +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h @@ -25,6 +25,7 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions }; std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, - const std::filesystem::path& TempFilePath); + const std::filesystem::path& TempFilePath, + bool Quiet); } // namespace zen diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 53e687983..a8c970323 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -267,7 +267,7 @@ namespace { ForceDisableBlocks, ForceDisableTempBlocks, AssumeHttp2}; - RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath); + RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false); } if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) @@ -364,7 +364,7 @@ namespace { ForceDisableTempBlocks, AssumeHttp2, MetaData}; - RemoteStore = CreateBuildsRemoteStore(Options, TempFilePath); + RemoteStore = CreateBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false); } if (!RemoteStore) diff --git a/src/zenutil/include/zenutil/wildcard.h b/src/zenutil/include/zenutil/wildcard.h new file mode 100644 index 000000000..9f402e100 --- /dev/null +++ b/src/zenutil/include/zenutil/wildcard.h @@ -0,0 +1,13 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <string_view> + +namespace zen { + +bool MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive); + +void wildcard_forcelink(); // internal + +} // namespace zen diff --git a/src/zenutil/wildcard.cpp b/src/zenutil/wildcard.cpp new file mode 100644 index 000000000..df69f6a5e --- /dev/null +++ b/src/zenutil/wildcard.cpp @@ -0,0 +1,112 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/string.h> +#include <zenutil/wildcard.h> + +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +#endif // ZEN_WITH_TESTS + +namespace zen { + +bool +MatchWildcard(std::string_view::const_iterator WildcardIt, + std::string_view::const_iterator WildcardEnd, + std::string_view::const_iterator StringIt, + std::string_view::const_iterator StringEnd) +{ + for (; WildcardIt != WildcardEnd; WildcardIt++) + { + switch (*WildcardIt) + { + case '?': + if (StringIt == StringEnd) + { + return false; + } + StringIt++; + break; + case '*': + { + if ((WildcardIt + 1) == WildcardEnd) + { + return true; + } + size_t Max = std::distance(StringIt, StringEnd); + for (size_t i = 0; i < Max; i++) + { + if (MatchWildcard(WildcardIt + 1, WildcardEnd, StringIt + i, StringEnd)) + { + return true; + } + } + return false; + } + default: + if (*StringIt != *WildcardIt) + { + return false; + } + ++StringIt; + } + } + return StringIt == StringEnd; +} + +bool +MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive) +{ + if (CaseSensitive) + { + return MatchWildcard(begin(Wildcard), end(Wildcard), begin(String), end(String)); + } + else + { + std::string LowercaseWildcard = ToLower(Wildcard); + std::string LowercaseString = ToLower(String); + std::string_view LowercaseWildcardView(LowercaseWildcard); + std::string_view LowercaseStringView(LowercaseString); + return MatchWildcard(begin(LowercaseWildcardView), + end(LowercaseWildcardView), + begin(LowercaseStringView), + end(LowercaseStringView)); + } +} + +#if ZEN_WITH_TESTS + +void +wildcard_forcelink() +{ +} + +TEST_CASE("Wildcard") +{ + CHECK(MatchWildcard("*.*", "normal.txt", true)); + CHECK(MatchWildcard("*.*", "normal.txt", false)); + + CHECK(!MatchWildcard("*.*", "normal", true)); + CHECK(!MatchWildcard("*.*", "normal", false)); + + CHECK(MatchWildcard("*", "hey/normal", true)); + CHECK(MatchWildcard("*", "hey/normal", false)); + + CHECK(MatchWildcard("hey/*.txt", "hey/normal.txt", true)); + CHECK(MatchWildcard("*/?ormal.txt", "hey/normal.txt", true)); + CHECK(!MatchWildcard("*/?rmal.txt", "hey/normal.txt", true)); + CHECK(MatchWildcard("*/?ormal.*", "hey/normal.txt", true)); + CHECK(MatchWildcard("*/?ormal", "hey/normal", true)); + + CHECK(MatchWildcard("hey/*.txt", "hey/normaL.txt", false)); + CHECK(MatchWildcard("*/?ormal.TXT", "hey/normal.txt", false)); + CHECK(MatchWildcard("*/?ORMAL.*", "hey/normal.txt", false)); + CHECK(MatchWildcard("*/?ormal", "HEY/normal", false)); + + CHECK(!MatchWildcard("hey/*.txt", "heY/normal.txt", true)); + CHECK(!MatchWildcard("*/?ormal.TXT", "hey/normal.txt", true)); + CHECK(!MatchWildcard("*/?ORMAL.*", "hey/normal.txt", true)); + CHECK(!MatchWildcard("*/?ormal", "hey/normaL", true)); +} + +#endif +} // namespace zen diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp index fe23b00c1..37b229c49 100644 --- a/src/zenutil/zenutil.cpp +++ b/src/zenutil/zenutil.cpp @@ -9,6 +9,7 @@ # include <zenutil/chunkedfile.h> # include <zenutil/commandlineoptions.h> # include <zenutil/parallelwork.h> +# include <zenutil/wildcard.h> namespace zen { @@ -21,6 +22,7 @@ zenutil_forcelinktests() chunkedfile_forcelink(); commandlineoptions_forcelink(); parallellwork_forcelink(); + wildcard_forcelink(); } } // namespace zen |