diff options
| author | Liam Mitchell <[email protected]> | 2025-08-21 23:58:51 +0000 |
|---|---|---|
| committer | Liam Mitchell <[email protected]> | 2025-08-21 23:58:51 +0000 |
| commit | 33209bd6931f49362dfc2d62c6cb6b87a42c99e1 (patch) | |
| tree | cfc7914634088b3f4feac2d4cec0b5650dfdcc3c /src | |
| parent | Fix changelog merge issues (diff) | |
| parent | avoid new in static IoBuffer (#472) (diff) | |
| download | zen-33209bd6931f49362dfc2d62c6cb6b87a42c99e1.tar.xz zen-33209bd6931f49362dfc2d62c6cb6b87a42c99e1.zip | |
Merge remote-tracking branch 'origin/main' into de/zen-service-command
Diffstat (limited to 'src')
70 files changed, 4543 insertions, 2208 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index cd27daa9e..8692e5941 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -33,6 +33,7 @@ #include <zenutil/jupiter/jupiterbuildstorage.h> #include <zenutil/jupiter/jupitersession.h> #include <zenutil/parallelwork.h> +#include <zenutil/wildcard.h> #include <zenutil/workerpools.h> #include <zenutil/zenserverprocess.h> @@ -65,6 +66,7 @@ using namespace std::literals; namespace { namespace zenutil { + #if ZEN_PLATFORM_WINDOWS class SecurityAttributes { @@ -368,6 +370,7 @@ namespace { const std::vector<std::string_view> DefaultExcludeExtensions({}); static bool IsVerbose = false; + static bool IsQuiet = false; static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) @@ -402,6 +405,20 @@ namespace { ); + bool IncludePath(const std::string_view IncludeWildcard, const std::string_view ExcludeWildcard, const std::filesystem::path& Path) + { + const std::string PathString = Path.generic_string(); + if (!IncludeWildcard.empty() && !MatchWildcard(IncludeWildcard, PathString, /*CaseSensitive*/ false)) + { + return false; + } + if (!ExcludeWildcard.empty() && MatchWildcard(ExcludeWildcard, PathString, /*CaseSensitive*/ false)) + { + return false; + } + return true; + } + bool IsFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; @@ -795,7 +812,7 @@ namespace { } uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); - if (ElapsedTimeMs >= 200) + if (ElapsedTimeMs >= 200 && !IsQuiet) { ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}", Path, @@ -2134,11 +2151,14 @@ namespace { Stopwatch Timer; auto _ = MakeGuard([&]() { - ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", - BuildId, - BuildPartId, - BuildPartName, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", + BuildId, + BuildPartId, + BuildPartName, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } }); ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount); @@ -2163,7 +2183,10 @@ namespace { CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); ValidateStats.BuildPartSize = BuildPart.GetSize(); - ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); + } std::vector<IoHash> ChunkAttachments; for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) { @@ -2904,8 +2927,10 @@ namespace { } else { - throw std::runtime_error( - fmt::format("Can not upload requested build blob {} as it was not generated by this upload", RawHash)); + ZEN_CONSOLE( + "Warning: Build blob {} was reported as needed for upload but it was reported as existing at the start of the " + "operation. Treating it as a transient inconsistent issue and will attempt to retry finalization", + RawHash); } } uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize; @@ -3768,14 +3793,17 @@ namespace { } } - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - LocalContent.Paths.size(), - NiceBytes(TotalRawSize), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load()), - Path, - NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + if (!IsQuiet) + { + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + LocalContent.Paths.size(), + NiceBytes(TotalRawSize), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load()), + Path, + NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } } const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); @@ -3788,15 +3816,18 @@ namespace { PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get(); - ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}", - NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), - CreateBuild ? "PutBuild" : "GetBuild", - NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), - NiceBytes(PrepBuildResult.PayloadSize), - IgnoreExistingBlocks ? "" - : fmt::format(". Found {} blocks in {}", - PrepBuildResult.KnownBlocks.size(), - NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + if (!IsQuiet) + { + ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}", + NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), + CreateBuild ? "PutBuild" : "GetBuild", + NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), + NiceBytes(PrepBuildResult.PayloadSize), + IgnoreExistingBlocks ? "" + : fmt::format(". Found {} blocks in {}", + PrepBuildResult.KnownBlocks.size(), + NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + } ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount); @@ -3827,7 +3858,10 @@ namespace { if (IgnoreExistingBlocks) { - ZEN_CONSOLE("Ignoring any existing blocks in store"); + if (!IsQuiet) + { + ZEN_CONSOLE("Ignoring any existing blocks in store"); + } NewBlockChunkIndexes = std::move(BlockChunkIndexes); } else @@ -3870,40 +3904,43 @@ namespace { FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) / (FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount) : 0.0; - ZEN_CONSOLE( - "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" - " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" - " Accepting {} ({}) redundant chunks ({:.1f}%)\n" - " Rejected {} ({}) chunks in {} blocks\n" - " Arranged {} ({}) chunks in {} new blocks\n" - " Keeping {} ({}) chunks as loose chunks\n" - " Discovery completed in {}", - FindBlocksStats.FoundBlockChunkCount, - FindBlocksStats.FoundBlockCount, - NiceBytes(FindBlocksStats.FoundBlockByteCount), - NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), + if (!IsQuiet) + { + ZEN_CONSOLE( + "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" + " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" + " Accepting {} ({}) redundant chunks ({:.1f}%)\n" + " Rejected {} ({}) chunks in {} blocks\n" + " Arranged {} ({}) chunks in {} new blocks\n" + " Keeping {} ({}) chunks as loose chunks\n" + " Discovery completed in {}", + FindBlocksStats.FoundBlockChunkCount, + FindBlocksStats.FoundBlockCount, + NiceBytes(FindBlocksStats.FoundBlockByteCount), + NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), - FindBlocksStats.AcceptedChunkCount, - NiceBytes(FindBlocksStats.AcceptedRawByteCount), - FindBlocksStats.AcceptedBlockCount, - AcceptedByteCountPercent, + FindBlocksStats.AcceptedChunkCount, + NiceBytes(FindBlocksStats.AcceptedRawByteCount), + FindBlocksStats.AcceptedBlockCount, + AcceptedByteCountPercent, - FindBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(FindBlocksStats.AcceptedReduntantByteCount), - AcceptedReduntantByteCountPercent, + FindBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(FindBlocksStats.AcceptedReduntantByteCount), + AcceptedReduntantByteCountPercent, - FindBlocksStats.RejectedChunkCount, - NiceBytes(FindBlocksStats.RejectedByteCount), - FindBlocksStats.RejectedBlockCount, + FindBlocksStats.RejectedChunkCount, + NiceBytes(FindBlocksStats.RejectedByteCount), + FindBlocksStats.RejectedBlockCount, - FindBlocksStats.NewBlocksChunkCount, - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), - FindBlocksStats.NewBlocksCount, + FindBlocksStats.NewBlocksChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), + FindBlocksStats.NewBlocksCount, - LooseChunksStats.ChunkCount, - NiceBytes(LooseChunksStats.ChunkByteCount), + LooseChunksStats.ChunkCount, + NiceBytes(LooseChunksStats.ChunkByteCount), - NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); + NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); + } DiskStatistics DiskStats; UploadStatistics UploadStats; @@ -3916,15 +3953,18 @@ namespace { Stopwatch GenerateBuildBlocksTimer; auto __ = MakeGuard([&]() { uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs(); - ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", - GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), - UploadStats.BlockCount.load(), - NiceBytes(UploadStats.BlocksBytes.load()), - NiceTimeSpanMs(BlockGenerateTimeUs / 1000), - NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - GenerateBlocksStats.GeneratedBlockByteCount)), - NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); + if (!IsQuiet) + { + ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + NiceTimeSpanMs(BlockGenerateTimeUs / 1000), + NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + GenerateBlocksStats.GeneratedBlockByteCount)), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); + } }); GenerateBuildBlocks(Path, LocalContent, @@ -3943,9 +3983,12 @@ namespace { CbObjectWriter PartManifestWriter; Stopwatch ManifestGenerationTimer; auto __ = MakeGuard([&]() { - ZEN_CONSOLE("Generated build part manifest in {} ({})", - NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), - NiceBytes(PartManifestWriter.GetSaveSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("Generated build part manifest in {} ({})", + NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), + NiceBytes(PartManifestWriter.GetSaveSize())); + } }); PartManifestWriter.AddObject("chunker"sv, ChunkerParameters); @@ -4094,10 +4137,13 @@ namespace { Stopwatch PutBuildPartResultTimer; std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); - ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", - NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), - NiceBytes(PartManifest.GetSize()), - PutBuildPartResult.second.size()); + if (!IsQuiet) + { + ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", + NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), + NiceBytes(PartManifest.GetSize()), + PutBuildPartResult.second.size()); + } IoHash PartHash = PutBuildPartResult.first; auto UploadAttachments = [&Storage, @@ -4120,24 +4166,27 @@ namespace { Stopwatch TempUploadTimer; auto __ = MakeGuard([&]() { - uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); - ZEN_CONSOLE( - "Uploaded {} ({}) blocks. " - "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " - "Transferred {} ({}bits/s) in {}", - TempUploadStats.BlockCount.load(), - NiceBytes(TempUploadStats.BlocksBytes), - - TempLooseChunksStats.CompressedChunkCount.load(), - NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), - NiceNum( - GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)), - TempUploadStats.ChunkCount.load(), - NiceBytes(TempUploadStats.ChunksBytes), - - NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), - NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); + if (!IsQuiet) + { + uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); + ZEN_CONSOLE( + "Uploaded {} ({}) blocks. " + "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " + "Transferred {} ({}bits/s) in {}", + TempUploadStats.BlockCount.load(), + NiceBytes(TempUploadStats.BlocksBytes), + + TempLooseChunksStats.CompressedChunkCount.load(), + NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, + TempLooseChunksStats.ChunkByteCount)), + TempUploadStats.ChunkCount.load(), + NiceBytes(TempUploadStats.ChunksBytes), + + NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), + NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); + } }); UploadPartBlobs(Storage, BuildId, @@ -4191,9 +4240,12 @@ namespace { { Stopwatch FinalizeBuildPartTimer; std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash); - ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", - NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), - Needs.size()); + if (!IsQuiet) + { + ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", + NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), + Needs.size()); + } if (Needs.empty()) { break; @@ -4206,7 +4258,10 @@ namespace { { Stopwatch FinalizeBuildTimer; Storage.BuildStorage->FinalizeBuild(BuildId); - ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); + } } if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag) @@ -4254,7 +4309,10 @@ namespace { { uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs(); UploadStats.ElapsedWallTimeUS += ElapsedUS; - ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); + if (!IsQuiet) + { + ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); + } } } @@ -4450,55 +4508,59 @@ namespace { NiceTimeSpanMs(ValidateStats.ElapsedWallTimeUS / 1000)); } - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}" - "{}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()), - - LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)), - NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000), - - FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount, - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(LooseChunksStats.ChunkByteCount), - NiceBytes(LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)), - NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(), - NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000), - - UploadStats.BlockCount.load(), - NiceBytes(UploadStats.BlocksBytes.load()), - - UploadStats.ChunkCount.load(), - NiceBytes(UploadStats.ChunksBytes.load()), - MultipartAttachmentStats, - - ValidateInfo); + if (!IsQuiet) + { + ZEN_CONSOLE( + "Uploaded part {} ('{}') to build {}, {}\n" + " Scanned files: {:>8} ({}), {}B/sec, {}\n" + " New data: {:>8} ({}) {:.1f}%\n" + " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" + " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" + " Uploaded: {:>8} ({}), {}bits/sec, {}\n" + " Blocks: {:>8} ({})\n" + " Chunks: {:>8} ({}){}" + "{}", + BuildPartId, + BuildPartName, + BuildId, + NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()), + + LocalFolderScanStats.FoundFileCount.load(), + NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)), + NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000), + + FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), + DeltaByteCountPercent, + + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum( + GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)), + NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), + + LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(LooseChunksStats.ChunkByteCount), + NiceBytes(LooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)), + NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), + + UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(), + NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)), + NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000), + + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + + UploadStats.ChunkCount.load(), + NiceBytes(UploadStats.ChunksBytes.load()), + MultipartAttachmentStats, + + ValidateInfo); + } Storage.BuildStorage->PutBuildPartStats( BuildId, @@ -5726,7 +5788,10 @@ namespace { const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + } const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath); @@ -6073,7 +6138,7 @@ namespace { Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); - std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavanging", + std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavenging", PathsScavenged.load(), ScavengePathCount, PathsFound.load(), @@ -6338,37 +6403,39 @@ namespace { } CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); } - - if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) + if (!IsQuiet) { - ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", - CachedSequenceHashesFound.size(), - NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), - CachedChunkHashesFound.size(), - NiceBytes(CacheMappingStats.CacheChunkByteCount), - CachedBlocksFound.size(), - NiceBytes(CacheMappingStats.CacheBlocksByteCount), - NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); - } + if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) + { + ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", + CachedSequenceHashesFound.size(), + NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), + CachedChunkHashesFound.size(), + NiceBytes(CacheMappingStats.CacheChunkByteCount), + CachedBlocksFound.size(), + NiceBytes(CacheMappingStats.CacheBlocksByteCount), + NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); + } - if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) - { - ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), - CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount), - NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); - } - if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) - { - ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}", - ScavengedPathsCount, - CacheMappingStats.ScavengedPathsMatchingSequencesCount, - NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), - CacheMappingStats.ScavengedChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), - NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); + if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", + LocalPathIndexesMatchingSequenceIndexes.size(), + NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), + CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount), + NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); + } + if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}", + ScavengedPathsCount, + CacheMappingStats.ScavengedPathsMatchingSequencesCount, + NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), + CacheMappingStats.ScavengedChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), + NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); + } } uint64_t BytesToWrite = 0; @@ -6756,7 +6823,7 @@ namespace { } } ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - if (!ExistsResult.ExistingBlobs.empty()) + if (!ExistsResult.ExistingBlobs.empty() && !IsQuiet) { ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}", ExistsResult.ExistingBlobs.size(), @@ -7918,10 +7985,13 @@ namespace { const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; ZEN_ASSERT(!IncompletePath.empty()); const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; - ZEN_CONSOLE("{}: Max count {}, Current count {}", - IncompletePath, - ExpectedSequenceCount, - SequenceIndexChunksLeftToWriteCounter.load()); + if (!IsQuiet) + { + ZEN_CONSOLE("{}: Max count {}, Current count {}", + IncompletePath, + ExpectedSequenceCount, + SequenceIndexChunksLeftToWriteCounter.load()); + } ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); } } @@ -7930,14 +8000,17 @@ namespace { const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); - ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}", - NiceBytes(DownloadedBytes), - NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), - NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceBytes(DiskStats.WriteByteCount.load()), - NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())), - NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}", + NiceBytes(DownloadedBytes), + NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), + NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), + NiceBytes(DiskStats.WriteByteCount.load()), + NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())), + NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), + NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); + } WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs(); WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(); @@ -8519,12 +8592,15 @@ namespace { { Stopwatch GetBuildTimer; CbObject BuildObject = Storage.GetBuild(BuildId); - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", - NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["name"sv].AsString(), - NiceBytes(BuildObject.GetSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", + NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), + BuildObject["name"sv].AsString(), + NiceBytes(BuildObject.GetSize())); - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); + } CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) @@ -8600,6 +8676,8 @@ namespace { ChunkedFolderContent GetRemoteContent(StorageInstance& Storage, const Oid& BuildId, const std::vector<std::pair<Oid, std::string>>& BuildParts, + std::string_view IncludeWildcard, + std::string_view ExcludeWildcard, std::unique_ptr<ChunkingController>& OutChunkController, std::vector<ChunkedFolderContent>& OutPartContents, std::vector<ChunkBlockDescription>& OutBlockDescriptions, @@ -8611,12 +8689,15 @@ namespace { const Oid BuildPartId = BuildParts[0].first; const std::string_view BuildPartName = BuildParts[0].second; CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); + } { CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); @@ -8629,6 +8710,8 @@ namespace { const Oid& BuildId, const Oid& BuildPartId, CbObject BuildPartManifest, + std::string_view IncludeWildcard, + std::string_view ExcludeWildcard, ChunkedFolderContent& OutRemoteContent, std::vector<ChunkBlockDescription>& OutBlockDescriptions, std::vector<IoHash>& OutLooseChunkHashes) { @@ -8656,7 +8739,7 @@ namespace { std::vector<ChunkBlockDescription> UnorderedList; tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup; - if (Storage.BuildCacheStorage) + if (Storage.BuildCacheStorage && !BlockRawHashes.empty()) { std::vector<CbObject> CacheBlockMetadatas = Storage.BuildCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes); UnorderedList.reserve(CacheBlockMetadatas.size()); @@ -8743,10 +8826,13 @@ namespace { } } - ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", - BuildPartId, - NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), - OutBlockDescriptions.size()); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), + OutBlockDescriptions.size()); + } } if (OutBlockDescriptions.size() != BlockRawHashes.size()) @@ -8830,6 +8916,36 @@ namespace { OutRemoteContent.ChunkedContent.ChunkHashes, OutRemoteContent.ChunkedContent.ChunkRawSizes, OutRemoteContent.ChunkedContent.ChunkOrders); + + { + std::vector<std::filesystem::path> DeletedPaths; + for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) + { + if (!IncludePath(IncludeWildcard, ExcludeWildcard, RemotePath)) + { + DeletedPaths.push_back(RemotePath); + } + } + + if (!DeletedPaths.empty()) + { + OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths); + + tsl::robin_set<IoHash, IoHash::Hasher> UsedLooseChunkHashes; + UsedLooseChunkHashes.insert(OutRemoteContent.RawHashes.begin(), OutRemoteContent.RawHashes.end()); + for (auto It = OutLooseChunkHashes.begin(); It != OutLooseChunkHashes.end();) + { + if (!UsedLooseChunkHashes.contains(*It)) + { + It = OutLooseChunkHashes.erase(It); + } + else + { + It++; + } + } + } + } }; OutPartContents.resize(1); @@ -8837,6 +8953,8 @@ namespace { BuildId, BuildPartId, BuildPartManifest, + IncludeWildcard, + ExcludeWildcard, OutPartContents[0], OutBlockDescriptions, OutLooseChunkHashes); @@ -8851,11 +8969,14 @@ namespace { const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; Stopwatch GetOverlayBuildPartTimer; CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); - ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", - OverlayBuildPartId, - OverlayBuildPartName, - NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(OverlayBuildPartManifest.GetSize())); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", + OverlayBuildPartId, + OverlayBuildPartName, + NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(OverlayBuildPartManifest.GetSize())); + } ChunkedFolderContent OverlayPartContent; std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions; @@ -8865,6 +8986,8 @@ namespace { BuildId, OverlayBuildPartId, OverlayBuildPartManifest, + IncludeWildcard, + ExcludeWildcard, OverlayPartContent, OverlayPartBlockDescriptions, OverlayPartLooseChunkHashes); @@ -8919,6 +9042,8 @@ namespace { const std::filesystem::path& StateFilePath, ChunkingController& ChunkController, std::span<const std::filesystem::path> ReferencePaths, + std::string_view IncludeWildcard, + std::string_view ExcludeWildcard, FolderContent& OutLocalFolderContent) { FolderContent LocalFolderState; @@ -8926,13 +9051,13 @@ namespace { Stopwatch ReadStateTimer; const bool HasLocalState = IsFile(StateFilePath) && ReadStateFile(StateFilePath, LocalFolderState, LocalContent); - if (HasLocalState) + if (HasLocalState && !IsQuiet) { ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); } { - const uint32_t LocalPathCount = gsl::narrow<uint32_t>(ReferencePaths.size()); - const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size()); + const uint32_t LocalPathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size()); + const uint32_t RemotePathCount = gsl::narrow<uint32_t>(ReferencePaths.size()); std::vector<std::filesystem::path> PathsToCheck; PathsToCheck.reserve(LocalPathCount + RemotePathCount); @@ -8942,34 +9067,44 @@ namespace { for (const std::filesystem::path& LocalPath : LocalFolderState.Paths) { - FileSet.insert(LocalPath.generic_string()); - PathsToCheck.push_back(LocalPath); + if (IncludePath(IncludeWildcard, ExcludeWildcard, LocalPath)) + { + FileSet.insert(LocalPath.generic_string()); + PathsToCheck.push_back(LocalPath); + } } for (const std::filesystem::path& RemotePath : ReferencePaths) { - if (FileSet.insert(RemotePath.generic_string()).second) + if (IncludePath(IncludeWildcard, ExcludeWildcard, RemotePath)) { - PathsToCheck.push_back(RemotePath); + if (FileSet.insert(RemotePath.generic_string()).second) + { + PathsToCheck.push_back(RemotePath); + } } } - ProgressBar ProgressBar(ProgressMode, "Check Files"); - OutLocalFolderContent = GetValidFolderContent( - LocalFolderScanStats, - Path, - PathsToCheck, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }); - ProgressBar.Finish(); + { + ProgressBar ProgressBar(ProgressMode, "Check Files"); + OutLocalFolderContent = + GetValidFolderContent(LocalFolderScanStats, + Path, + PathsToCheck, + [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { + std::string Details = fmt::format("{}/{} checked, {} found", + CompletedPathCount, + PathCount, + LocalFolderScanStats.FoundFileCount.load()); + ProgressBar.UpdateState({.Task = "Checking files ", + .Details = Details, + .TotalCount = PathCount, + .RemainingCount = PathCount - CompletedPathCount, + .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, + false); + }); + ProgressBar.Finish(); + } if (AbortFlag) { return {}; @@ -8982,7 +9117,7 @@ namespace { { if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) { - const size_t LocaStatePathCount = LocalFolderState.Paths.size(); + const size_t LocalStatePathCount = LocalFolderState.Paths.size(); std::vector<std::filesystem::path> DeletedPaths; FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); if (!DeletedPaths.empty()) @@ -8990,10 +9125,13 @@ namespace { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); } - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocaStatePathCount); + if (!IsQuiet) + { + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", + DeletedPaths.size(), + UpdatedContent.Paths.size(), + LocalStatePathCount); + } if (UpdatedContent.Paths.size() > 0) { uint64_t ByteCountToScan = 0; @@ -9062,6 +9200,85 @@ namespace { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); } } + + // Check files that are present in current folder state that matches the ReferencePaths but not known to the local state + { + FolderContent UpdatedContent; + + tsl::robin_set<std::string> LocalPathIndexLookup; + for (const std::filesystem::path& LocalPath : LocalContent.Paths) + { + LocalPathIndexLookup.insert(LocalPath.generic_string()); + } + + tsl::robin_set<std::string> RemotePathIndexLookup; + for (const std::filesystem::path& RemotePath : ReferencePaths) + { + RemotePathIndexLookup.insert(RemotePath.generic_string()); + } + + for (uint32_t LocalFolderPathIndex = 0; LocalFolderPathIndex < OutLocalFolderContent.Paths.size(); LocalFolderPathIndex++) + { + const std::filesystem::path& LocalFolderPath = OutLocalFolderContent.Paths[LocalFolderPathIndex]; + const std::string GenericLocalFolderPath = LocalFolderPath.generic_string(); + if (RemotePathIndexLookup.contains(GenericLocalFolderPath)) + { + if (!LocalPathIndexLookup.contains(GenericLocalFolderPath)) + { + UpdatedContent.Paths.push_back(LocalFolderPath); + UpdatedContent.RawSizes.push_back(OutLocalFolderContent.RawSizes[LocalFolderPathIndex]); + UpdatedContent.Attributes.push_back(OutLocalFolderContent.Attributes[LocalFolderPathIndex]); + UpdatedContent.ModificationTicks.push_back(OutLocalFolderContent.ModificationTicks[LocalFolderPathIndex]); + } + } + } + + if (UpdatedContent.Paths.size() > 0) + { + uint64_t ByteCountToScan = 0; + for (const uint64_t RawSize : UpdatedContent.RawSizes) + { + ByteCountToScan += RawSize; + } + ProgressBar ProgressBar(ProgressMode, "Scan Files"); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + ChunkingStats, + GetIOWorkerPool(), + Path, + UpdatedContent, + ChunkController, + GetUpdateDelayMS(ProgressMode), + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + UpdatedContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(ByteCountToScan), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = ByteCountToScan, + .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load(), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }, + AbortFlag, + PauseFlag); + + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + if (!AbortFlag) + { + LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); + } + } + } + ScanContent = false; } @@ -9145,20 +9362,25 @@ namespace { ZenStateFilePath(Path / ZenFolderName), ChunkController, Content.Paths, + {}, + {}, _); const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); const uint64_t ChunkedRawSize = std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Result.Paths.size(), - NiceBytes(TotalRawSize), - Result.ChunkedContent.ChunkHashes.size(), - NiceBytes(ChunkedRawSize), - Path, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + if (!IsQuiet) + { + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + Result.Paths.size(), + NiceBytes(TotalRawSize), + Result.ChunkedContent.ChunkHashes.size(), + NiceBytes(ChunkedRawSize), + Path, + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } return Result; }; @@ -9174,7 +9396,9 @@ namespace { bool WipeTargetFolder, bool PostDownloadVerify, bool PrimeCacheOnly, - bool EnableScavenging) + bool EnableScavenging, + std::string_view IncludeWildcard, + std::string_view ExcludeWildcard) { ZEN_TRACE_CPU("DownloadFolder"); @@ -9220,8 +9444,15 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); - ChunkedFolderContent RemoteContent = - GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); + ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, + BuildId, + AllBuildParts, + IncludeWildcard, + ExcludeWildcard, + ChunkController, + PartContents, + BlockDescriptions, + LooseChunkHashes); const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; GetFolderContentStatistics LocalFolderScanStats; @@ -9232,22 +9463,21 @@ namespace { { if (IsDir(Path)) { - if (!WipeTargetFolder) + if (!ChunkController && !IsQuiet) { - if (!ChunkController) - { - ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); - ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{}); - } - - LocalContent = GetLocalContent(LocalFolderScanStats, - ChunkingStats, - Path, - ZenStateFilePath(ZenFolderPath), - *ChunkController, - RemoteContent.Paths, - LocalFolderContent); + ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); + ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{}); } + + LocalContent = GetLocalContent(LocalFolderScanStats, + ChunkingStats, + Path, + ZenStateFilePath(ZenFolderPath), + *ChunkController, + RemoteContent.Paths, + IncludeWildcard, + ExcludeWildcard, + LocalFolderContent); } else { @@ -9258,7 +9488,6 @@ namespace { { return; } - auto CompareContent = [](const ChunkedFolderContent& Lhs, const ChunkedFolderContent& Rhs) { tsl::robin_map<std::string, size_t> RhsPathToIndex; const size_t RhsPathCount = Rhs.Paths.size(); @@ -9301,16 +9530,22 @@ namespace { return true; }; - if (CompareContent(RemoteContent, LocalContent)) + if (CompareContent(RemoteContent, LocalContent) && !WipeTargetFolder) { - ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", + NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + } Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path); CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + } AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); } @@ -9324,7 +9559,10 @@ namespace { uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); + if (!IsQuiet) + { + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); + } FolderContent LocalFolderState; DiskStatistics DiskStats; @@ -9371,7 +9609,10 @@ namespace { CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + if (!IsQuiet) + { + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + } AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); @@ -9388,30 +9629,33 @@ namespace { DownloadStats.DownloadedPartialBlockByteCount.load(); const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); - ZEN_CONSOLE( - "Downloaded build {}, parts:{} in {}\n" - " Download: {} ({}) {}bits/s\n" - " Write: {} ({}) {}B/s\n" - " Clean: {}\n" - " Finalize: {}\n" - " Verify: {}", - BuildId, - BuildPartString.ToView(), - NiceTimeSpanMs(DownloadTimeMs), + if (!IsQuiet) + { + ZEN_CONSOLE( + "Downloaded build {}, parts:{} in {}\n" + " Download: {} ({}) {}bits/s\n" + " Write: {} ({}) {}B/s\n" + " Clean: {}\n" + " Finalize: {}\n" + " Verify: {}", + BuildId, + BuildPartString.ToView(), + NiceTimeSpanMs(DownloadTimeMs), - DownloadCount, - NiceBytes(DownloadByteCount), - NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), + DownloadCount, + NiceBytes(DownloadByteCount), + NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - DiskStats.WriteCount.load(), - NiceBytes(DiskStats.WriteByteCount.load()), - NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())), + DiskStats.WriteCount.load(), + NiceBytes(DiskStats.WriteByteCount.load()), + NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())), - NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), + NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), - NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), + NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), - NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); + NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); + } } } if (PrimeCacheOnly) @@ -9419,13 +9663,16 @@ namespace { if (Storage.BuildCacheStorage) { Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { - if (Remaining == 0) + if (!IsQuiet) { - ZEN_CONSOLE("Build cache upload complete"); - } - else - { - ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); + if (Remaining == 0) + { + ZEN_CONSOLE("Build cache upload complete"); + } + else + { + ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); + } } return !AbortFlag; }); @@ -9440,6 +9687,87 @@ namespace { } } + void ListBuild(StorageInstance& Storage, + const Oid& BuildId, + const std::vector<Oid>& BuildPartIds, + std::span<const std::string> BuildPartNames, + std::string_view IncludeWildcard, + std::string_view ExcludeWildcard) + { + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + std::vector<std::pair<Oid, std::string>> AllBuildParts = + ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + + Stopwatch GetBuildPartTimer; + + for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) + { + const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; + const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); + + if (!IsQuiet) + { + ZEN_CONSOLE("{}Part: {} ('{}'):\n", + BuildPartIndex > 0 ? "\n" : "", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + } + + std::vector<std::filesystem::path> Paths; + std::vector<IoHash> RawHashes; + std::vector<uint64_t> RawSizes; + std::vector<uint32_t> Attributes; + + SourcePlatform Platform; + std::vector<IoHash> SequenceRawHashes; + std::vector<uint32_t> ChunkCounts; + std::vector<uint32_t> AbsoluteChunkOrders; + std::vector<IoHash> LooseChunkHashes; + std::vector<uint64_t> LooseChunkRawSizes; + std::vector<IoHash> BlockRawHashes; + + ReadBuildContentFromCompactBinary(BuildPartManifest, + Platform, + Paths, + RawHashes, + RawSizes, + Attributes, + SequenceRawHashes, + ChunkCounts, + AbsoluteChunkOrders, + LooseChunkHashes, + LooseChunkRawSizes, + BlockRawHashes); + + std::vector<size_t> Order(Paths.size()); + std::iota(Order.begin(), Order.end(), 0); + + std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { + const std::filesystem::path& LhsPath = Paths[Lhs]; + const std::filesystem::path& RhsPath = Paths[Rhs]; + return LhsPath < RhsPath; + }); + + for (size_t Index : Order) + { + const std::filesystem::path& Path = Paths[Index]; + if (IncludePath(IncludeWildcard, ExcludeWildcard, Path)) + { + const IoHash& RawHash = RawHashes[Index]; + const uint64_t RawSize = RawSizes[Index]; + const uint32_t Attribute = Attributes[Index]; + ZEN_UNUSED(Attribute); + + ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); + } + } + } + } + void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) { ZEN_TRACE_CPU("DiffFolders"); @@ -9772,6 +10100,7 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_LogProgress), "<logprogress>"); Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); }; auto AddWorkerOptions = [this](cxxopts::Options& Ops) { @@ -9791,6 +10120,23 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_ZenFolderPath), "<boostworkers>"); }; + + auto AddWildcardOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "wildcard", + "Windows style wildcard (using * and ?) to match file paths to include", + cxxopts::value(m_IncludeWildcard), + "<wildcard>"); + + Ops.add_option("", + "", + "exclude-wildcard", + "Windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter", + cxxopts::value(m_ExcludeWildcard), + "<excludewildcard>"); + }; + m_Options.add_option("", "v", "verb", @@ -9925,6 +10271,8 @@ BuildsCommand::BuildsCommand() AddCacheOptions(m_DownloadOptions); AddZenFolderOptions(m_DownloadOptions); AddWorkerOptions(m_DownloadOptions); + AddWildcardOptions(m_DownloadOptions); + m_DownloadOptions.add_option("cache", "", "cache-prime-only", @@ -9975,6 +10323,37 @@ BuildsCommand::BuildsCommand() m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); m_DownloadOptions.positional_help("local-path build-id build-part-name"); + // ls + AddSystemOptions(m_LsOptions); + AddCloudOptions(m_LsOptions); + AddFileOptions(m_LsOptions); + AddOutputOptions(m_LsOptions); + AddCacheOptions(m_LsOptions); + AddZenFolderOptions(m_LsOptions); + AddWorkerOptions(m_LsOptions); + AddWildcardOptions(m_LsOptions); + + m_LsOptions.add_options()("h,help", "Print help"); + m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + m_LsOptions.add_option( + "", + "", + "build-part-id", + "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", + cxxopts::value(m_BuildPartIds), + "<id>"); + m_LsOptions.add_option("", + "", + "build-part-name", + "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " + "all parts will be downloaded", + cxxopts::value(m_BuildPartNames), + "<name>"); + + m_LsOptions.parse_positional({"build-id", "wildcard"}); + m_LsOptions.positional_help("build-id wildcard"); + + // diff AddOutputOptions(m_DiffOptions); AddWorkerOptions(m_DiffOptions); m_DiffOptions.add_options()("h,help", "Print help"); @@ -9989,6 +10368,7 @@ BuildsCommand::BuildsCommand() m_DiffOptions.parse_positional({"local-path", "compare-path"}); m_DiffOptions.positional_help("local-path compare-path"); + // test AddSystemOptions(m_TestOptions); AddCloudOptions(m_TestOptions); AddFileOptions(m_TestOptions); @@ -10018,6 +10398,7 @@ BuildsCommand::BuildsCommand() m_TestOptions.parse_positional({"local-path"}); m_TestOptions.positional_help("local-path"); + // fetch-blob AddSystemOptions(m_FetchBlobOptions); AddCloudOptions(m_FetchBlobOptions); AddFileOptions(m_FetchBlobOptions); @@ -10045,6 +10426,7 @@ BuildsCommand::BuildsCommand() m_AbortOptions.parse_positional({"process-id"}); m_AbortOptions.positional_help("process-id"); + // validate-part AddSystemOptions(m_ValidateBuildPartOptions); AddCloudOptions(m_ValidateBuildPartOptions); AddFileOptions(m_ValidateBuildPartOptions); @@ -10068,6 +10450,7 @@ BuildsCommand::BuildsCommand() m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"}); m_ValidateBuildPartOptions.positional_help("build-id build-part-id"); + // multi-test-download AddSystemOptions(m_MultiTestDownloadOptions); AddCloudOptions(m_MultiTestDownloadOptions); AddFileOptions(m_MultiTestDownloadOptions); @@ -10191,13 +10574,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_EncryptionKey.empty()) { m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; - ZEN_CONSOLE("Warning: Using default encryption key"); + if (!IsQuiet) + { + ZEN_CONSOLE("Warning: Using default encryption key"); + } } if (m_EncryptionIV.empty()) { m_EncryptionIV = "0123456789abcdef"; - ZEN_CONSOLE("Warning: Using default encryption initialization vector"); + if (!IsQuiet) + { + ZEN_CONSOLE("Warning: Using default encryption initialization vector"); + } } AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth", @@ -10299,7 +10688,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) { const std::string& CloudHost = m_OverrideHost.empty() ? m_Host : m_OverrideHost; - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost); + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost, IsQuiet); } if (!ClientSettings.AccessTokenProvider) @@ -10310,15 +10699,36 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseOutputOptions = [&]() { + if (m_Verbose && m_Quiet) + { + throw std::runtime_error("--verbose option is not compatible with --quiet option"); + } IsVerbose = m_Verbose; + IsQuiet = m_Quiet; if (m_LogProgress) { + if (IsQuiet) + { + throw std::runtime_error("--quiet option is not compatible with --log-progress option"); + } ProgressMode = ProgressBar::Mode::Log; } - else if (m_Verbose || m_PlainProgress) + if (m_PlainProgress) { + if (IsQuiet) + { + throw std::runtime_error("--quiet option is not compatible with --plain-progress option"); + } ProgressMode = ProgressBar::Mode::Plain; } + else if (m_Verbose) + { + ProgressMode = ProgressBar::Mode::Plain; + } + else if (IsQuiet) + { + ProgressMode = ProgressBar::Mode::Quiet; + } else { ProgressMode = ProgressBar::Mode::Pretty; @@ -10375,9 +10785,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON)); if (!ServerInfoResponse.IsSuccess()) { - throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'", - m_Host, - ServerInfoResponse.ErrorMessage(""))); + ServerInfoResponse.ThrowError(fmt::format("Failed to get list of servers from discovery url '{}'", m_Host)); } std::string_view JsonResponse = ServerInfoResponse.AsText(); @@ -10487,7 +10895,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); } - if (m_ZenCacheHost.empty()) + if (m_ZenCacheHost.empty() && !IsQuiet) { ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); } @@ -10576,10 +10984,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName; } - ZEN_CONSOLE("Remote: {}", StorageDescription); - if (!Result.CacheName.empty()) + if (!IsQuiet) { - ZEN_CONSOLE("Cache : {}", CacheDescription); + ZEN_CONSOLE("Remote: {}", StorageDescription); + if (!Result.CacheName.empty()) + { + ZEN_CONSOLE("Cache : {}", CacheDescription); + } } return Result; }; @@ -10592,6 +11003,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) MakeSafeAbsolutePathÍnPlace(m_Path); }; + auto ParseFileFilters = [&]() { + for (auto It = begin(m_IncludeWildcard); It != end(m_IncludeWildcard); It++) + { + if (*It == '\\') + { + *It = '/'; + } + } + + for (auto It = begin(m_ExcludeWildcard); It != end(m_ExcludeWildcard); It++) + { + if (*It == '\\') + { + *It = '/'; + } + } + }; + auto ParseDiffPath = [&]() { if (m_DiffPath.empty()) { @@ -10740,10 +11169,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_ListResultPath.empty()) { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } } BuildStorage::Statistics StorageStats; @@ -10768,7 +11200,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); if (m_ListResultPath.empty()) { ExtendableStringBuilder<1024> SB; @@ -10801,10 +11233,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_ListResultPath.empty()) { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } } CbObject QueryObject; if (m_ListQueryPath.empty()) @@ -10859,7 +11294,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); if (m_ListResultPath.empty()) { ExtendableStringBuilder<1024> SB; @@ -10886,7 +11321,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } ZenState InstanceState; @@ -10955,34 +11396,44 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (false) { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.StorageName, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); + if (!IsQuiet) + { + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Storage.StorageName, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); + } } return AbortFlag ? 11 : 0; } if (SubOption == &m_DownloadOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } ZenState InstanceState; ParsePath(); + ParseFileFilters(); if (m_ZenFolderPath.empty()) { @@ -11020,6 +11471,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::vector<Oid> BuildPartIds = ParseBuildPartIds(); std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + // TODO: Add file filters DownloadFolder(Storage, BuildId, BuildPartIds, @@ -11032,10 +11484,52 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Clean, m_PostDownloadVerify, m_PrimeCacheOnly, - m_EnableScavenging); + m_EnableScavenging, + m_IncludeWildcard, + m_ExcludeWildcard); + + return AbortFlag ? 11 : 0; + } + + if (SubOption == &m_LsOptions) + { + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } + + ZenState InstanceState; + + ParseFileFilters(); + + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); + + const Oid BuildId = ParseBuildId(); + + std::vector<Oid> BuildPartIds = ParseBuildPartIds(); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + + ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard); return AbortFlag ? 11 : 0; } + if (SubOption == &m_DiffOptions) { ParsePath(); @@ -11047,7 +11541,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -11084,16 +11584,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { return 11; } - ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", - BlobHash, - CompressedSize, - DecompressedSize); + if (!IsQuiet) + { + ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", + BlobHash, + CompressedSize, + DecompressedSize); + } return 0; } if (SubOption == &m_ValidateBuildPartOptions) { - ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); + } ZenState InstanceState; @@ -11181,15 +11690,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildIdString == m_BuildIds.front(), true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Download cancelled"); return 11; } - ZEN_CONSOLE("\n"); + if (!IsQuiet) + { + ZEN_CONSOLE("\n"); + } + } + if (!IsQuiet) + { + ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } - ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); return 0; } @@ -11351,7 +11868,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) true, true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -11375,7 +11894,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) false, true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -11493,7 +12014,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) false, true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -11545,7 +12068,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) false, true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -11565,7 +12090,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) false, true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -11585,7 +12112,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) false, true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -11605,7 +12134,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) false, true, false, - m_EnableScavenging); + m_EnableScavenging, + ""sv, + ""sv); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -11633,6 +12164,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 3; } } + catch (const HttpClientError& HttpEx) + { + ZEN_CONSOLE("Operation failed: {}", HttpEx.what()); + return HttpEx.m_Error != 0 ? HttpEx.m_Error : (int)HttpEx.m_ResponseCode; + } catch (const std::exception& Ex) { ZEN_ERROR("{}", Ex.what()); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index b3466c0d3..d057d24ac 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -32,6 +32,7 @@ private: bool m_Verbose = false; bool m_BoostWorkerThreads = false; bool m_UseSparseFiles = true; + bool m_Quiet = false; std::filesystem::path m_ZenFolderPath; @@ -107,6 +108,10 @@ private: bool m_PostDownloadVerify = false; bool m_EnableScavenging = true; + cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"}; + std::string m_IncludeWildcard; + std::string m_ExcludeWildcard; + cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; std::filesystem::path m_DiffPath; bool m_OnlyChunked = false; @@ -127,7 +132,7 @@ private: cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; std::vector<std::string> m_BuildIds; - cxxopts::Options* m_SubCommands[12] = {&m_ListNamespacesOptions, + cxxopts::Options* m_SubCommands[13] = {&m_ListNamespacesOptions, &m_ListOptions, &m_UploadOptions, &m_DownloadOptions, @@ -135,6 +140,7 @@ private: &m_ResumeOptions, &m_AbortOptions, &m_DiffOptions, + &m_LsOptions, &m_FetchBlobOptions, &m_ValidateBuildPartOptions, &m_TestOptions, diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp index 469dddf55..5bc8a8ed8 100644 --- a/src/zen/cmds/print_cmd.cpp +++ b/src/zen/cmds/print_cmd.cpp @@ -15,25 +15,41 @@ using namespace std::literals; namespace zen { static void -PrintCbObject(CbObject Object) +PrintCbObject(CbObject Object, bool AddTypeComment) { ExtendableStringBuilder<1024> ObjStr; - CompactBinaryToJson(Object, ObjStr); + CompactBinaryToJson(Object, ObjStr, AddTypeComment); ZEN_CONSOLE("{}", ObjStr); } static void -PrintCompactBinary(IoBuffer Data) +PrintCompactBinary(IoBuffer Data, bool AddTypeComment) { ExtendableStringBuilder<1024> StreamString; - CompactBinaryToJson(Data.GetView(), StreamString); + CompactBinaryToJson(Data.GetView(), StreamString, AddTypeComment); ZEN_CONSOLE("{}", StreamString); } +static CbValidateError +MakeFilteredResult(CbValidateError Result) +{ + CbValidateError FilteredResult = Result; + EnumRemoveFlags(FilteredResult, + CbValidateError::InvalidString | CbValidateError::InvalidInteger | CbValidateError::InvalidFloat | + CbValidateError::NonUniformObject | CbValidateError::NonUniformArray | CbValidateError::Padding); + return FilteredResult; +} + PrintCommand::PrintCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "s", "source", "Object payload file (use '-' to read from STDIN)", cxxopts::value(m_Filename), "<file name>"); + m_Options.add_option("", + "", + "show-type-info", + "Add type info annotation to compact binary objects", + cxxopts::value(m_ShowCbObjectTypeInfo), + "<showtypes>"); m_Options.parse_positional({"source"}); } @@ -62,6 +78,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { + MakeSafeAbsolutePathÍnPlace(m_Filename); Fc = ReadFile(m_Filename); } @@ -126,22 +143,59 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("---8<---"); - PrintCbObject(Object); - } - else if (CbValidateError Result = ValidateCompactBinary(Data, - CbValidateMode::Default | CbValidateMode::Names | CbValidateMode::Format | - CbValidateMode::Package | CbValidateMode::PackageHash); - Result == CbValidateError::None) - { - PrintCompactBinary(Data); + CbValidateError Result = ValidateCompactBinary(Object.GetView(), CbValidateMode::All); + + CbValidateError FilteredResult = MakeFilteredResult(Result); + + if (FilteredResult == CbValidateError::None && FilteredResult != Result) + { + ZEN_WARN( + "Object in package message file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')", + m_Filename, + uint32_t(Result), + ToString(Result)); + } + + if (FilteredResult != CbValidateError::None) + { + ZEN_ERROR("Object in package message file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", + m_Filename, + uint32_t(FilteredResult), + ToString(FilteredResult)); + return 1; + } + else + { + PrintCbObject(Object, m_ShowCbObjectTypeInfo); + } } else { - ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x})", m_Filename, uint32_t(Result)); + CbValidateError Result = ValidateCompactBinary(Data, CbValidateMode::All); - return 1; - } + CbValidateError FilteredResult = MakeFilteredResult(Result); + + if (FilteredResult == CbValidateError::None && FilteredResult != Result) + { + ZEN_WARN("Data in file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')", + m_Filename, + uint32_t(Result), + ToString(Result)); + } + if (FilteredResult != CbValidateError::None) + { + ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", + m_Filename, + uint32_t(FilteredResult), + ToString(FilteredResult)); + return 1; + } + else + { + PrintCompactBinary(Data, m_ShowCbObjectTypeInfo); + } + } return 0; } @@ -151,6 +205,12 @@ PrintPackageCommand::PrintPackageCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "s", "source", "Package payload file", cxxopts::value(m_Filename), "<file name>"); + m_Options.add_option("", + "", + "show-type-info", + "Add type info annotation to compact binary objects", + cxxopts::value(m_ShowCbObjectTypeInfo), + "<showtypes>"); m_Options.parse_positional({"source"}); } @@ -173,6 +233,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar if (m_Filename.empty()) throw std::runtime_error("No file specified"); + MakeSafeAbsolutePathÍnPlace(m_Filename); FileContents Fc = ReadFile(m_Filename); IoBuffer Data = Fc.Flatten(); CbPackage Package; @@ -182,7 +243,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar if (Ok) { ExtendableStringBuilder<1024> ObjStr; - CompactBinaryToJson(Package.GetObject(), ObjStr); + CompactBinaryToJson(Package.GetObject(), ObjStr, m_ShowCbObjectTypeInfo); ZEN_CONSOLE("{}", ObjStr); } else diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h index 4d6a492b7..80729901e 100644 --- a/src/zen/cmds/print_cmd.h +++ b/src/zen/cmds/print_cmd.h @@ -19,8 +19,9 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"print", "Print compact binary object"}; - std::string m_Filename; + cxxopts::Options m_Options{"print", "Print compact binary object"}; + std::filesystem::path m_Filename; + bool m_ShowCbObjectTypeInfo = false; }; /** Print Compact Binary Package @@ -36,8 +37,9 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; - std::string m_Filename; + cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; + std::filesystem::path m_Filename; + bool m_ShowCbObjectTypeInfo = false; }; } // namespace zen diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 58af0577e..f919edc87 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -149,6 +149,16 @@ namespace { } } + class AsyncJobError : public std::runtime_error + { + public: + using _Mybase = runtime_error; + + AsyncJobError(const std::string& Message, int ReturnCode) : _Mybase(Message), m_ReturnCode(ReturnCode) {} + + const int m_ReturnCode = 0; + }; + void ExecuteAsyncOperation(HttpClient& Http, std::string_view Url, IoBuffer&& Payload, bool PlainProgress) { signal(SIGINT, SignalCallbackHandler); @@ -252,13 +262,14 @@ namespace { if (Status == "Aborted") { std::string_view AbortReason = StatusObject["AbortReason"].AsString(); + int ReturnCode = StatusObject["ReturnCode"].AsInt32(-1); if (!AbortReason.empty()) { - throw std::runtime_error(std::string(AbortReason)); + throw AsyncJobError(std::string(AbortReason), ReturnCode); } else { - throw std::runtime_error("Aborted"); + throw AsyncJobError("Aborted", ReturnCode); } break; } @@ -1335,24 +1346,44 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ZEN_CONSOLE("Saving oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, m_HostName, TargetDescription); - if (m_Async) + try { - if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), - std::move(Payload), - HttpClient::Accept(ZenContentType::kJSON)); - Result) + if (m_Async) { - ZEN_CONSOLE("{}", Result.ToText()); + if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), + std::move(Payload), + HttpClient::Accept(ZenContentType::kJSON)); + Result) + { + ZEN_CONSOLE("{}", Result.ToText()); + } + else + { + Result.ThrowError("failed requesting loading oplog export"sv); + } } else { - Result.ThrowError("failed requesting loading oplog export"sv); - return 1; + ExecuteAsyncOperation(Http, + fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), + std::move(Payload), + m_PlainProgress); } } - else + catch (const HttpClientError& Ex) { - ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload), m_PlainProgress); + ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); + return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode; + } + catch (const AsyncJobError& Ex) + { + ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); + return Ex.m_ReturnCode; + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); + return 1; } return 0; } @@ -1668,24 +1699,44 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ZEN_CONSOLE("Loading oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, SourceDescription, m_HostName); - if (m_Async) + try { - if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), - std::move(Payload), - HttpClient::Accept(ZenContentType::kJSON)); - Result) + if (m_Async) { - ZEN_CONSOLE("{}", Result.ToText()); + if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), + std::move(Payload), + HttpClient::Accept(ZenContentType::kJSON)); + Result) + { + ZEN_CONSOLE("{}", Result.ToText()); + } + else + { + Result.ThrowError("failed requesting loading oplog import"sv); + } } else { - Result.ThrowError("failed requesting loading oplog import"sv); - return 1; + ExecuteAsyncOperation(Http, + fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), + std::move(Payload), + m_PlainProgress); } } - else + catch (const HttpClientError& Ex) + { + ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what()); + return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode; + } + catch (const AsyncJobError& Ex) { - ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload), m_PlainProgress); + ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); + return Ex.m_ReturnCode; + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what()); + return 1; } return 0; } diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 15988080a..df1d8b5c1 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -787,7 +787,8 @@ main(int argc, char** argv) std::string_view ThisArg(argv[j]); PassthroughArgV.push_back(std::string(ThisArg)); - const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos); + const bool NeedsQuotes = + (ThisArg.find(' ') != std::string_view::npos) && !(ThisArg.starts_with("\"") && ThisArg.ends_with("\"")); if (NeedsQuotes) { diff --git a/src/zen/zen.h b/src/zen/zen.h index 995bd13b6..40c745bc7 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -105,7 +105,8 @@ public: { Plain, Pretty, - Log + Log, + Quiet }; static void SetLogOperationName(Mode InMode, std::string_view Name); diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp index 9b06d4575..b22f2ec1f 100644 --- a/src/zencore/callstack.cpp +++ b/src/zencore/callstack.cpp @@ -179,19 +179,26 @@ FormatCallstack(const CallstackFrames* Callstack, StringBuilderBase& SB, std::st bool First = true; for (const std::string& Symbol : GetFrameSymbols(Callstack)) { - if (!First) + try { - SB.Append("\n"); - } - else - { - First = false; + if (!First) + { + SB.Append("\n"); + } + else + { + First = false; + } + if (!Prefix.empty()) + { + SB.Append(Prefix); + } + SB.Append(Symbol); } - if (!Prefix.empty()) + catch (const std::exception&) { - SB.Append(Prefix); + break; } - SB.Append(Symbol); } } diff --git a/src/zencore/compactbinaryfile.cpp b/src/zencore/compactbinaryfile.cpp index f2121a0bd..1526c21d5 100644 --- a/src/zencore/compactbinaryfile.cpp +++ b/src/zencore/compactbinaryfile.cpp @@ -19,7 +19,7 @@ LoadCompactBinaryObject(const std::filesystem::path& FilePath) IoBuffer ObjectBuffer = ObjectFile.Flatten(); - if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::All); Result == CbValidateError::None) + if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::Default); Result == CbValidateError::None) { CbObject Object = LoadCompactBinaryObject(ObjectBuffer); const IoHash WorkerId = IoHash::HashBuffer(ObjectBuffer); diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp index 68ed09549..02f22ba4d 100644 --- a/src/zencore/compactbinaryjson.cpp +++ b/src/zencore/compactbinaryjson.cpp @@ -22,7 +22,10 @@ namespace zen { class CbJsonWriter { public: - explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } + explicit CbJsonWriter(StringBuilderBase& InBuilder, bool AddTypeComment) : Builder(InBuilder), m_AddTypeComment(AddTypeComment) + { + NewLineAndIndent << LINE_TERMINATOR_ANSI; + } void BeginObject() { @@ -74,11 +77,32 @@ public: switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) { case CbFieldType::Null: + if (m_AddTypeComment) + { + Builder << "[Null] "; + } Builder << "null"sv; break; case CbFieldType::Object: + { + if (m_AddTypeComment) + { + Builder << "[Object] "; + } + BeginObject(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndObject(); + } + break; case CbFieldType::UniformObject: { + if (m_AddTypeComment) + { + Builder << "[UniformObject] "; + } BeginObject(); for (CbFieldView It : Field) { @@ -88,8 +112,25 @@ public: } break; case CbFieldType::Array: + { + if (m_AddTypeComment) + { + Builder << "[Array] "; + } + BeginArray(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndArray(); + } + break; case CbFieldType::UniformArray: { + if (m_AddTypeComment) + { + Builder << "[UniformArray] "; + } BeginArray(); for (CbFieldView It : Field) { @@ -99,19 +140,39 @@ public: } break; case CbFieldType::Binary: + if (m_AddTypeComment) + { + Builder << "[Binary] "; + } AppendBase64String(Accessor.AsBinary()); break; case CbFieldType::String: + if (m_AddTypeComment) + { + Builder << "[String] "; + } AppendQuotedString(Accessor.AsU8String()); break; case CbFieldType::IntegerPositive: + if (m_AddTypeComment) + { + Builder << "[IntegerPositive] "; + } Builder << Accessor.AsIntegerPositive(); break; case CbFieldType::IntegerNegative: + if (m_AddTypeComment) + { + Builder << "[IntegerNegative] "; + } Builder << Accessor.AsIntegerNegative(); break; case CbFieldType::Float32: { + if (m_AddTypeComment) + { + Builder << "[Float32] "; + } const float Value = Accessor.AsFloat32(); if (std::isfinite(Value)) { @@ -125,6 +186,10 @@ public: break; case CbFieldType::Float64: { + if (m_AddTypeComment) + { + Builder << "[Float64] "; + } const double Value = Accessor.AsFloat64(); if (std::isfinite(Value)) { @@ -137,14 +202,36 @@ public: } break; case CbFieldType::BoolFalse: + if (m_AddTypeComment) + { + Builder << "[BoolFalse] "; + } Builder << "false"sv; break; case CbFieldType::BoolTrue: + if (m_AddTypeComment) + { + Builder << "[BoolTrue] "; + } Builder << "true"sv; break; case CbFieldType::ObjectAttachment: + { + if (m_AddTypeComment) + { + Builder << "[ObjectAttachment] "; + } + Builder << '"'; + Accessor.AsAttachment().ToHexString(Builder); + Builder << '"'; + } + break; case CbFieldType::BinaryAttachment: { + if (m_AddTypeComment) + { + Builder << "[BinaryAttachment] "; + } Builder << '"'; Accessor.AsAttachment().ToHexString(Builder); Builder << '"'; @@ -152,6 +239,10 @@ public: break; case CbFieldType::Hash: { + if (m_AddTypeComment) + { + Builder << "[Hash] "; + } Builder << '"'; Accessor.AsHash().ToHexString(Builder); Builder << '"'; @@ -159,16 +250,28 @@ public: break; case CbFieldType::Uuid: { + if (m_AddTypeComment) + { + Builder << "[Uuid] "; + } Builder << '"'; Accessor.AsUuid().ToString(Builder); Builder << '"'; } break; case CbFieldType::DateTime: + if (m_AddTypeComment) + { + Builder << "[DateTime] "; + } Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; break; case CbFieldType::TimeSpan: { + if (m_AddTypeComment) + { + Builder << "[TimeSpan] "; + } const TimeSpan Span(Accessor.AsTimeSpanTicks()); if (Span.GetDays() == 0) { @@ -181,12 +284,20 @@ public: break; } case CbFieldType::ObjectId: + if (m_AddTypeComment) + { + Builder << "[ObjectId] "; + } Builder << '"'; Accessor.AsObjectId().ToString(Builder); Builder << '"'; break; case CbFieldType::CustomById: { + if (m_AddTypeComment) + { + Builder << "[CustomById] "; + } CbCustomById Custom = Accessor.AsCustomById(); Builder << "{ \"Id\": "; Builder << Custom.Id; @@ -197,6 +308,10 @@ public: } case CbFieldType::CustomByName: { + if (m_AddTypeComment) + { + Builder << "[CustomByName] "; + } CbCustomByName Custom = Accessor.AsCustomByName(); Builder << "{ \"Name\": "; AppendQuotedString(Custom.Name); @@ -299,29 +414,30 @@ private: private: StringBuilderBase& Builder; ExtendableStringBuilder<32> NewLineAndIndent; + const bool m_AddTypeComment; bool NeedsComma{false}; bool NeedsNewLine{false}; }; void -CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) +CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment) { - CbJsonWriter Writer(Builder); + CbJsonWriter Writer(Builder, AddTypeComment); Writer.WriteField(Object.AsFieldView()); } void CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) { - CbJsonWriter Writer(Builder); + CbJsonWriter Writer(Builder, /*AddTypeComment*/ false); Writer.WriteField(Array.AsFieldView()); } void -CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) +CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment) { std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); - CbJsonWriter Writer(InBuilder); + CbJsonWriter Writer(InBuilder, AddTypeComment); if (!Fields.empty()) { if (Fields.size() == 1) diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 833649b88..d7292f405 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -705,10 +705,11 @@ ToString(const CbValidateError Error) ExtendableStringBuilder<128> Out; - auto AppendFlag = [&, IsFirst = false](std::string_view FlagString) { + auto AppendFlag = [&, IsFirst = true](std::string_view FlagString) mutable { if (!IsFirst) Out.Append('|'); Out.Append(FlagString); + IsFirst = false; }; #define _ENUM_CASE(V) \ @@ -737,7 +738,11 @@ ToString(const CbValidateError Error) #undef _ENUM_CASE - return "Error"; + if (Out.Size() == 0) + { + return "Error"; + } + return Out.ToString(); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h index 0fdb56d67..82ca055ab 100644 --- a/src/zencore/include/zencore/compactbinary.h +++ b/src/zencore/include/zencore/compactbinary.h @@ -996,7 +996,7 @@ private: /** * Serialize a compact binary object to JSON. */ -ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder); +ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment = false); /** * Serialize a compact binary object to YAML. */ @@ -1520,7 +1520,7 @@ end(CbFieldView&) /** * Serialize serialized compact binary blob to JSON. It must be 0 to n fields with including type for each field */ -ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder); +ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment = false); /** * Serialize serialized compact binary blob to YAML. It must be 0 to n fields with including type for each field diff --git a/src/zencore/include/zencore/jobqueue.h b/src/zencore/include/zencore/jobqueue.h index d5ec6255a..470ed3fc6 100644 --- a/src/zencore/include/zencore/jobqueue.h +++ b/src/zencore/include/zencore/jobqueue.h @@ -28,6 +28,16 @@ public: virtual void ReportProgress(std::string_view CurrentOp, std::string_view Details, ptrdiff_t TotalCount, ptrdiff_t RemainingCount) = 0; }; +class JobError : public std::runtime_error +{ +public: + using _Mybase = runtime_error; + + JobError(const std::string& Message, int ReturnCode) : _Mybase(Message), m_ReturnCode(ReturnCode) {} + + const int m_ReturnCode = 0; +}; + class JobQueue { public: @@ -73,6 +83,7 @@ public: std::chrono::system_clock::time_point StartTime; std::chrono::system_clock::time_point EndTime; int WorkerThreadId; + int ReturnCode; }; // Will only respond once when status is Complete or Aborted diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp index 8e9a37a27..be9b39e7a 100644 --- a/src/zencore/iobuffer.cpp +++ b/src/zencore/iobuffer.cpp @@ -429,7 +429,25 @@ IoBufferExtendedCore::SetDeleteOnClose(bool DeleteOnClose) ////////////////////////////////////////////////////////////////////////// -RefPtr<IoBufferCore> IoBuffer::NullBufferCore(new IoBufferCore); +static IoBufferCore* +GetNullBufferCore() +{ + // This is safe from a threading standpoint since the first call is non-threaded (during static init) and for the following + // calls Core is never nullptr + // We do this workaround since we don't want to call new (IoBufferCore) at static initializers + // Calling new during static initialize causes problem with memtracing since the flags are not set up correctly yet + + static IoBufferCore NullBufferCore; + static IoBufferCore* Core = nullptr; + if (Core == nullptr) + { + Core = &NullBufferCore; + Core->AddRef(); // Make sure we never deallocate it as it is a static instance + } + return Core; +} + +RefPtr<IoBufferCore> IoBuffer::NullBufferCore(GetNullBufferCore()); IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize)) { diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp index b97484458..5d727b69c 100644 --- a/src/zencore/jobqueue.cpp +++ b/src/zencore/jobqueue.cpp @@ -50,6 +50,7 @@ public: JobClock::Tick StartTick; JobClock::Tick EndTick; int WorkerThreadId; + int ReturnCode; virtual bool IsCancelled() const override { return CancelFlag.load(); } virtual void ReportMessage(std::string_view Message) override { Queue->ReportMessage(Id, Message); } @@ -101,6 +102,7 @@ public: NewJob->StartTick = JobClock::Never(); NewJob->EndTick = JobClock::Never(); NewJob->WorkerThreadId = 0; + NewJob->ReturnCode = -1; ZEN_DEBUG("Scheduling background job {}:'{}'", NewJob->Id.Id, NewJob->Name); QueueLock.WithExclusiveLock([&]() { QueuedJobs.emplace_back(std::move(NewJob)); }); @@ -274,7 +276,8 @@ public: .CreateTime = JobClock::TimePointFromTick(Job.CreateTick), .StartTime = JobClock::TimePointFromTick(Job.StartTick), .EndTime = JobClock::TimePointFromTick(Job.EndTick), - .WorkerThreadId = Job.WorkerThreadId}; + .WorkerThreadId = Job.WorkerThreadId, + .ReturnCode = Job.ReturnCode}; }; std::optional<JobDetails> Result; @@ -365,6 +368,7 @@ public: ZEN_DEBUG("Executing background job {}:'{}'", CurrentJob->Id.Id, CurrentJob->Name); CurrentJob->Callback(*CurrentJob); ZEN_DEBUG("Completed background job {}:'{}'", CurrentJob->Id.Id, CurrentJob->Name); + CurrentJob->ReturnCode = 0; QueueLock.WithExclusiveLock([&]() { CurrentJob->EndTick = JobClock::Now(); CurrentJob->WorkerThreadId = 0; @@ -383,6 +387,22 @@ public: AbortedJobs.insert_or_assign(CurrentJob->Id.Id, std::move(CurrentJob)); }); } + catch (const JobError& Ex) + { + ZEN_DEBUG("Background job {}:'{}' failed. Reason: '{}'. Return code {}", + CurrentJob->Id.Id, + CurrentJob->Name, + Ex.what(), + Ex.m_ReturnCode); + QueueLock.WithExclusiveLock([&]() { + CurrentJob->State.AbortReason = Ex.what(); + CurrentJob->EndTick = JobClock::Now(); + CurrentJob->WorkerThreadId = 0; + CurrentJob->ReturnCode = Ex.m_ReturnCode; + RunningJobs.erase(CurrentJob->Id.Id); + AbortedJobs.insert_or_assign(CurrentJob->Id.Id, std::move(CurrentJob)); + }); + } catch (const std::exception& Ex) { ZEN_DEBUG("Background job {}:'{}' aborted. Reason: '{}'", CurrentJob->Id.Id, CurrentJob->Name, Ex.what()); diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp index fef5c28a4..b8ec85a4a 100644 --- a/src/zencore/thread.cpp +++ b/src/zencore/thread.cpp @@ -80,8 +80,10 @@ SetNameInternal(DWORD thread_id, const char* name) void SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) { - StringBuilder<256> ThreadNameZ; - ThreadNameZ << ThreadName; + constexpr std::string_view::size_type MaxThreadNameLength = 255; + std::string_view LimitedThreadName = ThreadName.substr(0, MaxThreadNameLength); + StringBuilder<MaxThreadNameLength + 1> ThreadNameZ; + ThreadNameZ << LimitedThreadName; const int ThreadId = GetCurrentThreadId(); #if ZEN_WITH_TRACE @@ -95,8 +97,8 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) if (SetThreadDescriptionFunc) { - WideStringBuilder<256> ThreadNameW; - Utf8ToWide(ThreadName, ThreadNameW); + WideStringBuilder<MaxThreadNameLength + 1> ThreadNameW; + Utf8ToWide(LimitedThreadName, ThreadNameW); SetThreadDescriptionFunc(::GetCurrentThread(), ThreadNameW.c_str()); } diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp index 82d28c0e3..51e06ae14 100644 --- a/src/zencore/zencore.cpp +++ b/src/zencore/zencore.cpp @@ -124,7 +124,14 @@ AssertImpl::ExecAssert(const char* Filename, int LineNumber, const char* Functio AssertImpl* AssertImpl = CurrentAssertImpl; while (AssertImpl) { - AssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg, Callstack); + try + { + AssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg, Callstack); + } + catch (const std::exception&) + { + // Just keep exception silent - we don't want exception thrown from assert callbacks + } AssertImpl = AssertImpl->NextAssertImpl; } ThrowAssertException(Filename, LineNumber, FunctionName, Msg, Callstack); diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index a2d323b5e..30a2bfc65 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -1628,12 +1628,13 @@ HttpClient::Response::ErrorMessage(std::string_view Prefix) const } else if (StatusCode != HttpResponseCode::ImATeapot && (int)StatusCode) { - return fmt::format("{}{}HTTP error {} {} ({})", + std::string TextResponse = ToText(); + return fmt::format("{}{}HTTP error {} {}{}", Prefix, Prefix.empty() ? ""sv : ": "sv, (int)StatusCode, zen::ToString(StatusCode), - ToText()); + TextResponse.empty() ? ""sv : fmt::format(" ({})", TextResponse)); } else { @@ -1646,7 +1647,7 @@ HttpClient::Response::ThrowError(std::string_view ErrorPrefix) { if (!IsSuccess()) { - throw std::runtime_error(ErrorMessage(ErrorPrefix)); + throw HttpClientError(ErrorMessage(ErrorPrefix), Error.has_value() ? Error.value().ErrorCode : 0, StatusCode); } } diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp index 39efe1d0c..62e1b77bc 100644 --- a/src/zenhttp/httpclientauth.cpp +++ b/src/zenhttp/httpclientauth.cpp @@ -89,14 +89,25 @@ namespace zen { namespace httpclientauth { static HttpClientAccessToken GetOidcTokenFromExe(const std::filesystem::path& OidcExecutablePath, std::string_view CloudHost, - bool Unattended) + bool Unattended, + bool Quiet) { Stopwatch Timer; CreateProcOptions ProcOptions; + if (Quiet) + { + ProcOptions.StdoutFile = std::filesystem::temp_directory_path() / fmt::format(".zen-auth-output-{}", Oid::NewOid()); + } const std::filesystem::path AuthTokenPath(std::filesystem::temp_directory_path() / fmt::format(".zen-auth-{}", Oid::NewOid())); - auto _ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); }); + auto _ = MakeGuard([AuthTokenPath, &ProcOptions]() { + RemoveFile(AuthTokenPath); + if (!ProcOptions.StdoutFile.empty()) + { + RemoveFile(ProcOptions.StdoutFile); + } + }); const std::string ProcArgs = fmt::format("{} --AuthConfigUrl {} --OutFile {} --Unattended={}", OidcExecutablePath, @@ -164,13 +175,15 @@ namespace zen { namespace httpclientauth { } std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath, - std::string_view CloudHost) + std::string_view CloudHost, + bool Quiet) { - HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, false); + HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ false, Quiet); if (InitialToken.IsValid()) { return [OidcExecutablePath = std::filesystem::path(OidcExecutablePath), CloudHost = std::string(CloudHost), + Quiet, InitialToken]() mutable { if (InitialToken.IsValid()) { @@ -178,7 +191,7 @@ namespace zen { namespace httpclientauth { InitialToken = {}; return Result; } - return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, true); + return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ true, Quiet); }; } return {}; diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index c991a71ea..50bd5b53a 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -57,6 +57,29 @@ struct HttpClientSettings uint8_t RetryCount = 0; }; +class HttpClientError : public std::runtime_error +{ +public: + using _Mybase = runtime_error; + + HttpClientError(const std::string& Message, int Error, HttpResponseCode ResponseCode) + : _Mybase(Message) + , m_Error(Error) + , m_ResponseCode(ResponseCode) + { + } + + HttpClientError(const char* Message, int Error, HttpResponseCode ResponseCode) + : _Mybase(Message) + , m_Error(Error) + , m_ResponseCode(ResponseCode) + { + } + + const int m_Error = 0; + const HttpResponseCode m_ResponseCode = HttpResponseCode::ImATeapot; +}; + class HttpClient { public: diff --git a/src/zenhttp/include/zenhttp/httpclientauth.h b/src/zenhttp/include/zenhttp/httpclientauth.h index 5b9b9d305..32d00f87f 100644 --- a/src/zenhttp/include/zenhttp/httpclientauth.h +++ b/src/zenhttp/include/zenhttp/httpclientauth.h @@ -27,7 +27,8 @@ namespace httpclientauth { std::function<HttpClientAccessToken()> CreateFromDefaultOpenIdProvider(AuthMgr& AuthManager); std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath, - std::string_view CloudHost); + std::string_view CloudHost, + bool Quiet); } // namespace httpclientauth } // namespace zen diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index 9d423ecbc..0b7848f79 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -576,7 +576,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint if (!MalformedAttachments.empty()) { - StringBuilder<1024> SB; + ExtendableStringBuilder<1024> SB; SB << (uint64_t)MalformedAttachments.size() << " malformed attachments in package message:\n"; for (const auto& It : MalformedAttachments) { diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 78d8ab68a..77ed87cb1 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -13,6 +13,7 @@ #include <zencore/iohash.h> #include <zencore/logging.h> #include <zencore/memoryview.h> +#include <zencore/scopeguard.h> #include <zencore/stream.h> #include <zencore/string.h> #include <zencore/testutils.h> @@ -22,6 +23,7 @@ #include <zenhttp/httpclient.h> #include <zenhttp/packageformat.h> #include <zenhttp/zenhttp.h> +#include <zenutil/buildstoragecache.h> #include <zenutil/cache/cache.h> #include <zenutil/cache/cacherequests.h> #include <zenutil/chunkrequests.h> @@ -37,6 +39,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <cpr/cpr.h> +#include <tsl/robin_set.h> #undef GetObject ZEN_THIRD_PARTY_INCLUDES_END @@ -554,9 +557,13 @@ namespace utils { return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(), .Port = Port, .Args = std::move(Args)}; } - static ZenConfig NewWithUpstream(uint16_t Port, uint16_t UpstreamPort) + static ZenConfig NewWithUpstream(uint16_t Port, uint16_t UpstreamPort, std::string Args = "") { - return New(Port, fmt::format("--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", UpstreamPort)); + return New(Port, + fmt::format("{}{}--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", + Args, + Args.length() > 0 ? " " : "", + UpstreamPort)); } static ZenConfig NewWithThreadedUpstreams(uint16_t NewPort, std::span<uint16_t> UpstreamPorts, bool Debug) @@ -1302,8 +1309,10 @@ TEST_CASE("zcache.rpc") std::string_view Namespace, std::string_view Bucket, size_t Num, - size_t PayloadSize = 1024, - size_t KeyOffset = 1) -> std::vector<CacheKey> { + size_t PayloadSize = 1024, + size_t KeyOffset = 1, + CachePolicy PutPolicy = CachePolicy::Default, + std::vector<CbPackage>* OutPackages = nullptr) -> std::vector<CacheKey> { std::vector<zen::CacheKey> OutKeys; for (uint32_t Key = 1; Key <= Num; ++Key) @@ -1313,7 +1322,7 @@ TEST_CASE("zcache.rpc") const zen::CacheKey CacheKey = zen::CacheKey::Create(Bucket, KeyHash); cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)}; - AppendCacheRecord(Request, CacheKey, PayloadSize, CachePolicy::Default); + AppendCacheRecord(Request, CacheKey, PayloadSize, PutPolicy); OutKeys.push_back(CacheKey); CbPackage Package; @@ -1325,6 +1334,10 @@ TEST_CASE("zcache.rpc") cpr::Body{(const char*)Body.GetData(), Body.GetSize()}); CHECK(Result.status_code == 200); + if (OutPackages) + { + OutPackages->emplace_back(std::move(Package)); + } } return OutKeys; @@ -1503,6 +1516,339 @@ TEST_CASE("zcache.rpc") } } + SUBCASE("policy - 'QueryLocal' on put allows overwrite with differing value when not limiting overwrites") + { + using namespace utils; + + ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber()); + ZenServerInstance UpstreamServer(TestEnv); + SpawnServer(UpstreamServer, UpstreamCfg); + + ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port); + ZenServerInstance LocalServer(TestEnv); + SpawnServer(LocalServer, LocalCfg); + + size_t PayloadSize = 1024; + std::string_view Namespace("ue4.ddc"sv); + std::string_view Bucket("mastodon"sv); + const size_t NumRecords = 4; + std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize); + + for (const zen::CacheKey& CacheKey : Keys) + { + cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)}; + AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Default); + + CbPackage Package; + CHECK(Request.Format(Package)); + + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); + cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, + cpr::Body{(const char*)Body.GetData(), Body.GetSize()}); + + CHECK(Result.status_code == 200); + cacherequests::PutCacheRecordsResult ParsedResult; + CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); + CHECK(!Response.IsNull()); + CHECK(ParsedResult.Parse(Response)); + for (bool ResponseSuccess : ParsedResult.Success) + { + CHECK(ResponseSuccess); + } + } + + auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) { + CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); + GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy); + + CHECK(Result.Result.Results.size() == Keys.size()); + + for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) + { + CHECK(Record); + const CacheKey& ExpectedKey = Keys[Index++]; + CHECK(Record->Key == ExpectedKey); + for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values) + { + CHECK(Value.RawSize == PayloadSize * 2); + } + } + }; + + // Check that the records are present and overwritten in the local server + CheckRecordCorrectness(LocalCfg); + + // Check that the records are present and overwritten in the upstream server + CheckRecordCorrectness(UpstreamCfg); + } + + SUBCASE("policy - 'QueryLocal' on put denies overwrite with differing value when limiting overwrites") + { + using namespace utils; + + ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber()); + ZenServerInstance UpstreamServer(TestEnv); + SpawnServer(UpstreamServer, UpstreamCfg); + + ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port, "--cache-bucket-limit-overwrites"); + ZenServerInstance LocalServer(TestEnv); + SpawnServer(LocalServer, LocalCfg); + + size_t PayloadSize = 1024; + std::string_view Namespace("ue4.ddc"sv); + std::string_view Bucket("mastodon"sv); + const size_t NumRecords = 4; + std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize); + + for (const zen::CacheKey& CacheKey : Keys) + { + cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)}; + AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Default); + + CbPackage Package; + CHECK(Request.Format(Package)); + + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); + cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, + cpr::Body{(const char*)Body.GetData(), Body.GetSize()}); + + CHECK(Result.status_code == 200); + cacherequests::PutCacheRecordsResult ParsedResult; + CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); + CHECK(!Response.IsNull()); + CHECK(ParsedResult.Parse(Response)); + for (bool ResponseSuccess : ParsedResult.Success) + { + CHECK(!ResponseSuccess); + } + } + + auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) { + CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); + GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy); + + CHECK(Result.Result.Results.size() == Keys.size()); + + for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) + { + CHECK(Record); + const CacheKey& ExpectedKey = Keys[Index++]; + CHECK(Record->Key == ExpectedKey); + for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values) + { + CHECK(Value.RawSize == PayloadSize); + } + } + }; + + // Check that the records are present and not overwritten in the local server + CheckRecordCorrectness(LocalCfg); + + // Check that the records are present and not overwritten in the upstream server + CheckRecordCorrectness(UpstreamCfg); + } + + SUBCASE("policy - no 'QueryLocal' on put allows overwrite with differing value when limiting overwrites") + { + using namespace utils; + + ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber()); + ZenServerInstance UpstreamServer(TestEnv); + SpawnServer(UpstreamServer, UpstreamCfg); + + ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port, "--cache-bucket-limit-overwrites"); + ZenServerInstance LocalServer(TestEnv); + SpawnServer(LocalServer, LocalCfg); + + size_t PayloadSize = 1024; + std::string_view Namespace("ue4.ddc"sv); + std::string_view Bucket("mastodon"sv); + const size_t NumRecords = 4; + std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize); + + for (const zen::CacheKey& CacheKey : Keys) + { + cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)}; + AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Store); + + CbPackage Package; + CHECK(Request.Format(Package)); + + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); + cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, + cpr::Body{(const char*)Body.GetData(), Body.GetSize()}); + + CHECK(Result.status_code == 200); + cacherequests::PutCacheRecordsResult ParsedResult; + CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); + CHECK(!Response.IsNull()); + CHECK(ParsedResult.Parse(Response)); + for (bool ResponseSuccess : ParsedResult.Success) + { + CHECK(ResponseSuccess); + } + } + + auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) { + CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); + GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy); + + CHECK(Result.Result.Results.size() == Keys.size()); + + for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) + { + CHECK(Record); + const CacheKey& ExpectedKey = Keys[Index++]; + CHECK(Record->Key == ExpectedKey); + for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values) + { + CHECK(Value.RawSize == PayloadSize * 2); + } + } + }; + + // Check that the records are present and overwritten in the local server + CheckRecordCorrectness(LocalCfg); + + // Check that the records are present and overwritten in the upstream server + CheckRecordCorrectness(UpstreamCfg); + } + + SUBCASE("policy - 'QueryLocal' on put allows overwrite with equivalent value when limiting overwrites") + { + using namespace utils; + + ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber()); + ZenServerInstance UpstreamServer(TestEnv); + SpawnServer(UpstreamServer, UpstreamCfg); + + ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port, "--cache-bucket-limit-overwrites"); + ZenServerInstance LocalServer(TestEnv); + SpawnServer(LocalServer, LocalCfg); + + size_t PayloadSize = 1024; + std::string_view Namespace("ue4.ddc"sv); + std::string_view Bucket("mastodon"sv); + const size_t NumRecords = 4; + std::vector<CbPackage> Packages; + std::vector<zen::CacheKey> Keys = + PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize, 1, CachePolicy::Default, &Packages); + + for (const CbPackage& Package : Packages) + { + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); + cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, + cpr::Body{(const char*)Body.GetData(), Body.GetSize()}); + + CHECK(Result.status_code == 200); + cacherequests::PutCacheRecordsResult ParsedResult; + CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); + CHECK(!Response.IsNull()); + CHECK(ParsedResult.Parse(Response)); + for (bool ResponseSuccess : ParsedResult.Success) + { + CHECK(ResponseSuccess); + } + } + + auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) { + CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); + GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy); + + CHECK(Result.Result.Results.size() == Keys.size()); + + for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) + { + CHECK(Record); + const CacheKey& ExpectedKey = Keys[Index++]; + CHECK(Record->Key == ExpectedKey); + for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values) + { + CHECK(Value.RawSize == PayloadSize); + } + } + }; + + // Check that the records are present and unchanged in the local server + CheckRecordCorrectness(LocalCfg); + + // Check that the records are present and unchanged in the upstream server + CheckRecordCorrectness(UpstreamCfg); + } + + // TODO: Propagation for rejected PUTs + // SUBCASE("policy - 'QueryLocal' on put denies overwrite with differing value when limiting overwrites but allows propagation to + // upstream") + // { + // using namespace utils; + + // ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber()); + // ZenServerInstance UpstreamServer(TestEnv); + // SpawnServer(UpstreamServer, UpstreamCfg); + + // ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port, + // "--cache-bucket-limit-overwrites"); ZenServerInstance LocalServer(TestEnv); SpawnServer(LocalServer, LocalCfg); + + // size_t PayloadSize = 1024; + // std::string_view Namespace("ue4.ddc"sv); + // std::string_view Bucket("mastodon"sv); + // const size_t NumRecords = 4; + // std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize, 1, + // CachePolicy::Local); + + // for (const zen::CacheKey& CacheKey : Keys) + // { + // cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)}; + // AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Default); + + // CbPackage Package; + // CHECK(Request.Format(Package)); + + // IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); + // cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)}, + // cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, + // cpr::Body{(const char*)Body.GetData(), Body.GetSize()}); + + // CHECK(Result.status_code == 200); + // cacherequests::PutCacheRecordsResult ParsedResult; + // CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); + // CHECK(!Response.IsNull()); + // CHECK(ParsedResult.Parse(Response)); + // for (bool ResponseSuccess : ParsedResult.Success) + // { + // CHECK(!ResponseSuccess); + // } + // } + + // auto CheckRecordCorrectness = [&](const ZenConfig& Cfg, size_t ExpectedPayloadSize) { + // CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); + // GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy); + + // CHECK(Result.Result.Results.size() == Keys.size()); + + // for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) + // { + // CHECK(Record); + // const CacheKey& ExpectedKey = Keys[Index++]; + // CHECK(Record->Key == ExpectedKey); + // for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values) + // { + // CHECK(Value.RawSize == ExpectedPayloadSize); + // } + // } + // }; + + // // Check that the records are present and not overwritten in the local server + // CheckRecordCorrectness(LocalCfg, PayloadSize); + + // // Check that the records are present and are the newer size in the upstream server + // CheckRecordCorrectness(UpstreamCfg, PayloadSize*2); + // } + SUBCASE("RpcAcceptOptions") { using namespace utils; @@ -3817,6 +4163,478 @@ TEST_CASE("workspaces.share") CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)).StatusCode == HttpResponseCode::NotFound); } +TEST_CASE("buildstore.blobs") +{ + std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir(); + auto _ = MakeGuard([&SystemRootPath]() { DeleteDirectories(SystemRootPath); }); + + std::string_view Namespace = "ns"sv; + std::string_view Bucket = "bkt"sv; + Oid BuildId = Oid::NewOid(); + + std::vector<IoHash> CompressedBlobsHashes; + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + HttpClient::Response Result = + Client.Put(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, CompressedBlobsHashes.back()), Payload); + CHECK(Result); + } + + for (const IoHash& RawHash : CompressedBlobsHashes) + { + HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + CHECK(Result); + IoBuffer Payload = Result.ResponsePayload; + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + for (const IoHash& RawHash : CompressedBlobsHashes) + { + HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + CHECK(Result); + IoBuffer Payload = Result.ResponsePayload; + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + HttpClient::Response Result = + Client.Put(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, CompressedBlobsHashes.back()), Payload); + CHECK(Result); + } + } + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + for (const IoHash& RawHash : CompressedBlobsHashes) + { + HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + CHECK(Result); + IoBuffer Payload = Result.ResponsePayload; + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } +} + +namespace { + CbObject MakeMetadata(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues) + { + CbObjectWriter Writer; + Writer.AddHash("rawHash"sv, BlobHash); + Writer.BeginObject("values"); + { + for (const auto& V : KeyValues) + { + Writer.AddString(V.first, V.second); + } + } + Writer.EndObject(); // values + return Writer.Save(); + }; + +} // namespace + +TEST_CASE("buildstore.metadata") +{ + std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir(); + auto _ = MakeGuard([&SystemRootPath]() { DeleteDirectories(SystemRootPath); }); + + std::string_view Namespace = "ns"sv; + std::string_view Bucket = "bkt"sv; + Oid BuildId = Oid::NewOid(); + + std::vector<IoHash> BlobHashes; + std::vector<CbObject> Metadatas; + std::vector<IoHash> MetadataHashes; + + auto GetMetadatas = + [](HttpClient& Client, std::string_view Namespace, std::string_view Bucket, const Oid& BuildId, std::vector<IoHash> BlobHashes) { + CbObjectWriter Request; + + Request.BeginArray("blobHashes"sv); + for (const IoHash& BlobHash : BlobHashes) + { + Request.AddHash(BlobHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/getBlobMetadata", Namespace, Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + CHECK(Result); + + std::vector<CbObject> ResultMetadatas; + + CbPackage ResponsePackage = ParsePackageMessage(Result.ResponsePayload); + CbObject ResponseObject = ResponsePackage.GetObject(); + + CbArrayView BlobHashArray = ResponseObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadatasArray = ResponseObject["metadatas"sv].AsArrayView(); + ResultMetadatas.reserve(MetadatasArray.Num()); + auto BlobHashesIt = BlobHashes.begin(); + auto BlobHashArrayIt = begin(BlobHashArray); + auto MetadataArrayIt = begin(MetadatasArray); + while (MetadataArrayIt != end(MetadatasArray)) + { + const IoHash BlobHash = (*BlobHashArrayIt).AsHash(); + while (BlobHash != *BlobHashesIt) + { + ZEN_ASSERT(BlobHashesIt != BlobHashes.end()); + BlobHashesIt++; + } + + ZEN_ASSERT(BlobHash == *BlobHashesIt); + + const IoHash MetaHash = (*MetadataArrayIt).AsAttachment(); + const CbAttachment* MetaAttachment = ResponsePackage.FindAttachment(MetaHash); + ZEN_ASSERT(MetaAttachment); + + CbObject Metadata = MetaAttachment->AsObject(); + ResultMetadatas.emplace_back(std::move(Metadata)); + + BlobHashArrayIt++; + MetadataArrayIt++; + BlobHashesIt++; + } + return ResultMetadatas; + }; + + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + const size_t BlobCount = 5; + + for (size_t I = 0; I < BlobCount; I++) + { + BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I))); + Metadatas.push_back(MakeMetadata(BlobHashes.back(), {{"index", fmt::format("{}", I)}})); + MetadataHashes.push_back(IoHash::HashBuffer(Metadatas.back().GetBuffer().AsIoBuffer())); + } + + { + CbPackage RequestPackage; + std::vector<CbAttachment> Attachments; + tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter RequestWriter; + RequestWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobHashes.size(); BlockHashIndex++) + { + RequestWriter.AddHash(BlobHashes[BlockHashIndex]); + } + RequestWriter.EndArray(); // blobHashes + + RequestWriter.BeginArray("metadatas"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobHashes.size(); BlockHashIndex++) + { + const IoHash ObjectHash = Metadatas[BlockHashIndex].GetHash(); + RequestWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(Metadatas[BlockHashIndex], ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + + RequestWriter.EndArray(); // metadatas + + RequestPackage.SetObject(RequestWriter.Save()); + } + RequestPackage.AddAttachments(Attachments); + + CompositeBuffer RpcRequestBuffer = FormatPackageMessageBuffer(RequestPackage); + + HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/putBlobMetadata", Namespace, Bucket, BuildId), + RpcRequestBuffer, + ZenContentType::kCbPackage); + CHECK(Result); + } + + { + std::vector<CbObject> ResultMetadatas = GetMetadatas(Client, Namespace, Bucket, BuildId, BlobHashes); + + for (size_t Index = 0; Index < MetadataHashes.size(); Index++) + { + const IoHash& ExpectedHash = MetadataHashes[Index]; + IoHash Hash = IoHash::HashBuffer(ResultMetadatas[Index].GetBuffer().AsIoBuffer()); + CHECK_EQ(ExpectedHash, Hash); + } + } + } + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + std::vector<CbObject> ResultMetadatas = GetMetadatas(Client, Namespace, Bucket, BuildId, BlobHashes); + + for (size_t Index = 0; Index < MetadataHashes.size(); Index++) + { + const IoHash& ExpectedHash = MetadataHashes[Index]; + IoHash Hash = IoHash::HashBuffer(ResultMetadatas[Index].GetBuffer().AsIoBuffer()); + CHECK_EQ(ExpectedHash, Hash); + } + } +} + +TEST_CASE("buildstore.cache") +{ + std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir(); + std::filesystem::path TempDir = TestEnv.CreateNewTestDir(); + auto _ = MakeGuard([&SystemRootPath, &TempDir]() { + DeleteDirectories(SystemRootPath); + DeleteDirectories(TempDir); + }); + + std::string_view Namespace = "ns"sv; + std::string_view Bucket = "bkt"sv; + Oid BuildId = Oid::NewOid(); + + std::vector<IoHash> BlobHashes; + std::vector<CbObject> Metadatas; + std::vector<IoHash> MetadataHashes; + + const size_t BlobCount = 5; + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri()); + + BuildStorageCache::Statistics Stats; + std::unique_ptr<BuildStorageCache> Cache(CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false)); + + { + IoHash NoneBlob = IoHash::HashBuffer("data", 4); + std::vector<BuildStorageCache::BlobExistsResult> NoneExists = Cache->BlobsExists(BuildId, std::vector<IoHash>{NoneBlob}); + CHECK(NoneExists.size() == 1); + CHECK(!NoneExists[0].HasBody); + CHECK(!NoneExists[0].HasMetadata); + } + + for (size_t I = 0; I < BlobCount; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + BlobHashes.push_back(CompressedBlob.DecodeRawHash()); + Cache->PutBuildBlob(BuildId, BlobHashes.back(), ZenContentType::kCompressedBinary, CompressedBlob.GetCompressed()); + } + + Cache->Flush(500); + Cache = CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false); + + { + std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); + CHECK(Exists.size() == BlobHashes.size()); + for (size_t I = 0; I < BlobCount; I++) + { + CHECK(Exists[I].HasBody); + CHECK(!Exists[I].HasMetadata); + } + + std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); + CHECK_EQ(0, FetchedMetadatas.size()); + } + + { + for (size_t I = 0; I < BlobCount; I++) + { + IoBuffer BuildBlob = Cache->GetBuildBlob(BuildId, BlobHashes[I]); + CHECK(BuildBlob); + CHECK_EQ(BlobHashes[I], + IoHash::HashBuffer(CompressedBuffer::FromCompressedNoValidate(std::move(BuildBlob)).Decompress().AsIoBuffer())); + } + } + + { + for (size_t I = 0; I < BlobCount; I++) + { + CbObject Metadata = MakeMetadata(BlobHashes[I], + {{"key", fmt::format("{}", I)}, + {"key_plus_one", fmt::format("{}", I + 1)}, + {"block_hash", fmt::format("{}", BlobHashes[I])}}); + Metadatas.push_back(Metadata); + MetadataHashes.push_back(IoHash::HashBuffer(Metadata.GetBuffer().AsIoBuffer())); + } + Cache->PutBlobMetadatas(BuildId, BlobHashes, Metadatas); + } + + Cache->Flush(500); + Cache = CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false); + + { + std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); + CHECK(Exists.size() == BlobHashes.size()); + for (size_t I = 0; I < BlobCount; I++) + { + CHECK(Exists[I].HasBody); + CHECK(Exists[I].HasMetadata); + } + + std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); + CHECK_EQ(BlobCount, FetchedMetadatas.size()); + + for (size_t I = 0; I < BlobCount; I++) + { + CHECK_EQ(MetadataHashes[I], IoHash::HashBuffer(FetchedMetadatas[I].GetBuffer().AsIoBuffer())); + } + } + + for (size_t I = 0; I < BlobCount; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + BlobHashes.push_back(CompressedBlob.DecodeRawHash()); + Cache->PutBuildBlob(BuildId, BlobHashes.back(), ZenContentType::kCompressedBinary, CompressedBlob.GetCompressed()); + } + + Cache->Flush(500); + Cache = CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false); + + { + std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); + CHECK(Exists.size() == BlobHashes.size()); + for (size_t I = 0; I < BlobCount * 2; I++) + { + CHECK(Exists[I].HasBody); + CHECK_EQ(I < BlobCount, Exists[I].HasMetadata); + } + + std::vector<CbObject> MetaDatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); + CHECK_EQ(BlobCount, MetaDatas.size()); + + std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); + CHECK_EQ(BlobCount, FetchedMetadatas.size()); + + for (size_t I = 0; I < BlobCount; I++) + { + CHECK_EQ(MetadataHashes[I], IoHash::HashBuffer(FetchedMetadatas[I].GetBuffer().AsIoBuffer())); + } + } + } + + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri()); + + BuildStorageCache::Statistics Stats; + std::unique_ptr<BuildStorageCache> Cache(CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false)); + + std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); + CHECK(Exists.size() == BlobHashes.size()); + for (size_t I = 0; I < BlobCount * 2; I++) + { + CHECK(Exists[I].HasBody); + CHECK_EQ(I < BlobCount, Exists[I].HasMetadata); + } + + for (size_t I = 0; I < BlobCount * 2; I++) + { + IoBuffer BuildBlob = Cache->GetBuildBlob(BuildId, BlobHashes[I]); + CHECK(BuildBlob); + CHECK_EQ(BlobHashes[I], + IoHash::HashBuffer(CompressedBuffer::FromCompressedNoValidate(std::move(BuildBlob)).Decompress().AsIoBuffer())); + } + + std::vector<CbObject> MetaDatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); + CHECK_EQ(BlobCount, MetaDatas.size()); + + std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); + CHECK_EQ(BlobCount, FetchedMetadatas.size()); + + for (size_t I = 0; I < BlobCount; I++) + { + CHECK_EQ(MetadataHashes[I], IoHash::HashBuffer(FetchedMetadatas[I].GetBuffer().AsIoBuffer())); + } + } +} + # if 0 TEST_CASE("lifetime.owner") { diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 73166e608..8c2e6d771 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -17,14 +17,11 @@ # include <mimalloc.h> #endif -#include <zenstore/cidstore.h> #include <zenstore/gc.h> -#include <zenstore/buildstore/buildstore.h> #include <zenstore/cache/structuredcachestore.h> #include <zenutil/workerpools.h> #include "config.h" -#include "projectstore/projectstore.h" #include <chrono> @@ -104,17 +101,13 @@ GetStatsForStateDirectory(std::filesystem::path StateDir) HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore* CacheStore, - CidStore* CidStore, - ProjectStore* ProjectStore, - BuildStore* BuildStore, + std::function<void()>&& FlushFunction, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions) : m_GcScheduler(Scheduler) , m_BackgroundJobQueue(BackgroundJobQueue) , m_CacheStore(CacheStore) -, m_CidStore(CidStore) -, m_ProjectStore(ProjectStore) -, m_BuildStore(BuildStore) +, m_FlushFunction(std::move(FlushFunction)) , m_LogPaths(LogPaths) , m_ServerOptions(ServerOptions) { @@ -247,6 +240,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); + Obj.AddInteger("ReturnCode", CurrentState->ReturnCode); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; @@ -782,22 +776,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, "flush", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); - if (m_CidStore) - { - m_CidStore->Flush(); - } - if (m_CacheStore) - { - m_CacheStore->Flush(); - } - if (m_ProjectStore) - { - m_ProjectStore->Flush(); - } - if (m_BuildStore) - { - m_BuildStore->Flush(); - } + m_FlushFunction(); HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h index e7821dead..9a49f5120 100644 --- a/src/zenserver/admin/admin.h +++ b/src/zenserver/admin/admin.h @@ -4,15 +4,13 @@ #include <zencore/compactbinary.h> #include <zenhttp/httpserver.h> +#include <functional> namespace zen { class GcScheduler; class JobQueue; class ZenCacheStore; -class CidStore; -class ProjectStore; -class BuildStore; struct ZenServerOptions; class HttpAdminService : public zen::HttpService @@ -27,9 +25,7 @@ public: HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore* CacheStore, - CidStore* CidStore, - ProjectStore* ProjectStore, - BuildStore* BuildStore, + std::function<void()>&& FlushFunction, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions); ~HttpAdminService(); @@ -42,9 +38,7 @@ private: GcScheduler& m_GcScheduler; JobQueue& m_BackgroundJobQueue; ZenCacheStore* m_CacheStore; - CidStore* m_CidStore; - ProjectStore* m_ProjectStore; - BuildStore* m_BuildStore; + std::function<void()> m_FlushFunction; LogPaths m_LogPaths; const ZenServerOptions& m_ServerOptions; }; diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp index bcec74ce6..2a3ce41b7 100644 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -266,7 +266,7 @@ HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) BlobsArrayIt++; MetadataArrayIt++; } - m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads); + m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads, &GetSmallWorkerPool(EWorkloadType::Burst)); return ServerRequest.WriteResponse(HttpResponseCode::OK); } @@ -484,7 +484,7 @@ HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) ResponseWriter.BeginArray("metadataExists"sv); for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) { - ResponseWriter.AddBool(BlobExists.HasBody); + ResponseWriter.AddBool(BlobExists.HasMetadata); if (BlobExists.HasMetadata) { m_BuildStoreStats.BlobExistsMetaHitCount++; @@ -529,23 +529,11 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats(); Cbo << "count" << StorageStats.EntryCount; - Cbo << "bytes" << StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes + StorageStats.MetadataByteCount; + Cbo << "bytes" << StorageStats.BlobBytes + StorageStats.MetadataByteCount; Cbo.BeginObject("blobs"); { - Cbo << "count" << (StorageStats.LargeBlobCount + StorageStats.SmallBlobCount); - Cbo << "bytes" << (StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes); - Cbo.BeginObject("large"); - { - Cbo << "count" << StorageStats.LargeBlobCount; - Cbo << "bytes" << StorageStats.LargeBlobBytes; - } - Cbo.EndObject(); // large - Cbo.BeginObject("small"); - { - Cbo << "count" << StorageStats.SmallBlobCount; - Cbo << "bytes" << StorageStats.SmallBlobBytes; - } - Cbo.EndObject(); // small + Cbo << "count" << StorageStats.BlobCount; + Cbo << "bytes" << StorageStats.BlobBytes; } Cbo.EndObject(); // blobs diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index bb0c55618..19ac3a216 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -85,7 +85,7 @@ namespace { ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, - CidStore& InCidStore, + GetCidStoreFunc&& GetCidStore, HttpStatsService& StatsService, HttpStatusService& StatusService, UpstreamCache& UpstreamCache, @@ -95,11 +95,10 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach , m_CacheStore(InCacheStore) , m_StatsService(StatsService) , m_StatusService(StatusService) -, m_CidStore(InCidStore) , m_UpstreamCache(UpstreamCache) , m_DiskWriteBlocker(InDiskWriteBlocker) , m_OpenProcessCache(InOpenProcessCache) -, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker) +, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, std::move(GetCidStore), InDiskWriteBlocker) { m_StatsService.RegisterHandler("z$", *this); m_StatusService.RegisterHandler("z$", *this); @@ -131,24 +130,6 @@ HttpStructuredCacheService::Flush() } void -HttpStructuredCacheService::ScrubStorage(ScrubContext& Ctx) -{ - if (m_LastScrubTime == Ctx.ScrubTimestamp()) - { - return; - } - - ZenCacheStore::Info Info = m_CacheStore.GetInfo(); - - ZEN_INFO("scrubbing '{}'", Info.BasePath); - - m_LastScrubTime = Ctx.ScrubTimestamp(); - - m_CidStore.ScrubStorage(Ctx); - m_CacheStore.ScrubStorage(Ctx); -} - -void HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) { std::string_view Key = Request.RelativeUri(); @@ -243,6 +224,9 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) for (const auto& NamespaceIt : ValueDetails.Namespaces) { const std::string& Namespace = NamespaceIt.first; + + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace); + for (const auto& BucketIt : NamespaceIt.second.Buckets) { const std::string& Bucket = BucketIt.first; @@ -252,7 +236,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) { for (const IoHash& Hash : ValueIt.second.Attachments) { - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); CSVWriter << "\r\n" << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString() << ", " << gsl::narrow<uint64_t>(Payload.GetSize()); @@ -270,7 +254,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) size_t AttachmentsSize = 0; for (const IoHash& Hash : ValueIt.second.Attachments) { - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); AttachmentsSize += Payload.GetSize(); } CSVWriter << ", " << gsl::narrow<uint64_t>(AttachmentsSize); @@ -292,6 +276,9 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) for (const auto& NamespaceIt : ValueDetails.Namespaces) { const std::string& Namespace = NamespaceIt.first; + + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace); + Cbo.BeginObject(); { Cbo.AddString("name", Namespace); @@ -334,7 +321,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) { Cbo.BeginObject(); Cbo.AddHash("cid", Hash); - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); Cbo.AddInteger("size", gsl::narrow<uint64_t>(Payload.GetSize())); Cbo.EndObject(); } @@ -348,7 +335,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) size_t AttachmentsSize = 0; for (const IoHash& Hash : ValueIt.second.Attachments) { - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); AttachmentsSize += Payload.GetSize(); } Cbo.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize)); @@ -623,6 +610,8 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount); + CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName); + if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty()) { ResponseWriter.BeginObject("BucketSizes"); @@ -681,7 +670,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque uint64_t AttachmentsSize = 0; - m_CidStore.IterateChunks( + ChunkStore.IterateChunks( AllAttachments, [&](size_t Index, const IoBuffer& Payload) { ZEN_UNUSED(Index); @@ -749,6 +738,8 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount); + CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName); + if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true") { CacheContentStats ContentStats; @@ -775,7 +766,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); - m_CidStore.IterateChunks( + ChunkStore.IterateChunks( ContentStats.Attachments, [&](size_t Index, const IoBuffer& Payload) { ZEN_UNUSED(Index); @@ -850,6 +841,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; Stopwatch Timer; + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && m_CacheStore.Get(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { @@ -864,17 +857,17 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con uint32_t MissingCount = 0; CbObjectView CacheRecord(ClientResultValue.Value.Data()); - CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { + CacheRecord.IterateAttachments([this, &ChunkStore, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { if (SkipData) { - if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) + if (!ChunkStore.ContainsChunk(AttachmentHash.AsHash())) { MissingCount++; } } else { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + if (IoBuffer Chunk = ChunkStore.FindChunkByCid(AttachmentHash.AsHash())) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); if (Compressed) @@ -974,6 +967,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con { Success = true; + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); + ClientResultValue.Value = UpstreamResult.Value; ClientResultValue.Value.SetContentType(AcceptType); @@ -997,8 +992,14 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con if (Success && StoreLocal) { - m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, nullptr); - m_CacheStats.WriteCount++; + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = + m_CacheStore + .Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, Overwrite, nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + } } } else if (AcceptType == ZenContentType::kCbPackage) @@ -1018,6 +1019,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con CacheRecord.IterateAttachments([this, &Package, &Ref, + &ChunkStore, &WriteAttachmentBuffers, &WriteRawHashes, &ReferencedAttachments, @@ -1052,12 +1054,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con { if (SkipData) { - if (m_CidStore.ContainsChunk(Hash)) + if (ChunkStore.ContainsChunk(Hash)) { Count.Valid++; } } - else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash)) + else if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Hash)) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); if (Compressed) @@ -1083,30 +1085,35 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con if (StoreLocal) { - m_CacheStore.Put(RequestContext, - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - CacheValue, - ReferencedAttachments, - nullptr); - m_CacheStats.WriteCount++; - - if (!WriteAttachmentBuffers.empty()) + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + CacheValue, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status == zen::PutStatus::Success) { - std::vector<CidStore::InsertResult> InsertResults = - m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); - for (const CidStore::InsertResult& Result : InsertResults) + m_CacheStats.WriteCount++; + + if (!WriteAttachmentBuffers.empty()) { - if (Result.New) + std::vector<CidStore::InsertResult> InsertResults = + ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (const CidStore::InsertResult& Result : InsertResults) { - Count.New++; + if (Result.New) + { + Count.New++; + } } } - } - WriteAttachmentBuffers = {}; - WriteRawHashes = {}; + WriteAttachmentBuffers = {}; + WriteRawHashes = {}; + } } BinaryWriter MemStream; @@ -1197,6 +1204,24 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con return Request.WriteResponse(HttpResponseCode::InsufficientStorage); } + auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) { + ZEN_UNUSED(PutResult); + + HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError; + switch (PutResult.Status) + { + case zen::PutStatus::Conflict: + ResponseCode = HttpResponseCode::Conflict; + break; + case zen::PutStatus::Invalid: + ResponseCode = HttpResponseCode::BadRequest; + break; + } + + return PutResult.Message.empty() ? Request.WriteResponse(ResponseCode) + : Request.WriteResponse(ResponseCode, zen::HttpContentType::kText, PutResult.Message); + }; + const HttpContentType ContentType = Request.RequestContentType(); Body.SetContentType(ContentType); @@ -1225,18 +1250,28 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con { RawHash = IoHash::HashBuffer(SharedBuffer(Body)); } - m_CacheStore.Put(RequestContext, - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}, - {}, - nullptr); + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}, + {}, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } m_CacheStats.WriteCount++; if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) { - m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}); + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); + m_UpstreamCache.EnqueueUpstream( + {.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}, + [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); } ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' in {}", @@ -1270,17 +1305,34 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con std::vector<IoHash> ReferencedAttachments; int32_t TotalCount = 0; - CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { - const IoHash Hash = AttachmentHash.AsHash(); - ReferencedAttachments.push_back(Hash); - if (m_CidStore.ContainsChunk(Hash)) - { - ValidAttachments.emplace_back(Hash); - } - TotalCount++; - }); + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}, ReferencedAttachments, nullptr); + CacheRecord.IterateAttachments( + [this, &ChunkStore, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { + const IoHash Hash = AttachmentHash.AsHash(); + ReferencedAttachments.push_back(Hash); + if (ChunkStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + } + TotalCount++; + }); + + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body}, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } m_CacheStats.WriteCount++; ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}", @@ -1298,10 +1350,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con CachePolicy Policy = PolicyFromUrl; if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { - m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, - .Namespace = Ref.Namespace, - .Key = {Ref.BucketSegment, Ref.HashKey}, - .ValueContentIds = std::move(ValidAttachments)}); + m_UpstreamCache.EnqueueUpstream( + {.Type = ZenContentType::kCbObject, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}, + [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); } Request.WriteResponse(HttpResponseCode::Created); @@ -1334,38 +1388,46 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con WriteAttachmentBuffers.reserve(NumAttachments); WriteRawHashes.reserve(NumAttachments); - CacheRecord.IterateAttachments( - [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count]( - CbFieldView HashView) { - const IoHash Hash = HashView.AsHash(); - ReferencedAttachments.push_back(Hash); - if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) - { - if (Attachment->IsCompressedBinary()) - { - WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); - WriteRawHashes.push_back(Hash); - ValidAttachments.emplace_back(Hash); - Count.Valid++; - } - else - { - ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(HttpContentType::kCbPackage), - Hash); - Count.Invalid++; - } - } - else if (m_CidStore.ContainsChunk(Hash)) + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); + + CacheRecord.IterateAttachments([this, + &Ref, + &Package, + &ChunkStore, + &WriteAttachmentBuffers, + &WriteRawHashes, + &ValidAttachments, + &ReferencedAttachments, + &Count](CbFieldView HashView) { + const IoHash Hash = HashView.AsHash(); + ReferencedAttachments.push_back(Hash); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) + { + if (Attachment->IsCompressedBinary()) { + WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Hash); ValidAttachments.emplace_back(Hash); Count.Valid++; } - Count.Total++; - }); + else + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(HttpContentType::kCbPackage), + Hash); + Count.Invalid++; + } + } + else if (ChunkStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + Count.Valid++; + } + Count.Total++; + }); if (Count.Invalid > 0) { @@ -1373,15 +1435,23 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); } + const bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal); + ZenCacheValue CacheValue; CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = + m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments, Overwrite); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } m_CacheStats.WriteCount++; if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (const CidStore::InsertResult& InsertResult : InsertResults) { if (InsertResult.New) @@ -1408,10 +1478,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { - m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .Namespace = Ref.Namespace, - .Key = {Ref.BucketSegment, Ref.HashKey}, - .ValueContentIds = std::move(ValidAttachments)}); + m_UpstreamCache.EnqueueUpstream( + {.Type = ZenContentType::kCbPackage, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}, + [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); } Request.WriteResponse(HttpResponseCode::Created); @@ -1445,7 +1517,9 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons { Stopwatch Timer; - IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); + + IoBuffer Value = ChunkStore.FindChunkByCid(Ref.ValueContentId); const UpstreamEndpointInfo* Source = nullptr; CachePolicy Policy = PolicyFromUrl; @@ -1467,7 +1541,7 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons { if (AreDiskWritesAllowed()) { - m_CidStore.AddChunk(UpstreamResult.Value, RawHash); + ChunkStore.AddChunk(UpstreamResult.Value, RawHash); } Source = UpstreamResult.Source; } @@ -1561,7 +1635,9 @@ HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, cons "ValueContentId does not match attachment hash"sv); } - CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash); + CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); + + CidStore::InsertResult Result = ChunkStore.AddChunk(Body, RawHash); ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}", Ref.Namespace, @@ -1776,16 +1852,51 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) EmitSnapshot("requests", m_HttpRequests, Cbo); - const uint64_t HitCount = m_CacheStats.HitCount; - const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; - const uint64_t MissCount = m_CacheStats.MissCount; - const uint64_t WriteCount = m_CacheStats.WriteCount; - const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; - struct CidStoreStats StoreStats = m_CidStore.Stats(); - const uint64_t ChunkHitCount = StoreStats.HitCount; - const uint64_t ChunkMissCount = StoreStats.MissCount; - const uint64_t ChunkWriteCount = StoreStats.WriteCount; - const uint64_t TotalCount = HitCount + MissCount; + const uint64_t HitCount = m_CacheStats.HitCount; + const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; + const uint64_t MissCount = m_CacheStats.MissCount; + const uint64_t WriteCount = m_CacheStats.WriteCount; + const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; + + uint64_t TotalChunkHitCount = 0; + uint64_t TotalChunkMissCount = 0; + uint64_t TotalChunkWriteCount = 0; + CidStoreSize TotalCidSize; + + tsl::robin_map<CidStore*, std::string> UniqueStores; + { + std::vector<std::string> NamespaceNames = m_CacheStore.GetNamespaces(); + + for (const std::string& NamespaceName : NamespaceNames) + { + CidStore* Store = &m_RpcHandler.GetCidStore(NamespaceName); + if (auto It = UniqueStores.find(Store); It == UniqueStores.end()) + { + UniqueStores.insert_or_assign(Store, NamespaceName); + } + else + { + UniqueStores.insert_or_assign(Store, std::string{}); + } + } + + for (auto It : UniqueStores) + { + CidStore* ChunkStore = It.first; + + CidStoreStats StoreStats = ChunkStore->Stats(); + CidStoreSize StoreSize = ChunkStore->TotalSize(); + + TotalChunkHitCount += StoreStats.HitCount; + TotalChunkMissCount += StoreStats.MissCount; + TotalChunkWriteCount += StoreStats.WriteCount; + + TotalCidSize.TinySize += StoreSize.TinySize; + TotalCidSize.SmallSize += StoreSize.SmallSize; + TotalCidSize.LargeSize += StoreSize.LargeSize; + TotalCidSize.TotalSize += StoreSize.TotalSize; + } + } const uint64_t RpcRequests = m_CacheStats.RpcRequests; const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; @@ -1795,17 +1906,11 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; - const CidStoreSize CidSize = m_CidStore.TotalSize(); - const GcStorageSize CacheSize = m_CacheStore.StorageSize(); + const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true"; bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true"; - CidStoreStats CidStoreStats = {}; - if (ShowCidStoreStats) - { - CidStoreStats = m_CidStore.Stats(); - } ZenCacheStore::CacheStoreStats CacheStoreStats = {}; if (ShowCacheStoreStats) { @@ -1840,6 +1945,7 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) Cbo.EndObject(); Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; + const uint64_t TotalCount = HitCount + MissCount; Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); if (m_UpstreamCache.IsActive()) @@ -1850,7 +1956,9 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); } - Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; + Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount; + const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount; + Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0); if (ShowCacheStoreStats) { @@ -1959,20 +2067,58 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) { Cbo.BeginObject("size"); { - Cbo << "tiny" << CidSize.TinySize; - Cbo << "small" << CidSize.SmallSize; - Cbo << "large" << CidSize.LargeSize; - Cbo << "total" << CidSize.TotalSize; + Cbo << "tiny" << TotalCidSize.TinySize; + Cbo << "small" << TotalCidSize.SmallSize; + Cbo << "large" << TotalCidSize.LargeSize; + Cbo << "total" << TotalCidSize.TotalSize; } Cbo.EndObject(); if (ShowCidStoreStats) { Cbo.BeginObject("store"); - Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount; - EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo); - EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo); - // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo); + + auto OutputStats = [&](CidStore& ChunkStore) { + CidStoreStats StoreStats = ChunkStore.Stats(); + Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount; + const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount; + Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0); + EmitSnapshot("read", StoreStats.FindChunkOps, Cbo); + EmitSnapshot("write", StoreStats.AddChunkOps, Cbo); + }; + + if (UniqueStores.size() > 1) + { + Cbo.BeginArray("namespaces"); + for (auto It : UniqueStores) + { + CidStore* ChunkStore = It.first; + const std::string& Namespace = It.second; + CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); + Cbo.BeginObject(); + { + Cbo << "namespace" << Namespace; + Cbo.BeginObject("stats"); + OutputStats(*ChunkStore); + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + Cbo << "tiny" << ChunkStoreSize.TinySize; + Cbo << "small" << ChunkStoreSize.SmallSize; + Cbo << "large" << ChunkStoreSize.LargeSize; + Cbo << "total" << ChunkStoreSize.TotalSize; + } + Cbo.EndObject(); + } + Cbo.EndObject(); + } + Cbo.EndArray(); // namespaces + } + else if (UniqueStores.size() != 0) + { + OutputStats(*UniqueStores.begin()->first); + } Cbo.EndObject(); } } diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h index 13c1d6475..d46ca145d 100644 --- a/src/zenserver/cache/httpstructuredcache.h +++ b/src/zenserver/cache/httpstructuredcache.h @@ -70,8 +70,10 @@ namespace cache { class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider { public: + typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc; + HttpStructuredCacheService(ZenCacheStore& InCacheStore, - CidStore& InCidStore, + GetCidStoreFunc&& GetCidStore, HttpStatsService& StatsService, HttpStatusService& StatusService, UpstreamCache& UpstreamCache, @@ -83,7 +85,6 @@ public: virtual void HandleRequest(HttpServerRequest& Request) override; void Flush(); - void ScrubStorage(ScrubContext& Ctx); private: struct CacheRef @@ -116,9 +117,7 @@ private: ZenCacheStore& m_CacheStore; HttpStatsService& m_StatsService; HttpStatusService& m_StatusService; - CidStore& m_CidStore; UpstreamCache& m_UpstreamCache; - uint64_t m_LastScrubTime = 0; metrics::OperationTiming m_HttpRequests; metrics::OperationTiming m_UpstreamGetRequestTiming; CacheStats m_CacheStats; diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 1f9ae5fb6..23bb3ad78 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -342,6 +342,7 @@ public: Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites)); } Writer.EndContainer(); } @@ -397,6 +398,8 @@ public: } BucketConfig.LargeObjectThreshold = LargeObjectThreshold; + BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites); + Value.push_back(std::make_pair(std::move(Name), BucketConfig)); } } @@ -542,6 +545,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, "cache-bucket-largeobjectthreshold"sv); + LuaOptions.AddOption("cache.bucket.limitoverwrites"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites, + "cache-bucket-limit-overwrites"sv); ////// cache.upstream LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); @@ -1120,6 +1126,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), ""); + options.add_option("cache", + "", + "cache-bucket-limit-overwrites", + "Whether to require policy flag pattern before allowing overwrites in cache bucket", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"), + ""); + options.add_option("gc", "", "gc-cache-attachment-store", diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 9753e3ae2..4a022a807 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -125,6 +125,7 @@ struct ZenStructuredCacheBucketConfig uint32_t PayloadAlignment = 1u << 4; uint64_t MemCacheSizeThreshold = 1 * 1024; uint64_t LargeObjectThreshold = 128 * 1024; + bool LimitOverwrites = false; }; struct ZenStructuredCacheConfig diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip Binary files differindex 5778fa3d2..1de4c3d74 100644 --- a/src/zenserver/frontend/html.zip +++ b/src/zenserver/frontend/html.zip diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js index 69ee234fa..c0cbb7e11 100644 --- a/src/zenserver/frontend/html/indexer/worker.js +++ b/src/zenserver/frontend/html/indexer/worker.js @@ -73,33 +73,38 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride) else if (field.is_named("packagedata")) pkg_data = field; else if (field.is_named("bulkdata")) bulk_data = field; } - if (key == undefined || pkg_data == undefined) + + if (key == undefined) continue; var id = 0n; var size = 0n; var raw_size = 0n; - for (const item of pkg_data.as_array()) - { - var found = 0, pkg_id = undefined; - for (const field of item.as_object()) + + if (pkg_data) + { + for (const item of pkg_data.as_array()) { - if (!id && field.is_named("id")) pkg_id = field.as_value(); - else if (field.is_named("size")) size += field.as_value(); - else if (field.is_named("rawsize")) raw_size += field.as_value(); - else continue; - if (found++ >= 3) - break; - } + var found = 0, pkg_id = undefined; + for (const field of item.as_object()) + { + if (!id && field.is_named("id")) pkg_id = field.as_value(); + else if (field.is_named("size")) size += field.as_value(); + else if (field.is_named("rawsize")) raw_size += field.as_value(); + else continue; + if (found++ >= 3) + break; + } - if (pkg_id === undefined) - continue; + if (pkg_id === undefined) + continue; - pkg_id = pkg_id.subarray(0, 8); - for (var i = 7; i >= 0; --i) - { - id <<= 8n; - id |= BigInt(pkg_id[i]); + pkg_id = pkg_id.subarray(0, 8); + for (var i = 7; i >= 0; --i) + { + id <<= 8n; + id |= BigInt(pkg_id[i]); + } } } @@ -119,9 +124,6 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride) } } - if (id == 0) - continue; - result[count] = [id, key.as_value(), size, raw_size]; count++; } diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 54fb11c18..08589b090 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -239,30 +239,34 @@ export class Page extends ZenPage _convert_legacy_to_tree(entry) { - const pkg_data = entry.find("packagedata"); - if (pkg_data == undefined) - return + const raw_pkgst_entry = entry.find("packagestoreentry"); + if (raw_pkgst_entry == undefined) //if there is no packagestorentry then don't show the fancy webpage, just show the raw json + return; const tree = {}; - var id = 0n; - for (var item of pkg_data.as_array()) + const pkg_data = entry.find("packagedata"); + if (pkg_data) { - var pkg_id = item.as_object().find("id"); - if (pkg_id == undefined) - continue; - - pkg_id = pkg_id.as_value().subarray(0, 8); - for (var i = 7; i >= 0; --i) + var id = 0n; + for (var item of pkg_data.as_array()) { - id <<= 8n; - id |= BigInt(pkg_id[i]); + var pkg_id = item.as_object().find("id"); + if (pkg_id == undefined) + continue; + + pkg_id = pkg_id.as_value().subarray(0, 8); + for (var i = 7; i >= 0; --i) + { + id <<= 8n; + id |= BigInt(pkg_id[i]); + } + break; } - break; + tree["$id"] = id; } - tree["$id"] = id; - const pkgst_entry = entry.find("packagestoreentry").as_object(); + const pkgst_entry = raw_pkgst_entry.as_object(); for (const field of pkgst_entry) { diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 76e022017..4ea4ee87e 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -167,6 +167,14 @@ ZenEntryPoint::Run() ZEN_INFO("ServerState.Sweep()"); ServerState.Sweep(); + auto NotifyReady = [&] { + if (!m_ServerOptions.ChildId.empty()) + { + NamedEvent ParentEvent{m_ServerOptions.ChildId}; + ParentEvent.Set(); + } + }; + uint32_t AttachSponsorProcessRetriesLeft = 3; ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(m_ServerOptions.BasePort); while (Entry) @@ -205,6 +213,7 @@ ZenEntryPoint::Run() // Sponsor processes are checked every second, so 2 second wait time should be enough if (Entry->AddSponsorProcess(m_ServerOptions.OwnerPid, 2000)) { + NotifyReady(); std::exit(0); } if (AttachSponsorProcessRetriesLeft-- > 0) @@ -350,14 +359,8 @@ ZenEntryPoint::Run() Server.SetIsReadyFunc([&] { m_LockFile.Update(MakeLockData(true), Ec); - - if (!m_ServerOptions.ChildId.empty()) - { - NamedEvent ParentEvent{m_ServerOptions.ChildId}; - ParentEvent.Set(); - } - ReportServiceStatus(ServiceStatus::Running); + NotifyReady(); }); Server.Run(); diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index ab96ae92d..52cdc5983 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -3,14 +3,12 @@ #include "buildsremoteprojectstore.h" #include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryvalidation.h> #include <zencore/compress.h> #include <zencore/fmtutils.h> +#include <zencore/scopeguard.h> #include <zenhttp/httpclientauth.h> - -#include <zenutil/jupiter/jupiterclient.h> -#include <zenutil/jupiter/jupitersession.h> +#include <zenutil/jupiter/jupiterbuildstorage.h> namespace zen { @@ -21,20 +19,24 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv; class BuildsRemoteStore : public RemoteProjectStore { public: - BuildsRemoteStore(Ref<JupiterClient>&& InJupiterClient, - std::string_view Namespace, - std::string_view Bucket, - const Oid& BuildId, - const IoBuffer& MetaData, - bool ForceDisableBlocks, - bool ForceDisableTempBlocks, - const std::filesystem::path& TempFilePath) - : m_JupiterClient(std::move(InJupiterClient)) + BuildsRemoteStore(std::unique_ptr<BuildStorage::Statistics>&& BuildStorageStats, + std::unique_ptr<HttpClient>&& BuildStorageHttp, + std::unique_ptr<BuildStorage>&& BuildStorage, + std::string_view Url, + std::string_view Namespace, + std::string_view Bucket, + const Oid& BuildId, + const IoBuffer& MetaData, + bool ForceDisableBlocks, + bool ForceDisableTempBlocks) + : m_BuildStorageStats(std::move(BuildStorageStats)) + , m_BuildStorageHttp(std::move(BuildStorageHttp)) + , m_BuildStorage(std::move(BuildStorage)) + , m_Url(Url) , m_Namespace(Namespace) , m_Bucket(Bucket) , m_BuildId(BuildId) , m_MetaData(MetaData) - , m_TempFilePath(TempFilePath) , m_EnableBlocks(!ForceDisableBlocks) , m_UseTempBlocks(!ForceDisableTempBlocks) { @@ -47,63 +49,91 @@ public: .UseTempBlockFiles = m_UseTempBlocks, .AllowChunking = true, .ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId), - .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)}; + .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_Url, m_Namespace, m_Bucket, m_BuildId)}; } virtual Stats GetStats() const override { - return {.m_SentBytes = m_SentBytes.load(), - .m_ReceivedBytes = m_ReceivedBytes.load(), - .m_RequestTimeNS = m_RequestTimeNS.load(), - .m_RequestCount = m_RequestCount.load(), - .m_PeakSentBytes = m_PeakSentBytes.load(), - .m_PeakReceivedBytes = m_PeakReceivedBytes.load(), - .m_PeakBytesPerSec = m_PeakBytesPerSec.load()}; + return { + .m_SentBytes = m_BuildStorageStats->TotalBytesWritten.load(), + .m_ReceivedBytes = m_BuildStorageStats->TotalBytesRead.load(), + .m_RequestTimeNS = m_BuildStorageStats->TotalRequestTimeUs.load() * 1000, + .m_RequestCount = m_BuildStorageStats->TotalRequestCount.load(), + .m_PeakSentBytes = m_BuildStorageStats->PeakSentBytes.load(), + .m_PeakReceivedBytes = m_BuildStorageStats->PeakReceivedBytes.load(), + .m_PeakBytesPerSec = m_BuildStorageStats->PeakBytesPerSec.load(), + }; } virtual CreateContainerResult CreateContainer() override { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - - IoBuffer Payload = m_MetaData; - Payload.SetContentType(ZenContentType::kCbObject); - JupiterResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload); - AddStats(PutResult); + CreateContainerResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); - CreateContainerResult Result{ConvertResult(PutResult)}; - if (Result.ErrorCode) + CbObject Payload = LoadCompactBinaryObject(m_MetaData); + try { - Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - Result.Reason); + CbObject PutBuildResult = m_BuildStorage->PutBuild(m_BuildId, Payload); + ZEN_UNUSED(PutBuildResult); + m_OplogBuildPartId = Oid::NewOid(); } - m_OplogBuildPartId = Oid::NewOid(); + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = + fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); + Result.Reason = + fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + } + return Result; } virtual SaveResult SaveContainer(const IoBuffer& Payload) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - PutBuildPartResult PutResult = - Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload); - AddStats(PutResult); - SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash}; - if (Result.ErrorCode) + SaveResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); + + try { - Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - m_OplogBuildPartId, - Result.Reason); + CbObject ObjectPayload = LoadCompactBinaryObject(Payload); + + std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = + m_BuildStorage->PutBuildPart(m_BuildId, m_OplogBuildPartId, OplogContainerPartName, ObjectPayload); + Result.RawHash = PutBuildPartResult.first; + Result.Needs = std::unordered_set<IoHash, IoHash::Hasher>(PutBuildPartResult.second.begin(), PutBuildPartResult.second.end()); + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + m_OplogBuildPartId, + Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + m_OplogBuildPartId, + Ex.what()); } return Result; @@ -114,52 +144,84 @@ public: ChunkBlockDescription&& Block) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - JupiterResult PutResult = - Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); - AddStats(PutResult); + SaveAttachmentResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); - SaveAttachmentResult Result{ConvertResult(PutResult)}; - if (Result.ErrorCode) + try { - Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - RawHash, - Result.Reason); - return Result; - } + m_BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); - if (Block.BlockHash == RawHash) - { - CbObjectWriter BlockMetaData; - BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string()); - - IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer(); - MetaPayload.SetContentType(ZenContentType::kCbObject); - JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, RawHash, MetaPayload); - AddStats(PutMetaResult); - RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult); - if (MetaDataResult.ErrorCode) + if (Block.BlockHash == RawHash) { - ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - RawHash, - MetaDataResult.Reason); + try + { + CbObjectWriter BlockMetaData; + BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string()); + CbObject MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()); + if (!m_BuildStorage->PutBlockMetadata(m_BuildId, RawHash, MetaPayload)) + { + ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + RawHash, + "not found"); + } + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } } } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + return Result; } virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override { SaveAttachmentsResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); for (const SharedBuffer& Chunk : Chunks) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer()); @@ -177,38 +239,68 @@ public: ZEN_UNUSED(RawHash); ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - FinalizeBuildPartResult FinalizeRefResult = - Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash); - AddStats(FinalizeRefResult); + FinalizeResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); - FinalizeResult Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}}; - if (Result.ErrorCode) + try + { + std::vector<IoHash> Needs = m_BuildStorage->FinalizeBuildPart(m_BuildId, m_OplogBuildPartId, RawHash); + Result.Needs = std::unordered_set<IoHash, IoHash::Hasher>(Needs.begin(), Needs.end()); + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0; + Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + m_OplogBuildPartId, + Ex.what()); + } + catch (const std::exception& Ex) { - Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - m_OplogBuildPartId, - Result.Reason); + Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + m_OplogBuildPartId, + Ex.what()); } - else if (Result.Needs.empty()) + + if (!Result.ErrorCode && Result.Needs.empty()) { - JupiterResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId); - AddStats(FinalizeBuildResult); - FinalizeBuildResult.ElapsedSeconds += FinalizeRefResult.ElapsedSeconds; - Result = {ConvertResult(FinalizeBuildResult)}; - if (Result.ErrorCode) + try { - Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - FinalizeBuildResult.Reason); + m_BuildStorage->FinalizeBuild(m_BuildId); + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error + : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode + : 0; + Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); } } + return Result; } @@ -216,161 +308,128 @@ public: { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId); - AddStats(GetBuildResult); - LoadContainerResult Result{ConvertResult(GetBuildResult)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog container build from {}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - Result.Reason); - return Result; - } - CbObject BuildObject = LoadCompactBinaryObject(GetBuildResult.Response); - if (!BuildObject) - { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The build {}/{}/{}/{} payload is not formatted as a compact binary object"sv, - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId); - return Result; - } - CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); - if (!PartsObject) + LoadContainerResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); + + try { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv, - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId); - return Result; + CbObject BuildObject = m_BuildStorage->GetBuild(m_BuildId); + + CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); + if (!PartsObject) + { + throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv, + m_Url, + m_Namespace, + m_Bucket, + m_BuildId)); + } + m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId(); + if (m_OplogBuildPartId == Oid::Zero) + { + throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv, + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + OplogContainerPartName)); + } + + Result.ContainerObject = m_BuildStorage->GetBuildPart(m_BuildId, m_OplogBuildPartId); } - m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId(); - if (m_OplogBuildPartId == Oid::Zero) + catch (const HttpClientError& Ex) { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv, - m_JupiterClient->ServiceUrl(), + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'", + m_Url, m_Namespace, m_Bucket, m_BuildId, - OplogContainerPartName); - return Result; + Ex.what()); } - - JupiterResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId); - AddStats(GetBuildPartResult); - Result = {ConvertResult(GetBuildResult)}; - Result.ElapsedSeconds += GetBuildResult.ElapsedSeconds; - if (Result.ErrorCode) + catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed fetching oplog build part from {}/{}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), + Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'", + m_Url, m_Namespace, m_Bucket, m_BuildId, - m_OplogBuildPartId, - Result.Reason); - return Result; + Ex.what()); } - CbObject ContainerObject = LoadCompactBinaryObject(GetBuildPartResult.Response); - if (!ContainerObject) - { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The build part for oplog container {}/{}/{}/{}/{} is not formatted as a compact binary object"sv, - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - m_OplogBuildPartId); - return Result; - } - Result.ContainerObject = std::move(ContainerObject); return Result; } virtual GetKnownBlocksResult GetKnownBlocks() override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1); - AddStats(FindResult); - GetKnownBlocksResult Result{ConvertResult(FindResult)}; - if (Result.ErrorCode) + + GetKnownBlocksResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); + + try { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - Result.Reason); - return Result; + CbObject KnownBlocks = m_BuildStorage->FindBlocks(m_BuildId, 10000u); + std::optional<std::vector<ChunkBlockDescription>> Blocks = ParseChunkBlockDescriptionList(KnownBlocks); + Result.Blocks.reserve(Blocks.value().size()); + for (ChunkBlockDescription& BlockDescription : Blocks.value()) + { + Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash, + .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)}); + } } - if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None) + catch (const HttpClientError& Ex) { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a compact binary object"sv, - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId); - return Result; + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = + fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } - std::optional<std::vector<ChunkBlockDescription>> Blocks = - ParseChunkBlockDescriptionList(LoadCompactBinaryObject(FindResult.Response)); - if (!Blocks) + catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a list of blocks"sv, - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId); - return Result; - } - Result.Blocks.reserve(Blocks.value().size()); - for (ChunkBlockDescription& BlockDescription : Blocks.value()) - { - Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash, - .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)}); + Result.Reason = + fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } + return Result; } virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath); - AddStats(GetResult); - LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)}; - if (GetResult.ErrorCode) + LoadAttachmentResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); + + try + { + Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = + fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + } + catch (const std::exception& Ex) { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_BuildId, - RawHash, - Result.Reason); + Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); + Result.Reason = + fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } + return Result; } virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override { LoadAttachmentsResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); for (const IoHash& Hash : RawHashes) { LoadAttachmentResult ChunkResult = LoadAttachment(Hash); @@ -386,81 +445,27 @@ public: } private: - void AddStats(const JupiterResult& Result) - { - m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes)); - m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes)); - m_RequestTimeNS.fetch_add(static_cast<uint64_t>(Result.ElapsedSeconds * 1000000000)); - SetAtomicMax(m_PeakSentBytes, Result.SentBytes); - SetAtomicMax(m_PeakReceivedBytes, Result.ReceivedBytes); - if (Result.ElapsedSeconds > 0.0) - { - uint64_t BytesPerSec = static_cast<uint64_t>((Result.SentBytes + Result.ReceivedBytes) / Result.ElapsedSeconds); - SetAtomicMax(m_PeakBytesPerSec, BytesPerSec); - } - - m_RequestCount.fetch_add(1); - } - - static Result ConvertResult(const JupiterResult& Response) + static int MakeErrorCode(const HttpClientError& Ex) { - std::string Text; - int32_t ErrorCode = 0; - if (Response.ErrorCode != 0 || !Response.Success) - { - if (Response.Response) - { - HttpContentType ContentType = Response.Response.GetContentType(); - if (ContentType == ZenContentType::kText || ContentType == ZenContentType::kJSON) - { - ExtendableStringBuilder<256> SB; - SB.Append("\n"); - SB.Append(std::string_view(reinterpret_cast<const std::string::value_type*>(Response.Response.GetData()), - Response.Response.GetSize())); - Text = SB.ToString(); - } - else if (ContentType == ZenContentType::kCbObject) - { - ExtendableStringBuilder<256> SB; - SB.Append("\n"); - CompactBinaryToJson(Response.Response.GetView(), SB); - Text = SB.ToString(); - } - } - } - if (Response.ErrorCode != 0) - { - ErrorCode = Response.ErrorCode; - } - else if (!Response.Success) - { - ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - } - return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text}; + return Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0; } - Ref<JupiterClient> m_JupiterClient; - const std::string m_Namespace; - const std::string m_Bucket; - const Oid m_BuildId; - IoBuffer m_MetaData; - Oid m_OplogBuildPartId = Oid::Zero; - std::filesystem::path m_TempFilePath; - const bool m_EnableBlocks = true; - const bool m_UseTempBlocks = true; - const bool m_AllowRedirect = false; - - std::atomic_uint64_t m_SentBytes = {}; - std::atomic_uint64_t m_ReceivedBytes = {}; - std::atomic_uint64_t m_RequestTimeNS = {}; - std::atomic_uint64_t m_RequestCount = {}; - std::atomic_uint64_t m_PeakSentBytes = {}; - std::atomic_uint64_t m_PeakReceivedBytes = {}; - std::atomic_uint64_t m_PeakBytesPerSec = {}; + std::unique_ptr<BuildStorage::Statistics> m_BuildStorageStats; + std::unique_ptr<HttpClient> m_BuildStorageHttp; + std::unique_ptr<BuildStorage> m_BuildStorage; + const std::string m_Url; + const std::string m_Namespace; + const std::string m_Bucket; + const Oid m_BuildId; + IoBuffer m_MetaData; + Oid m_OplogBuildPartId = Oid::Zero; + const bool m_EnableBlocks = true; + const bool m_UseTempBlocks = true; + const bool m_AllowRedirect = false; }; std::shared_ptr<RemoteProjectStore> -CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath) +CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet) { std::string Url = Options.Url; if (Url.find("://"sv) == std::string::npos) @@ -468,13 +473,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file // Assume https URL Url = fmt::format("https://{}"sv, Url); } - JupiterClientOptions ClientOptions{.Name = "Remote store"sv, - .ServiceUrl = Url, - .ConnectTimeout = std::chrono::milliseconds(2000), - .Timeout = std::chrono::milliseconds(1800000), - .AssumeHttp2 = Options.AssumeHttp2, - .AllowResume = true, - .RetryCount = 4}; + // 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider // 2) Access token as parameter in request // 3) Environment variable (different win vs linux/mac) @@ -491,7 +490,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file } else if (!Options.OidcExePath.empty()) { - if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe) + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe) { TokenProvider = TokenProviderMaybe.value(); } @@ -502,16 +501,31 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); } - Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider))); + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .ConnectTimeout = std::chrono::milliseconds(2000), + .Timeout = std::chrono::milliseconds(1800000), + .AccessTokenProvider = std::move(TokenProvider), + .AssumeHttp2 = Options.AssumeHttp2, + .AllowResume = true, + .RetryCount = 4}; + + std::unique_ptr<BuildStorage::Statistics> BuildStorageStats(std::make_unique<BuildStorage::Statistics>()); + + std::unique_ptr<HttpClient> BuildStorageHttp = std::make_unique<HttpClient>(Url, ClientSettings); + + std::unique_ptr<BuildStorage> BuildStorage = + CreateJupiterBuildStorage(Log(), *BuildStorageHttp, *BuildStorageStats, Options.Namespace, Options.Bucket, false, TempFilePath); - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(Client), + std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(BuildStorageStats), + std::move(BuildStorageHttp), + std::move(BuildStorage), + Url, Options.Namespace, Options.Bucket, Options.BuildId, Options.MetaData, Options.ForceDisableBlocks, - Options.ForceDisableTempBlocks, - TempFilePath); + Options.ForceDisableTempBlocks); return RemoteStore; } diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h index c52b13886..60b6caef7 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.h +++ b/src/zenserver/projectstore/buildsremoteprojectstore.h @@ -24,7 +24,8 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions IoBuffer MetaData; }; -std::shared_ptr<RemoteProjectStore> CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, - const std::filesystem::path& TempFilePath); +std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, + const std::filesystem::path& TempFilePath, + bool Quiet); } // namespace zen diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 317a419eb..9600133f3 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -235,13 +235,11 @@ namespace { ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(CidStore& Store, - ProjectStore* Projects, +HttpProjectService::HttpProjectService(ProjectStore* Projects, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr) : m_Log(logging::Get("project")) -, m_CidStore(Store) , m_ProjectStore(Projects) , m_StatusService(StatusService) , m_StatsService(StatsService) @@ -407,8 +405,45 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { ZEN_TRACE_CPU("ProjectService::Stats"); - const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); - const CidStoreSize CidSize = m_CidStore.TotalSize(); + bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true"; + + const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); + uint64_t TotalChunkHitCount = 0; + uint64_t TotalChunkMissCount = 0; + uint64_t TotalChunkWriteCount = 0; + CidStoreSize TotalCidSize; + + tsl::robin_map<CidStore*, std::string> UniqueStores; + { + m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) { + CidStore* Store = &Project.GetCidStore(); + if (auto It = UniqueStores.find(Store); It == UniqueStores.end()) + { + UniqueStores.insert_or_assign(Store, Project.Identifier); + } + else + { + UniqueStores.insert_or_assign(Store, std::string{}); + } + }); + + for (auto It : UniqueStores) + { + CidStore* ChunkStore = It.first; + + CidStoreStats ChunkStoreStats = ChunkStore->Stats(); + CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); + + TotalChunkHitCount += ChunkStoreStats.HitCount; + TotalChunkMissCount += ChunkStoreStats.MissCount; + TotalChunkWriteCount += ChunkStoreStats.WriteCount; + + TotalCidSize.TinySize += ChunkStoreSize.TinySize; + TotalCidSize.SmallSize += ChunkStoreSize.SmallSize; + TotalCidSize.LargeSize += ChunkStoreSize.LargeSize; + TotalCidSize.TotalSize += ChunkStoreSize.TotalSize; + } + } CbObjectWriter Cbo; @@ -460,12 +495,66 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { Cbo.BeginObject("size"); { - Cbo << "tiny" << CidSize.TinySize; - Cbo << "small" << CidSize.SmallSize; - Cbo << "large" << CidSize.LargeSize; - Cbo << "total" << CidSize.TotalSize; + Cbo << "tiny" << TotalCidSize.TinySize; + Cbo << "small" << TotalCidSize.SmallSize; + Cbo << "large" << TotalCidSize.LargeSize; + Cbo << "total" << TotalCidSize.TotalSize; } Cbo.EndObject(); + + if (ShowCidStoreStats) + { + Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount; + const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount; + Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0); + + Cbo.BeginObject("store"); + + auto OutputStats = [&](CidStore& ChunkStore) { + CidStoreStats StoreStats = ChunkStore.Stats(); + Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount; + const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount; + Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0); + EmitSnapshot("read", StoreStats.FindChunkOps, Cbo); + EmitSnapshot("write", StoreStats.AddChunkOps, Cbo); + }; + + if (UniqueStores.size() > 1) + { + Cbo.BeginArray("projects"); + for (auto It : UniqueStores) + { + CidStore* ChunkStore = It.first; + const std::string& ProjectId = It.second; + CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); + + Cbo.BeginObject(); + { + Cbo << "project" << ProjectId; + Cbo.BeginObject("stats"); + OutputStats(*ChunkStore); + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + Cbo << "tiny" << ChunkStoreSize.TinySize; + Cbo << "small" << ChunkStoreSize.SmallSize; + Cbo << "large" << ChunkStoreSize.LargeSize; + Cbo << "total" << ChunkStoreSize.TotalSize; + } + Cbo.EndObject(); + } + Cbo.EndObject(); + } + Cbo.EndArray(); // projects + } + else if (UniqueStores.size() != 0) + { + CidStore& ChunkStore = *UniqueStores.begin()->first; + OutputStats(ChunkStore); + } + Cbo.EndObject(); + } } Cbo.EndObject(); @@ -1125,6 +1214,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; IoBuffer Payload = HttpReq.ReadPayload(); @@ -1137,7 +1228,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) std::vector<IoHash> MissingChunks; CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { - if (m_CidStore.ContainsChunk(Hash)) + if (ChunkStore.ContainsChunk(Hash)) { // Return null attachment as we already have it, no point in reading it and storing it again return {}; @@ -1393,6 +1484,8 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString)) @@ -1407,7 +1500,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) Op.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsAttachment(); - IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash); + IoBuffer Payload = ChunkStore.FindChunkByCid(AttachmentHash); if (Payload) { switch (Payload.GetContentType()) @@ -2036,11 +2129,14 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) CSVHeader(Details, AttachmentDetails, CSVWriter); m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { + CidStore& ChunkStore = Project.GetCidStore(); + Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); }); }); @@ -2054,8 +2150,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) m_ProjectStore->DiscoverProjects(); m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { - std::vector<std::string> OpLogs = Project.ScanForOplogs(); - CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + CidStore& ChunkStore = Project.GetCidStore(); + std::vector<std::string> OpLogs = Project.ScanForOplogs(); + CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); }); } Cbo.EndArray(); @@ -2084,7 +2181,8 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Project& Project = *FoundProject.Get(); + CidStore& ChunkStore = Project.GetCidStore(); if (CSV) { @@ -2092,10 +2190,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) CSVHeader(Details, AttachmentDetails, CSVWriter); FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } @@ -2105,7 +2204,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) std::vector<std::string> OpLogs = FoundProject->ScanForOplogs(); Cbo.BeginArray("projects"); { - CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); @@ -2141,16 +2240,17 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Project& Project = *FoundProject.Get(); - ProjectStore::Oplog& Oplog = *FoundLog; + ProjectStore::Project& Project = *FoundProject.Get(); + CidStore& ChunkStore = Project.GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; if (CSV) { ExtendableStringBuilder<4096> CSVWriter; CSVHeader(Details, AttachmentDetails, CSVWriter); Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + [this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } @@ -2159,7 +2259,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) CbObjectWriter Cbo; Cbo.BeginArray("oplogs"); { - CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); + CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); @@ -2204,9 +2304,10 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId)); } - const Oid ObjId = Oid::FromHexString(OpId); - ProjectStore::Project& Project = *FoundProject.Get(); - ProjectStore::Oplog& Oplog = *FoundLog; + const Oid ObjId = Oid::FromHexString(OpId); + ProjectStore::Project& Project = *FoundProject.Get(); + CidStore& ChunkStore = Project.GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId); if (!Op.has_value()) @@ -2224,7 +2325,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) ExtendableStringBuilder<4096> CSVWriter; CSVHeader(Details, AttachmentDetails, CSVWriter); - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter); + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } else @@ -2232,7 +2333,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) CbObjectWriter Cbo; Cbo.BeginArray("ops"); { - CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo); + CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 295defa5c..5782188e6 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -35,11 +35,7 @@ class ProjectStore; class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(CidStore& Store, - ProjectStore* InProjectStore, - HttpStatusService& StatusService, - HttpStatsService& StatsService, - AuthMgr& AuthMgr); + HttpProjectService(ProjectStore* InProjectStore, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr); ~HttpProjectService(); virtual const char* BaseUri() const override; @@ -92,7 +88,6 @@ private: inline LoggerRef Log() { return m_Log; } LoggerRef m_Log; - CidStore& m_CidStore; HttpRequestRouter m_Router; Ref<ProjectStore> m_ProjectStore; HttpStatusService& m_StatusService; diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index 3728babb5..dba5cd4a7 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -337,7 +337,7 @@ private: }; std::shared_ptr<RemoteProjectStore> -CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath) +CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet) { std::string Url = Options.Url; if (Url.find("://"sv) == std::string::npos) @@ -368,7 +368,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi } else if (!Options.OidcExePath.empty()) { - if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe) + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe) { TokenProvider = TokenProviderMaybe.value(); } diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h index 8bf79d563..ac2d25b47 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.h +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h @@ -25,6 +25,7 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions }; std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, - const std::filesystem::path& TempFilePath); + const std::filesystem::path& TempFilePath, + bool Quiet); } // namespace zen diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 53e687983..d6dd6ef9b 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -267,7 +267,7 @@ namespace { ForceDisableBlocks, ForceDisableTempBlocks, AssumeHttp2}; - RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath); + RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false); } if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) @@ -364,7 +364,7 @@ namespace { ForceDisableTempBlocks, AssumeHttp2, MetaData}; - RemoteStore = CreateBuildsRemoteStore(Options, TempFilePath); + RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false); } if (!RemoteStore) @@ -1176,7 +1176,7 @@ ProjectStore::Oplog::Flush() } void -ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx) +ProjectStore::Oplog::Scrub(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetProjectstoreTag()); @@ -1492,7 +1492,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f if (ValidationError != CbValidateError::None) { - ZEN_ERROR("validation error {} hit for oplog config at '{}'", int(ValidationError), StateFilePath); + ZEN_ERROR("validation error {} hit for oplog config at '{}'", ToString(ValidationError), StateFilePath); return CbObject(); } @@ -3168,7 +3168,7 @@ ProjectStore::Project::Read() } else { - ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath); + ZEN_ERROR("validation error {} hit for '{}'", ToString(ValidationError), ProjectStateFilePath); } ReadAccessTimes(); @@ -3260,7 +3260,7 @@ ProjectStore::Project::ReadAccessTimes() } else { - ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, int(ValidationError), ProjectAccessTimesFilePath); + ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, ToString(ValidationError), ProjectAccessTimesFilePath); } } @@ -3575,7 +3575,7 @@ ProjectStore::Project::Flush() } void -ProjectStore::Project::ScrubStorage(ScrubContext& Ctx) +ProjectStore::Project::Scrub(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetProjectstoreTag()); // Scrubbing needs to check all existing oplogs @@ -3587,7 +3587,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx) IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) { if (!IsExpired(GcClock::TimePoint::min(), Ops)) { - Ops.ScrubStorage(Ctx); + Ops.Scrub(Ctx); } }); } @@ -3832,7 +3832,7 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const ////////////////////////////////////////////////////////////////////////// -ProjectStore::ProjectStore(CidStore& Store, +ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, @@ -3840,7 +3840,7 @@ ProjectStore::ProjectStore(CidStore& Store, const Configuration& Config) : m_Log(logging::Get("project")) , m_Gc(Gc) -, m_CidStore(Store) +, m_GetCidStore(std::move(GetCidStore)) , m_JobQueue(JobQueue) , m_OpenProcessCache(InOpenProcessCache) , m_ProjectBasePath(BasePath) @@ -3973,7 +3973,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx) } for (const Ref<Project>& Project : Projects) { - Project->ScrubStorage(Ctx); + Project->Scrub(Ctx); } } @@ -4025,6 +4025,8 @@ ProjectStore::OpenProject(std::string_view ProjectId) } } + CidStore& ChunkStore = m_GetCidStore(ProjectId); + RwLock::ExclusiveLockScope _(m_ProjectsLock); if (auto ProjIt = m_Projects.find(std::string{ProjectId}); ProjIt != m_Projects.end()) { @@ -4041,7 +4043,7 @@ ProjectStore::OpenProject(std::string_view ProjectId) Ref<Project>& Prj = m_Projects - .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) + .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath))) .first->second; Prj->Identifier = ProjectId; Prj->Read(); @@ -4068,12 +4070,14 @@ ProjectStore::NewProject(const std::filesystem::path& BasePath, ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Store::NewProject"); + CidStore& ChunkStore = m_GetCidStore(ProjectId); + RwLock::ExclusiveLockScope _(m_ProjectsLock); ZEN_INFO("project '{}': creating project at '{}'", ProjectId, BasePath); Ref<Project>& Prj = - m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) + m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath))) .first->second; Prj->Identifier = ProjectId; Prj->RootDir = RootDir; @@ -4802,7 +4806,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } const IoHash Hash = IoHash::FromHexString(Cid); - OutChunk = m_CidStore.FindChunkByCid(Hash); + OutChunk = Project->GetCidStore().FindChunkByCid(Hash); if (!OutChunk) { @@ -4865,7 +4869,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId, } FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash}); - CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash); + CidStore::InsertResult Result = Project->GetCidStore().AddChunk(Chunk, Hash); return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}}; } @@ -4894,18 +4898,19 @@ ProjectStore::GetChunks(const std::string_view ProjectId, } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + if (RequestObject["chunks"sv].IsArray()) { // Legacy full chunks only by rawhash - CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); - + CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); CbObjectWriter ResponseWriter; ResponseWriter.BeginArray("chunks"sv); for (CbFieldView FieldView : ChunksArray) { IoHash RawHash = FieldView.AsHash(); - IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); + IoBuffer ChunkBuffer = ChunkStore.FindChunkByCid(RawHash); if (ChunkBuffer) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)); @@ -5057,7 +5062,7 @@ ProjectStore::GetChunks(const std::string_view ProjectId, if (ChunkRequest.Input.Id.index() == 0) { const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id); - IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash); + IoBuffer Payload = ChunkStore.FindChunkByCid(ChunkHash); if (Payload) { ChunkRequest.Output.Exists = true; @@ -5244,7 +5249,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie return {HttpResponseCode::BadRequest, "Invalid payload format"}; } - CidStore& ChunkStore = m_CidStore; + CidStore& ChunkStore = Project->GetCidStore(); RwLock AttachmentsLock; tsl::robin_set<IoHash, IoHash::Hasher> Attachments; @@ -5350,7 +5355,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId, } } - CidStore& ChunkStore = m_CidStore; + CidStore& ChunkStore = Project->GetCidStore(); RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( ChunkStore, @@ -5462,6 +5467,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + if (Method == "import"sv) { if (!AreDiskWritesAllowed()) @@ -5543,7 +5550,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, } Oplog->CaptureAddedAttachments(WriteRawHashes); - m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); + ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); } HttpReq.WriteResponse(HttpResponseCode::OK); return true; @@ -5716,14 +5723,14 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, ResponseObj.EndArray(); } - // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new - // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have + // Ops that have moved chunks to a compressed buffer for storage in ChunkStore have been rewritten with references to the new + // chunk(s). Make sure we add the chunks to ChunkStore, and do it after we update the oplog so GC doesn't think we have // unreferenced chunks. for (auto It : AddedChunks) { const IoHash& RawHash = It.first; AddedChunk& Chunk = It.second; - CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash); + CidStore::InsertResult Result = ChunkStore.AddChunk(Chunk.Buffer, RawHash); if (Result.New) { InlinedBytes += Chunk.RawSize; @@ -5786,7 +5793,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op EmbedLooseFile, Force, IgnoreMissingAttachments](JobContext& Context) { - RemoteProjectStore::Result Result = SaveOplog(m_CidStore, + RemoteProjectStore::Result Result = SaveOplog(Project->GetCidStore(), *ActualRemoteStore, *Project.Get(), *OplogPtr, @@ -5801,7 +5808,8 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second); if (!IsHttpSuccessCode(Response.first)) { - throw std::runtime_error(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second); + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); } }); @@ -5830,19 +5838,26 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog, } std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + CidStore& ChunkStore = Project.GetCidStore(); ZEN_INFO("Loading oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description); JobId JobId = m_JobQueue.QueueJob( fmt::format("Import oplog '{}/{}'", Project.Identifier, Oplog.OplogId()), - [this, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, Force, IgnoreMissingAttachments, CleanOplog]( - JobContext& Context) { + [this, + ChunkStore = &ChunkStore, + ActualRemoteStore = std::move(RemoteStore), + OplogPtr = &Oplog, + Force, + IgnoreMissingAttachments, + CleanOplog](JobContext& Context) { RemoteProjectStore::Result Result = - LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context); + LoadOplog(*ChunkStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context); auto Response = ConvertResult(Result); ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); if (!IsHttpSuccessCode(Response.first)) { - throw std::runtime_error(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second); + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); } }); @@ -6875,6 +6890,11 @@ namespace testutils { return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags); } + ProjectStore::GetCidStoreFunc SingleChunkStore(CidStore& ChunkStore) + { + return [ChunkStore = &ChunkStore](std::string_view) -> CidStore& { return *ChunkStore; }; + } + } // namespace testutils TEST_CASE("project.opkeys") @@ -6937,13 +6957,18 @@ TEST_CASE("project.store.create") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::string_view ProjectName("proj1"sv); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -6968,12 +6993,17 @@ TEST_CASE("project.store.lifetimes") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -7031,12 +7061,17 @@ TEST_CASE_TEMPLATE("project.store.export", auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -7070,7 +7105,7 @@ TEST_CASE_TEMPLATE("project.store.export", std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options); RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - RemoteProjectStore::Result ExportResult = SaveOplog(CidStore, + RemoteProjectStore::Result ExportResult = SaveOplog(ChunkStore, *RemoteStore, *Project.Get(), *Oplog, @@ -7087,7 +7122,7 @@ TEST_CASE_TEMPLATE("project.store.export", ProjectStore::Oplog* OplogImport = Project->NewOplog("oplog2", {}); CHECK(OplogImport != nullptr); - RemoteProjectStore::Result ImportResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ false, @@ -7096,7 +7131,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportResult.ErrorCode == 0); - RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportForceResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ true, @@ -7105,7 +7140,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportForceResult.ErrorCode == 0); - RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportCleanResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ false, @@ -7114,7 +7149,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportCleanResult.ErrorCode == 0); - RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ true, @@ -7134,12 +7169,17 @@ TEST_CASE("project.store.gc") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -7335,12 +7375,17 @@ TEST_CASE("project.store.gc.prep") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -7383,7 +7428,7 @@ TEST_CASE("project.store.gc.prep") // Equivalent of a `prep` existance check call for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7397,7 +7442,7 @@ TEST_CASE("project.store.gc.prep") // If a gc comes in between our prep and op write the chunks will be removed for (auto Attachment : OpAttachments) { - CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7417,7 +7462,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7431,7 +7476,7 @@ TEST_CASE("project.store.gc.prep") // Attachments should now be retained for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7445,7 +7490,7 @@ TEST_CASE("project.store.gc.prep") // Attachments should now be retained across multiple GCs if retain time is still valud for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7457,7 +7502,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); @@ -7474,7 +7519,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7507,7 +7552,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } Sleep(200); @@ -7522,7 +7567,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } // This pass the retention time has expired and the last GC pass cleared the entries @@ -7536,7 +7581,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } } @@ -7550,12 +7595,17 @@ TEST_CASE("project.store.rpc.getchunks") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; @@ -8472,12 +8522,17 @@ TEST_CASE("project.store.partial.read") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; @@ -8650,12 +8705,17 @@ TEST_CASE("project.store.iterateoplog") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv"; diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 368da5ea4..eb27665f9 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -68,7 +68,9 @@ public: { }; - ProjectStore(CidStore& Store, + typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc; + + ProjectStore(GetCidStoreFunc&& GetCidStore, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, @@ -156,7 +158,7 @@ public: LoggerRef Log() { return m_OuterProject->Log(); } void Flush(); - void ScrubStorage(ScrubContext& Ctx); + void Scrub(ScrubContext& Ctx); static uint64_t TotalSize(const std::filesystem::path& BasePath); uint64_t TotalSize() const; @@ -326,11 +328,13 @@ public: Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath); virtual ~Project(); + CidStore& GetCidStore() { return m_CidStore; }; + void Read(); void Write(); [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath); void Flush(); - void ScrubStorage(ScrubContext& Ctx); + void Scrub(ScrubContext& Ctx); LoggerRef Log() const; static uint64_t TotalSize(const std::filesystem::path& BasePath); uint64_t TotalSize() const; @@ -405,6 +409,7 @@ public: LoggerRef Log() { return m_Log; } const std::filesystem::path& BasePath() const { return m_ProjectBasePath; } + // GcStorage virtual void ScrubStorage(ScrubContext& Ctx) override; virtual GcStorageSize StorageSize() const override; @@ -498,7 +503,7 @@ public: private: LoggerRef m_Log; GcManager& m_Gc; - CidStore& m_CidStore; + GetCidStoreFunc m_GetCidStore; JobQueue& m_JobQueue; OpenProcessCache& m_OpenProcessCache; std::filesystem::path m_ProjectBasePath; diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp index 744b861dd..a1c460bc0 100644 --- a/src/zenserver/upstream/upstreamcache.cpp +++ b/src/zenserver/upstream/upstreamcache.cpp @@ -1475,12 +1475,17 @@ namespace detail { class UpstreamCacheImpl final : public UpstreamCache { + struct EnqueuedRequest + { + UpstreamCacheRecord Record; + std::function<IoBuffer(const IoHash& ChunkHash)> GetValueFunc; + }; + public: - UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) + UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore) : m_Log(logging::Get("upstream")) , m_Options(Options) , m_CacheStore(CacheStore) - , m_CidStore(CidStore) { } @@ -1836,17 +1841,17 @@ public: } } - virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) override + virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) override { if (m_RunState.IsRunning && m_Options.WriteUpstream && m_Endpoints.size() > 0) { if (!m_UpstreamThreads.empty()) { - m_UpstreamQueue.Enqueue(std::move(CacheRecord)); + m_UpstreamQueue.Enqueue(EnqueuedRequest{.Record = std::move(CacheRecord), .GetValueFunc = GetValueFunc}); } else { - ProcessCacheRecord(std::move(CacheRecord)); + ProcessCacheRecord(std::move(CacheRecord), std::move(GetValueFunc)); } } } @@ -1900,7 +1905,7 @@ public: } private: - void ProcessCacheRecord(UpstreamCacheRecord CacheRecord) + void ProcessCacheRecord(const UpstreamCacheRecord& CacheRecord, std::function<IoBuffer(const IoHash& ChunkHash)>&& GetValueFunc) { ZEN_TRACE_CPU("Upstream::ProcessCacheRecord"); @@ -1918,7 +1923,7 @@ private: for (const IoHash& ValueContentId : CacheRecord.ValueContentIds) { - if (IoBuffer Payload = m_CidStore.FindChunkByCid(ValueContentId)) + if (IoBuffer Payload = GetValueFunc(ValueContentId)) { Payloads.push_back(Payload); } @@ -1970,19 +1975,19 @@ private: for (;;) { - UpstreamCacheRecord CacheRecord; - if (m_UpstreamQueue.WaitAndDequeue(CacheRecord)) + EnqueuedRequest Request; + if (m_UpstreamQueue.WaitAndDequeue(Request)) { try { - ProcessCacheRecord(std::move(CacheRecord)); + ProcessCacheRecord(Request.Record, std::move(Request.GetValueFunc)); } catch (const std::exception& Err) { ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'", - CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, + Request.Record.Namespace, + Request.Record.Key.Bucket, + Request.Record.Key.Hash, Err.what()); } } @@ -2076,7 +2081,7 @@ private: LoggerRef Log() { return m_Log; } - using UpstreamQueue = BlockingQueue<UpstreamCacheRecord>; + using UpstreamQueue = BlockingQueue<EnqueuedRequest>; struct RunState { @@ -2102,7 +2107,6 @@ private: LoggerRef m_Log; UpstreamCacheOptions m_Options; ZenCacheStore& m_CacheStore; - CidStore& m_CidStore; UpstreamQueue m_UpstreamQueue; std::shared_mutex m_EndpointsMutex; std::vector<std::unique_ptr<UpstreamEndpoint>> m_Endpoints; @@ -2126,9 +2130,9 @@ UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, con } std::unique_ptr<UpstreamCache> -CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) +CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore) { - return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore); + return std::make_unique<UpstreamCacheImpl>(Options, CacheStore); } } // namespace zen diff --git a/src/zenserver/upstream/upstreamcache.h b/src/zenserver/upstream/upstreamcache.h index 26e5decac..e4b9a73ad 100644 --- a/src/zenserver/upstream/upstreamcache.h +++ b/src/zenserver/upstream/upstreamcache.h @@ -24,7 +24,6 @@ class AuthMgr; class CbObjectView; class CbPackage; class CbObjectWriter; -class CidStore; class ZenCacheStore; struct JupiterClientOptions; class JupiterAccessTokenProvider; @@ -162,6 +161,6 @@ struct UpstreamCacheOptions bool WriteUpstream = true; }; -std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore); +std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore); } // namespace zen diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 27ec4c690..5cab54acc 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -239,18 +239,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen CidStoreConfiguration Config; Config.RootDirectory = m_DataRoot / "cas"; - m_CidStore = std::make_unique<CidStore>(m_GcManager); - m_CidStore->Initialize(Config); + m_CidStores.insert_or_assign({}, std::make_unique<CidStore>(m_GcManager)); + m_CidStores.at({})->Initialize(Config); ZEN_INFO("instantiating project service"); - m_ProjectStore = new ProjectStore(*m_CidStore, + m_ProjectStore = new ProjectStore([this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); }, m_DataRoot / "projects", m_GcManager, *m_JobQueue, *m_OpenProcessCache, ProjectStore::Configuration{}); - m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr}); + m_HttpProjectService.reset(new HttpProjectService(m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr)); if (ServerOptions.WorksSpacesConfig.Enabled) { @@ -265,10 +265,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen if (ServerOptions.BuildStoreConfig.Enabled) { + CidStoreConfiguration BuildCidConfig; + BuildCidConfig.RootDirectory = m_DataRoot / "builds_cas"; + m_BuildCidStore = std::make_unique<CidStore>(m_GcManager); + m_BuildCidStore->Initialize(BuildCidConfig); + BuildStoreConfig BuildsCfg; BuildsCfg.RootDirectory = m_DataRoot / "builds"; BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; - m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager); + m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); } if (ServerOptions.StructuredCacheConfig.Enabled) @@ -357,17 +362,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_GcScheduler.Initialize(GcConfig); // Create and register admin interface last to make sure all is properly initialized - m_AdminService = - std::make_unique<HttpAdminService>(m_GcScheduler, - *m_JobQueue, - m_CacheStore.Get(), - m_CidStore.get(), - m_ProjectStore, - m_BuildStore.get(), - HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, - .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", - .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, - ServerOptions); + m_AdminService = std::make_unique<HttpAdminService>( + m_GcScheduler, + *m_JobQueue, + m_CacheStore.Get(), + [this]() { Flush(); }, + HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, + .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", + .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, + ServerOptions); m_Http->RegisterService(*m_AdminService); return EffectiveBasePort; @@ -419,10 +422,10 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); ValidationResult != CbValidateError::None) { - ZEN_WARN("Manifest validation failed: {}, state will be wiped", uint32_t(ValidationResult)); + ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult)); WipeState = true; - WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, uint32_t(ValidationResult)); + WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult)); } else { @@ -547,6 +550,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) Config.AllowAutomaticCreationOfNamespaces = true; Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; + for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs) { const std::string& BucketName = It.first; @@ -554,7 +558,8 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, .PayloadAlignment = ZenBucketConfig.PayloadAlignment, .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, - .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold}; + .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold, + .LimitOverwrites = ZenBucketConfig.LimitOverwrites}; Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); } Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, @@ -564,6 +569,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites; Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; @@ -587,7 +593,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount); } - m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore); + m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore); m_UpstreamService = std::make_unique<HttpUpstreamService>(*m_UpstreamCache, *m_AuthMgr); m_UpstreamCache->Initialize(); @@ -651,19 +657,24 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) } } - m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(*m_CacheStore, - *m_CidStore, - m_StatsService, - m_StatusService, - *m_UpstreamCache, - m_GcManager.GetDiskWriteBlocker(), - *m_OpenProcessCache); + m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>( + *m_CacheStore, + [this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); }, + m_StatsService, + m_StatusService, + *m_UpstreamCache, + m_GcManager.GetDiskWriteBlocker(), + *m_OpenProcessCache); m_Http->RegisterService(*m_StructuredCacheService); m_Http->RegisterService(*m_UpstreamService); m_StatsReporter.AddProvider(m_CacheStore.Get()); - m_StatsReporter.AddProvider(m_CidStore.get()); + for (const auto& It : m_CidStores) + { + m_StatsReporter.AddProvider(It.second.get()); + } + m_StatsReporter.AddProvider(m_BuildCidStore.get()); } void @@ -836,6 +847,7 @@ ZenServer::Cleanup() m_BuildStoreService.reset(); m_BuildStore = {}; + m_BuildCidStore.reset(); m_StructuredCacheService.reset(); m_UpstreamService.reset(); @@ -847,7 +859,7 @@ ZenServer::Cleanup() m_Workspaces.reset(); m_HttpProjectService.reset(); m_ProjectStore = {}; - m_CidStore.reset(); + m_CidStores.clear(); m_AuthService.reset(); m_AuthMgr.reset(); m_Http = {}; @@ -1028,37 +1040,21 @@ ZenServer::CheckOwnerPid() } void -ZenServer::ScrubStorage() -{ - Stopwatch Timer; - ZEN_INFO("Storage validation STARTING"); - - WorkerThreadPool ThreadPool{1, "Scrub"}; - ScrubContext Ctx{ThreadPool}; - m_CidStore->ScrubStorage(Ctx); - m_ProjectStore->ScrubStorage(Ctx); - m_StructuredCacheService->ScrubStorage(Ctx); - - const uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); - - ZEN_INFO("Storage validation DONE in {}, ({} in {} chunks - {})", - NiceTimeSpanMs(ElapsedTimeMs), - NiceBytes(Ctx.ScrubbedBytes()), - Ctx.ScrubbedChunks(), - NiceByteRate(Ctx.ScrubbedBytes(), ElapsedTimeMs)); -} - -void ZenServer::Flush() { - if (m_CidStore) - m_CidStore->Flush(); + for (auto& It : m_CidStores) + { + It.second->Flush(); + } if (m_StructuredCacheService) m_StructuredCacheService->Flush(); if (m_ProjectStore) m_ProjectStore->Flush(); + + if (m_BuildCidStore) + m_BuildCidStore->Flush(); } void diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 5cfa04ba1..0a17446ae 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -82,7 +82,6 @@ public: void CheckStateExitFlag(); void CheckOwnerPid(); bool UpdateProcessMonitor(); - void ScrubStorage(); void Flush(); virtual void HandleStatusRequest(HttpServerRequest& Request) override; @@ -116,19 +115,20 @@ private: inline void SetNewState(ServerState NewState) { m_CurrentState = NewState; } static std::string_view ToString(ServerState Value); - StatsReporter m_StatsReporter; - Ref<HttpServer> m_Http; - std::unique_ptr<AuthMgr> m_AuthMgr; - std::unique_ptr<HttpAuthService> m_AuthService; - HttpStatusService m_StatusService; - HttpStatsService m_StatsService; - GcManager m_GcManager; - GcScheduler m_GcScheduler{m_GcManager}; - std::unique_ptr<CidStore> m_CidStore; - Ref<ZenCacheStore> m_CacheStore; - std::unique_ptr<OpenProcessCache> m_OpenProcessCache; - HttpTestService m_TestService; - std::unique_ptr<BuildStore> m_BuildStore; + StatsReporter m_StatsReporter; + Ref<HttpServer> m_Http; + std::unique_ptr<AuthMgr> m_AuthMgr; + std::unique_ptr<HttpAuthService> m_AuthService; + HttpStatusService m_StatusService; + HttpStatsService m_StatsService; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + tsl::robin_map<std::string, std::unique_ptr<CidStore>> m_CidStores; + Ref<ZenCacheStore> m_CacheStore; + std::unique_ptr<OpenProcessCache> m_OpenProcessCache; + HttpTestService m_TestService; + std::unique_ptr<CidStore> m_BuildCidStore; + std::unique_ptr<BuildStore> m_BuildStore; #if ZEN_WITH_TESTS HttpTestingService m_TestingService; diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 20dc55bca..1b2cf036b 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -3,6 +3,7 @@ #include <zenstore/buildstore/buildstore.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compress.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/memory/llm.h> @@ -20,7 +21,6 @@ ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_WITH_TESTS # include <zencore/compactbinarybuilder.h> -# include <zencore/compress.h> # include <zencore/testing.h> # include <zencore/testutils.h> # include <zenutil/workerpools.h> @@ -45,7 +45,7 @@ namespace blobstore::impl { const char* LogExtension = ".slog"; const char* AccessTimeExtension = ".zacs"; - const uint32_t ManifestVersion = (1 << 16) | (0 << 8) | (0); + const uint32_t ManifestVersion = (2 << 16) | (0 << 8) | (0); std::filesystem::path GetManifestPath(const std::filesystem::path& RootDirectory) { @@ -106,13 +106,11 @@ namespace blobstore::impl { } // namespace blobstore::impl -BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) +BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc, CidStore& BlobStore) : m_Log(logging::Get("builds")) , m_Config(Config) , m_Gc(Gc) -, m_LargeBlobStore(m_Gc) -, m_SmallBlobStore(Gc) -, m_MetadataBlockStore() +, m_BlobStore(BlobStore) { ZEN_TRACE_CPU("BuildStore::BuildStore"); ZEN_MEMSCOPE(GetBuildstoreTag()); @@ -170,75 +168,16 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) ManifestWriter.AddDateTime("createdAt", DateTime::Now()); TemporaryFile::SafeWriteFile(ManifestPath, ManifestWriter.Save().GetBuffer().AsIoBuffer()); } - m_LargeBlobStore.Initialize(Config.RootDirectory / "file_cas", IsNew); - m_SmallBlobStore.Initialize(Config.RootDirectory, - "blob_cas", - m_Config.SmallBlobBlockStoreMaxBlockSize, - m_Config.SmallBlobBlockStoreAlignement, - IsNew); - m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20); - - BlockStore::BlockIndexSet KnownBlocks; - for (const BlobEntry& Blob : m_BlobEntries) - { - if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex) - { - const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex]; - KnownBlocks.insert(Metadata.Location.BlockIndex); - } - } - BlockStore::BlockIndexSet MissingBlocks = m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks); m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite); m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite); - if (!MissingBlocks.empty()) - { - std::vector<MetadataDiskEntry> MissingMetadatas; - for (auto& It : m_BlobLookup) - { - const IoHash& BlobHash = It.first; - const BlobIndex ReadBlobIndex = It.second; - const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; - if (ReadBlobEntry.Metadata) - { - const MetadataEntry& MetaData = m_MetadataEntries[ReadBlobEntry.Metadata]; - if (MissingBlocks.contains(MetaData.Location.BlockIndex)) - { - MissingMetadatas.push_back( - MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = BlobHash}); - MissingMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone; - m_MetadataEntries[ReadBlobEntry.Metadata] = {}; - m_BlobEntries[ReadBlobIndex].Metadata = {}; - } - } - } - ZEN_ASSERT(!MissingMetadatas.empty()); - - for (const MetadataDiskEntry& Entry : MissingMetadatas) - { - auto It = m_BlobLookup.find(Entry.BlobHash); - ZEN_ASSERT(It != m_BlobLookup.end()); - - const BlobIndex ReadBlobIndex = It->second; - const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; - if (!ReadBlobEntry.Payload) - { - m_BlobLookup.erase(It); - } - } - m_MetadatalogFile.Append(MissingMetadatas); - CompactState(); - } - m_Gc.AddGcReferencer(*this); m_Gc.AddGcReferenceLocker(*this); - m_Gc.AddGcStorage(this); } catch (const std::exception& Ex) { ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what()); - m_Gc.RemoveGcStorage(this); m_Gc.RemoveGcReferenceLocker(*this); m_Gc.RemoveGcReferencer(*this); } @@ -249,7 +188,6 @@ BuildStore::~BuildStore() try { ZEN_TRACE_CPU("BuildStore::~BuildStore"); - m_Gc.RemoveGcStorage(this); m_Gc.RemoveGcReferenceLocker(*this); m_Gc.RemoveGcReferencer(*this); Flush(); @@ -280,21 +218,12 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) } } - uint64_t PayloadSize = Payload.GetSize(); - PayloadEntry Entry; - if (Payload.GetSize() > m_Config.SmallBlobBlockStoreMaxBlockEmbedSize) - { - CasStore::InsertResult Result = m_LargeBlobStore.InsertChunk(Payload, BlobHash); - ZEN_UNUSED(Result); - Entry = PayloadEntry(PayloadEntry::kStandalone, PayloadSize); - } - else - { - CasStore::InsertResult Result = m_SmallBlobStore.InsertChunk(Payload, BlobHash); - ZEN_UNUSED(Result); - Entry = PayloadEntry(0, PayloadSize); - } + uint64_t PayloadSize = Payload.GetSize(); + CidStore::InsertResult Result = m_BlobStore.AddChunk(Payload, BlobHash); + PayloadEntry Entry = PayloadEntry(0, PayloadSize); + ; + IoHash MetadataHash; { RwLock::ExclusiveLockScope _(m_Lock); if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) @@ -310,6 +239,10 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) Blob.Payload = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size())); m_PayloadEntries.push_back(Entry); } + if (Blob.Metadata) + { + MetadataHash = m_MetadataEntries[Blob.Metadata].MetadataHash; + } Blob.LastAccessTime = GcClock::TickCount(); } else @@ -322,6 +255,16 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); m_BlobLookup.insert({BlobHash, NewBlobIndex}); } + + m_LastAccessTimeUpdateCount++; + if (m_TrackedBlobKeys) + { + m_TrackedBlobKeys->push_back(BlobHash); + if (MetadataHash != IoHash::Zero) + { + m_TrackedBlobKeys->push_back(BlobHash); + } + } } m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); m_LastAccessTimeUpdateCount++; @@ -340,21 +283,9 @@ BuildStore::GetBlob(const IoHash& BlobHash) Blob.LastAccessTime = GcClock::TickCount(); if (Blob.Payload) { - const PayloadEntry& Entry = m_PayloadEntries[Blob.Payload]; - const bool IsStandalone = (Entry.GetFlags() & PayloadEntry::kStandalone) != 0; Lock.ReleaseNow(); + IoBuffer Chunk = m_BlobStore.FindChunkByCid(BlobHash); - IoBuffer Chunk; - if (IsStandalone) - { - ZEN_TRACE_CPU("GetLarge"); - Chunk = m_LargeBlobStore.FindChunk(BlobHash); - } - else - { - ZEN_TRACE_CPU("GetSmall"); - Chunk = m_SmallBlobStore.FindChunk(BlobHash); - } if (Chunk) { Chunk.SetContentType(ZenContentType::kCompressedBinary); @@ -362,7 +293,7 @@ BuildStore::GetBlob(const IoHash& BlobHash) } else { - ZEN_WARN("Inconsistencies in build store, {} is in index but not {}", BlobHash, IsStandalone ? "on disk" : "in block"); + ZEN_WARN("Inconsistencies in build store, {} is in index but not in blob store", BlobHash); } } } @@ -381,10 +312,10 @@ BuildStore::BlobsExists(std::span<const IoHash> BlobHashes) { if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) { - const BlobIndex ExistingBlobIndex = It->second; - BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; - bool HasPayload = !!Blob.Payload; - bool HasMetadata = !!Blob.Metadata; + const BlobIndex ExistingBlobIndex = It->second; + const BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + bool HasPayload = !!Blob.Payload; + bool HasMetadata = !!Blob.Metadata; Result.push_back(BlobExistsResult{.HasBody = HasPayload, .HasMetadata = HasMetadata}); } else @@ -396,20 +327,82 @@ BuildStore::BlobsExists(std::span<const IoHash> BlobHashes) } void -BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas) +BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> Metadatas, WorkerThreadPool* OptionalWorkerPool) { ZEN_TRACE_CPU("BuildStore::PutMetadatas"); ZEN_MEMSCOPE(GetBuildstoreTag()); - size_t WriteBlobIndex = 0; - m_MetadataBlockStore.WriteChunks(MetaDatas, m_Config.MetadataBlockStoreAlignement, [&](std::span<BlockStoreLocation> Locations) { + std::vector<IoHash> MetadataHashes; + std::vector<IoBuffer> CompressedMetadataBuffers; + + auto CompressOne = [&BlobHashes, &MetadataHashes, &CompressedMetadataBuffers](const IoBuffer& Buffer, size_t Index) { + if (Buffer.GetContentType() == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + if (!CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize)) + { + throw std::runtime_error( + fmt::format("Invalid compressed buffer provided when storing metadata for blob {}", BlobHashes[Index])); + } + else + { + CompressedMetadataBuffers[Index] = Buffer; + MetadataHashes[Index] = RawHash; + } + } + else + { + CompressedBuffer Compressed = + CompressedBuffer::Compress(SharedBuffer(Buffer), OodleCompressor::Mermaid, OodleCompressionLevel::None); + MetadataHashes[Index] = Compressed.DecodeRawHash(); + CompressedMetadataBuffers[Index] = std::move(Compressed.GetCompressed()).Flatten().AsIoBuffer(); + CompressedMetadataBuffers[Index].SetContentType(ZenContentType::kCompressedBinary); + } + }; + + MetadataHashes.resize(Metadatas.size()); + CompressedMetadataBuffers.resize(Metadatas.size()); + if (OptionalWorkerPool) + { + std::atomic<bool> AbortFlag; + std::atomic<bool> PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); + for (size_t Index = 0; Index < Metadatas.size(); Index++) + { + Work.ScheduleWork( + *OptionalWorkerPool, + [Index, &BlobHashes, &Metadatas, &MetadataHashes, &CompressedMetadataBuffers, &CompressOne](std::atomic<bool>&) { + const IoBuffer& Buffer = Metadatas[Index]; + CompressOne(Buffer, Index); + }, + {}); + } + Work.Wait(); + } + else + { + for (size_t Index = 0; Index < Metadatas.size(); Index++) + { + const IoBuffer& Buffer = Metadatas[Index]; + CompressOne(Buffer, Index); + } + } + + std::vector<MetadataDiskEntry> AddedMetadataEntries; + AddedMetadataEntries.reserve(MetadataHashes.size()); + + std::vector<CidStore::InsertResult> InsertResults = m_BlobStore.AddChunks(CompressedMetadataBuffers, MetadataHashes); + ZEN_UNUSED(InsertResults); + { RwLock::ExclusiveLockScope _(m_Lock); - for (size_t LocationIndex = 0; LocationIndex < Locations.size(); LocationIndex++) + for (size_t Index = 0; Index < BlobHashes.size(); Index++) { - const IoBuffer& Data = MetaDatas[WriteBlobIndex]; - const IoHash& BlobHash = BlobHashes[WriteBlobIndex]; - const BlockStoreLocation& Location = Locations[LocationIndex]; + const ZenContentType ContentType = Metadatas[Index].GetContentType(); + const IoHash& BlobHash = BlobHashes[Index]; + const IoHash& MetadataHash = MetadataHashes[Index]; + const IoBuffer& Metadata = CompressedMetadataBuffers[Index]; - MetadataEntry Entry = {.Location = Location, .ContentType = Data.GetContentType(), .Flags = 0}; + MetadataEntry Entry(MetadataHash, Metadata.GetSize(), ContentType, 0); if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) { @@ -435,17 +428,16 @@ BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoB m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); m_BlobLookup.insert({BlobHash, NewBlobIndex}); } - - m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); - m_LastAccessTimeUpdateCount++; - WriteBlobIndex++; - if (m_TrackedCacheKeys) + if (m_TrackedBlobKeys) { - m_TrackedCacheKeys->insert(BlobHash); + m_TrackedBlobKeys->push_back(BlobHash); + m_TrackedBlobKeys->push_back(MetadataHash); } + AddedMetadataEntries.push_back(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); } - }); + } + m_MetadatalogFile.Append(AddedMetadataEntries); } std::vector<IoBuffer> @@ -453,9 +445,9 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O { ZEN_TRACE_CPU("BuildStore::GetMetadatas"); ZEN_MEMSCOPE(GetBuildstoreTag()); - std::vector<BlockStoreLocation> MetaLocations; - std::vector<size_t> MetaLocationResultIndexes; - MetaLocations.reserve(BlobHashes.size()); + std::vector<IoHash> MetadataHashes; + std::vector<size_t> MetaLocationResultIndexes; + MetadataHashes.reserve(BlobHashes.size()); MetaLocationResultIndexes.reserve(BlobHashes.size()); tsl::robin_set<uint32_t> ReferencedBlocks; @@ -475,10 +467,9 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O if (ExistingBlobEntry.Metadata) { const MetadataEntry& ExistingMetadataEntry = m_MetadataEntries[ExistingBlobEntry.Metadata]; - MetaLocations.push_back(ExistingMetadataEntry.Location); + MetadataHashes.push_back(ExistingMetadataEntry.MetadataHash); MetaLocationResultIndexes.push_back(Index); - ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex); - ResultContentTypes[Index] = ExistingMetadataEntry.ContentType; + ResultContentTypes[Index] = ExistingMetadataEntry.GetContentType(); } ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount()); m_LastAccessTimeUpdateCount++; @@ -486,100 +477,35 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O } } - auto DoOneBlock = [this](std::span<const BlockStoreLocation> MetaLocations, - std::span<const size_t> MetaLocationResultIndexes, - std::span<const size_t> ChunkIndexes, - std::vector<IoBuffer>& Result) { - if (ChunkIndexes.size() < 4) - { - for (size_t ChunkIndex : ChunkIndexes) + m_BlobStore.IterateChunks( + MetadataHashes, + [this, &BlobHashes, &MetadataHashes, &MetaLocationResultIndexes, &Result](size_t Index, const IoBuffer& Payload) { + if (Payload) { - IoBuffer Chunk = m_MetadataBlockStore.TryGetChunk(MetaLocations[ChunkIndex]); - if (Chunk) - { - size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; - Result[ResultIndex] = std::move(Chunk); - } - } - return true; - } - return m_MetadataBlockStore.IterateBlock( - MetaLocations, - ChunkIndexes, - [&MetaLocationResultIndexes, &Result](size_t ChunkIndex, const void* Data, uint64_t Size) { - if (Data != nullptr) + size_t ResultIndex = MetaLocationResultIndexes[Index]; + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer CompressedBuffer = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize); + if (CompressedBuffer) { - size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; - Result[ResultIndex] = IoBuffer(IoBuffer::Clone, Data, Size); - } - return true; - }, - [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { - size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; - Result[ResultIndex] = File.GetChunk(Offset, Size); - return true; - }, - 8u * 1024u); - }; - - if (!MetaLocations.empty()) - { - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; - ParallelWork Work(AbortFlag, PauseFlag); - - try - { - m_MetadataBlockStore.IterateChunks( - MetaLocations, - [this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock]( - uint32_t BlockIndex, - std::span<const size_t> ChunkIndexes) -> bool { - ZEN_UNUSED(BlockIndex); - if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) + IoBuffer Decompressed = CompressedBuffer.DecompressToComposite().Flatten().AsIoBuffer(); + if (Decompressed) { - return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); + Result[ResultIndex] = std::move(Decompressed); } else { - ZEN_ASSERT(OptionalWorkerPool != nullptr); - std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); - Work.ScheduleWork( - *OptionalWorkerPool, - [this, - &Result, - &MetaLocations, - &MetaLocationResultIndexes, - DoOneBlock, - ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic<bool>& AbortFlag) { - if (AbortFlag) - { - return; - } - try - { - if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result)) - { - AbortFlag.store(true); - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); - } - }); - return !Work.IsAborted(); + ZEN_WARN("Metadata {} for blob {} is malformed (not a compressed binary format)", + MetadataHashes[ResultIndex], + BlobHashes[ResultIndex]); } - }); - } - catch (const std::exception& Ex) - { - AbortFlag.store(true); - ZEN_WARN("Failed iterating block metadata chunks in {}. Reason: '{}'", m_Config.RootDirectory, Ex.what()); - } + } + } + return true; + }, + OptionalWorkerPool, + 8u * 1024u); - Work.Wait(); - } for (size_t Index = 0; Index < Result.size(); Index++) { if (Result[Index]) @@ -600,9 +526,7 @@ BuildStore::Flush() const auto _ = MakeGuard( [&] { ZEN_INFO("Flushed build store at {} in {}", m_Config.RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - m_LargeBlobStore.Flush(); - m_SmallBlobStore.Flush(); - m_MetadataBlockStore.Flush(false); + m_BlobStore.Flush(); m_PayloadlogFile.Flush(); m_MetadatalogFile.Flush(); @@ -636,22 +560,14 @@ BuildStore::GetStorageStats() const { const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload]; uint64_t Size = Payload.GetSize(); - if ((Payload.GetFlags() & PayloadEntry::kStandalone) != 0) - { - Result.LargeBlobCount++; - Result.LargeBlobBytes += Size; - } - else - { - Result.SmallBlobCount++; - Result.SmallBlobBytes += Size; - } + Result.BlobCount++; + Result.BlobBytes += Size; } if (ReadBlobEntry.Metadata) { const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata]; Result.MetadataCount++; - Result.MetadataByteCount += Metadata.Location.Size; + Result.MetadataByteCount += Metadata.GetSize(); } } } @@ -882,10 +798,9 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy CasLog.Replay( [&](const MetadataDiskEntry& Record) { std::string InvalidEntryReason; - if (Record.Entry.Flags & MetadataEntry::kTombStone) + if (Record.Entry.GetFlags() & MetadataEntry::kTombStone) { // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation - // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) { if (!m_BlobEntries[ExistingIt->second].Payload) @@ -1058,7 +973,7 @@ BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason = fmt::format("Invalid blob hash {}", Entry.BlobHash.ToHexString()); return false; } - if (Entry.Entry.GetFlags() & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone)) + if (Entry.Entry.GetFlags() & ~(PayloadEntry::kTombStone)) { OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.GetFlags(), Entry.BlobHash.ToHexString()); return false; @@ -1083,30 +998,20 @@ BuildStore::ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::strin OutReason = fmt::format("Invalid blob hash {} for meta entry", Entry.BlobHash.ToHexString()); return false; } - if (Entry.Entry.Location.Size == 0) + if (Entry.Entry.GetSize() == 0) { - OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.Location.Size); + OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.GetSize()); return false; } - if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0) - { - OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); - return false; - } - if (Entry.Entry.Flags & MetadataEntry::kTombStone) + if (Entry.Entry.GetFlags() & MetadataEntry::kTombStone) { return true; } - if (Entry.Entry.ContentType == ZenContentType::kCOUNT) + if (Entry.Entry.GetContentType() == ZenContentType::kCOUNT) { OutReason = fmt::format("Invalid content type for meta entry {}", Entry.BlobHash.ToHexString()); return false; } - if (Entry.Reserved1 != 0 || Entry.Reserved2 != 0 || Entry.Reserved3 != 0 || Entry.Reserved4 != 0) - { - OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); - return false; - } return true; } @@ -1114,31 +1019,76 @@ class BuildStoreGcReferenceChecker : public GcReferenceChecker { public: BuildStoreGcReferenceChecker(BuildStore& Store) : m_Store(Store) {} + ~BuildStoreGcReferenceChecker() + { + try + { + m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedBlobKeys.reset(); }); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~BuildStoreGcReferenceChecker threw exception: '{}'", Ex.what()); + } + } virtual std::string GetGcName(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); } - virtual void PreCache(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); } - - virtual void UpdateLockedState(GcCtx& Ctx) override + virtual void PreCache(GcCtx& Ctx) override { - ZEN_TRACE_CPU("Builds::UpdateLockedState"); - ZEN_MEMSCOPE(GetBuildstoreTag()); + ZEN_TRACE_CPU("Builds::PreCache"); auto Log = [&Ctx]() { return Ctx.Logger; }; - m_References.reserve(m_Store.m_BlobLookup.size()); - for (const auto& It : m_Store.m_BlobLookup) + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: builds [PRECACHE] '{}': found {} references in {}", + m_Store.m_Config.RootDirectory, + m_PrecachedReferences.size(), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedBlobKeys = std::make_unique<std::vector<IoHash>>(); }); + { - const BuildStore::BlobIndex ExistingBlobIndex = It.second; - if (m_Store.m_BlobEntries[ExistingBlobIndex].Payload) + m_PrecachedReferences.reserve(m_Store.m_BlobLookup.size()); + RwLock::SharedLockScope __(m_Store.m_Lock); + for (const auto& It : m_Store.m_BlobLookup) { - m_References.push_back(It.first); + const BuildStore::BlobIndex ExistingBlobIndex = It.second; + const BuildStore::BlobEntry& Entry = m_Store.m_BlobEntries[ExistingBlobIndex]; + if (Entry.Payload) + { + m_PrecachedReferences.push_back(It.first); + } + if (Entry.Metadata) + { + const BuildStore::MetadataEntry& MetadataEntry = m_Store.m_MetadataEntries[Entry.Metadata]; + m_PrecachedReferences.push_back(MetadataEntry.MetadataHash); + } } } - FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", "buildstore"), m_References); + FilterReferences(Ctx, fmt::format("buildstore [PRECACHE] '{}'", m_Store.m_Config.RootDirectory), m_PrecachedReferences); + } + + virtual void UpdateLockedState(GcCtx& Ctx) override + { + ZEN_TRACE_CPU("Builds::UpdateLockedState"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + ZEN_ASSERT(m_Store.m_TrackedBlobKeys); + + m_AddedReferences = std::move(*m_Store.m_TrackedBlobKeys); + + FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", m_Store.m_Config.RootDirectory), m_AddedReferences); } virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override @@ -1165,14 +1115,16 @@ public: NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids); + std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_PrecachedReferences, IoCids); + UnusedReferences = KeepUnusedReferences(m_AddedReferences, UnusedReferences); UsedCount = IoCids.size() - UnusedReferences.size(); return UnusedReferences; } private: BuildStore& m_Store; - std::vector<IoHash> m_References; + std::vector<IoHash> m_PrecachedReferences; + std::vector<IoHash> m_AddedReferences; }; std::string @@ -1184,201 +1136,6 @@ BuildStore::GetGcName(GcCtx& Ctx) return fmt::format("buildstore: '{}'", m_Config.RootDirectory.string()); } -class BuildStoreGcCompator : public GcStoreCompactor -{ - using BlobEntry = BuildStore::BlobEntry; - using PayloadEntry = BuildStore::PayloadEntry; - using MetadataEntry = BuildStore::MetadataEntry; - using MetadataDiskEntry = BuildStore::MetadataDiskEntry; - using BlobIndex = BuildStore::BlobIndex; - using PayloadIndex = BuildStore::PayloadIndex; - using MetadataIndex = BuildStore::MetadataIndex; - -public: - BuildStoreGcCompator(BuildStore& Store, std::vector<IoHash>&& RemovedBlobs) : m_Store(Store), m_RemovedBlobs(std::move(RemovedBlobs)) {} - - virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override - { - ZEN_UNUSED(ClaimDiskReserveCallback); - ZEN_TRACE_CPU("Builds::CompactStore"); - ZEN_MEMSCOPE(GetBuildstoreTag()); - - auto Log = [&Ctx]() { return Ctx.Logger; }; - - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: buildstore [COMPACT] '{}': RemovedDisk: {} in {}", - m_Store.m_Config.RootDirectory, - NiceBytes(Stats.RemovedDisk), - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - - const auto __ = MakeGuard([&] { m_Store.Flush(); }); - - if (!m_RemovedBlobs.empty()) - { - if (Ctx.Settings.CollectSmallObjects) - { - m_Store.m_Lock.WithExclusiveLock([this]() { m_Store.m_TrackedCacheKeys = std::make_unique<HashSet>(); }); - auto __ = MakeGuard([this]() { m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedCacheKeys.reset(); }); }); - - BlockStore::BlockUsageMap BlockUsage; - { - RwLock::SharedLockScope __(m_Store.m_Lock); - - for (auto LookupIt : m_Store.m_BlobLookup) - { - const BlobIndex ReadBlobIndex = LookupIt.second; - const BlobEntry& ReadBlobEntry = m_Store.m_BlobEntries[ReadBlobIndex]; - - if (ReadBlobEntry.Metadata) - { - const MetadataEntry& ReadMetadataEntry = m_Store.m_MetadataEntries[ReadBlobEntry.Metadata]; - - uint32_t BlockIndex = ReadMetadataEntry.Location.BlockIndex; - uint64_t ChunkSize = RoundUp(ReadMetadataEntry.Location.Size, m_Store.m_Config.MetadataBlockStoreAlignement); - - if (auto BlockUsageIt = BlockUsage.find(BlockIndex); BlockUsageIt != BlockUsage.end()) - { - BlockStore::BlockUsageInfo& Info = BlockUsageIt.value(); - Info.EntryCount++; - Info.DiskUsage += ChunkSize; - } - else - { - BlockUsage.insert_or_assign(BlockIndex, - BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1}); - } - } - } - } - - BlockStore::BlockEntryCountMap BlocksToCompact = m_Store.m_MetadataBlockStore.GetBlocksToCompact(BlockUsage, 90); - BlockStoreCompactState BlockCompactState; - std::vector<IoHash> BlockCompactStateKeys; - BlockCompactState.IncludeBlocks(BlocksToCompact); - - if (BlocksToCompact.size() > 0) - { - { - RwLock::SharedLockScope ___(m_Store.m_Lock); - for (const auto& Entry : m_Store.m_BlobLookup) - { - BlobIndex Index = Entry.second; - - if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) - { - if (BlockCompactState.AddKeepLocation(m_Store.m_MetadataEntries[Meta].Location)) - { - BlockCompactStateKeys.push_back(Entry.first); - } - } - } - } - - if (Ctx.Settings.IsDeleteMode) - { - if (Ctx.Settings.Verbose) - { - ZEN_INFO("GCV2: buildstore [COMPACT] '{}': compacting {} blocks", - m_Store.m_Config.RootDirectory, - BlocksToCompact.size()); - } - - m_Store.m_MetadataBlockStore.CompactBlocks( - BlockCompactState, - m_Store.m_Config.MetadataBlockStoreAlignement, - [&](const BlockStore::MovedChunksArray& MovedArray, - const BlockStore::ChunkIndexArray& ScrubbedArray, - uint64_t FreedDiskSpace) { - std::vector<MetadataDiskEntry> MovedEntries; - MovedEntries.reserve(MovedArray.size()); - RwLock::ExclusiveLockScope _(m_Store.m_Lock); - for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray) - { - size_t ChunkIndex = Moved.first; - const IoHash& Key = BlockCompactStateKeys[ChunkIndex]; - - ZEN_ASSERT(m_Store.m_TrackedCacheKeys); - if (m_Store.m_TrackedCacheKeys->contains(Key)) - { - continue; - } - - if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end()) - { - const BlobIndex Index = It->second; - - if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) - { - m_Store.m_MetadataEntries[Meta].Location = Moved.second; - MovedEntries.push_back( - MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key}); - } - } - } - - for (size_t Scrubbed : ScrubbedArray) - { - const IoHash& Key = BlockCompactStateKeys[Scrubbed]; - if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end()) - { - const BlobIndex Index = It->second; - - if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) - { - MovedEntries.push_back( - MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key}); - MovedEntries.back().Entry.Flags |= MetadataEntry::kTombStone; - m_Store.m_MetadataEntries[Meta] = {}; - m_Store.m_BlobEntries[Index].Metadata = {}; - } - } - } - - m_Store.m_MetadatalogFile.Append(MovedEntries); - - Stats.RemovedDisk += FreedDiskSpace; - if (Ctx.IsCancelledFlag.load()) - { - return false; - } - return true; - }, - ClaimDiskReserveCallback, - fmt::format("GCV2: buildstore [COMPACT] '{}': ", m_Store.m_Config.RootDirectory)); - } - else - { - if (Ctx.Settings.Verbose) - { - ZEN_INFO("GCV2: buildstore [COMPACT] '{}': skipped compacting of {} eligible blocks", - m_Store.m_Config.RootDirectory, - BlocksToCompact.size()); - } - } - } - } - } - } - - virtual std::string GetGcName(GcCtx& Ctx) override - { - ZEN_UNUSED(Ctx); - ZEN_MEMSCOPE(GetBuildstoreTag()); - - return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); - } - -private: - BuildStore& m_Store; - const std::vector<IoHash> m_RemovedBlobs; -}; - GcStoreCompactor* BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) { @@ -1413,10 +1170,9 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) uint64_t BlobSize = 0; }; - bool DiskSizeExceeded = false; - const uint64_t CurrentDiskSize = - m_LargeBlobStore.StorageSize().DiskSize + m_SmallBlobStore.StorageSize().DiskSize + m_MetadataBlockStore.TotalSize(); - if (CurrentDiskSize > m_Config.MaxDiskSpaceLimit) + bool DiskSizeExceeded = false; + const uint64_t CurrentBlobsDiskSize = m_BlobStore.TotalSize().TotalSize; + if ((m_Config.MaxDiskSpaceLimit > 0) && (CurrentBlobsDiskSize > m_Config.MaxDiskSpaceLimit)) { DiskSizeExceeded = true; } @@ -1444,14 +1200,14 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) if (ReadBlobEntry.Metadata) { const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata]; - Size += Metadata.Location.Size; + Size += Metadata.GetSize(); } const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; if (AccessTick < ExpireTicks) { ExpiredBlobs.push_back(It.first); - ExpiredDataSize += ExpiredDataSize; + ExpiredDataSize += Size; } else if (DiskSizeExceeded) { @@ -1469,7 +1225,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) const uint64_t NewSizeLimit = m_Config.MaxDiskSpaceLimit - (m_Config.MaxDiskSpaceLimit >> 4); // Remove a bit more than just below the limit so we have some space to grow - if ((CurrentDiskSize - ExpiredDataSize) > NewSizeLimit) + if ((CurrentBlobsDiskSize - ExpiredDataSize) > NewSizeLimit) { std::vector<size_t> NonExpiredOrder; NonExpiredOrder.resize(NonExpiredBlobSizeInfos.size()); @@ -1487,7 +1243,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) while (It != NonExpiredOrder.end()) { const SizeInfo& Info = NonExpiredBlobSizeInfos[*It]; - if ((CurrentDiskSize - ExpiredDataSize) < NewSizeLimit) + if ((CurrentBlobsDiskSize - ExpiredDataSize) < NewSizeLimit) { break; } @@ -1539,7 +1295,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) { RemoveMetadatas.push_back( MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = ExpiredBlob}); - RemoveMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone; + RemoveMetadatas.back().Entry.AddFlag(MetadataEntry::kTombStone); m_MetadataEntries[ReadBlobEntry.Metadata] = {}; m_BlobEntries[ReadBlobIndex].Metadata = {}; } @@ -1568,7 +1324,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) CompactState(); } - return new BuildStoreGcCompator(*this, std::move(RemovedBlobs)); + return nullptr; } std::vector<GcReferenceChecker*> @@ -1595,21 +1351,6 @@ BuildStore::LockState(GcCtx& Ctx) return Locks; } -void -BuildStore::ScrubStorage(ScrubContext& ScrubCtx) -{ - ZEN_UNUSED(ScrubCtx); - // TODO -} - -GcStorageSize -BuildStore::StorageSize() const -{ - GcStorageSize Result; - Result.DiskSize = m_MetadataBlockStore.TotalSize(); - return Result; -} - /* ___________ __ \__ ___/___ _______/ |_ ______ @@ -1630,8 +1371,10 @@ TEST_CASE("BuildStore.Blobs") std::vector<IoHash> CompressedBlobsHashes; { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (size_t I = 0; I < 5; I++) { @@ -1658,10 +1401,13 @@ TEST_CASE("BuildStore.Blobs") IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); CHECK(IoHash::HashBuffer(Decompressed) == RawHash); } + BlobStore.Flush(); } { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (const IoHash& RawHash : CompressedBlobsHashes) { IoBuffer Payload = Store.GetBlob(RawHash); @@ -1689,8 +1435,10 @@ TEST_CASE("BuildStore.Blobs") } } { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (const IoHash& RawHash : CompressedBlobsHashes) { IoBuffer Payload = Store.GetBlob(RawHash); @@ -1709,7 +1457,7 @@ TEST_CASE("BuildStore.Blobs") } namespace blockstore::testing { - IoBuffer MakeMetaData(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues) + IoBuffer MakeMetadata(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues) { CbObjectWriter Writer; Writer.AddHash("rawHash"sv, BlobHash); @@ -1740,16 +1488,18 @@ TEST_CASE("BuildStore.Metadata") std::vector<IoHash> BlobHashes; std::vector<IoBuffer> MetaPayloads; { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (size_t I = 0; I < 5; I++) { BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I))); - MetaPayloads.push_back(MakeMetaData(BlobHashes.back(), {{"index", fmt::format("{}", I)}})); + MetaPayloads.push_back(MakeMetadata(BlobHashes.back(), {{"index", fmt::format("{}", I)}})); MetaPayloads.back().SetContentType(ZenContentType::kCbObject); } - Store.PutMetadatas(BlobHashes, MetaPayloads); + Store.PutMetadatas(BlobHashes, MetaPayloads, &WorkerPool); std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool); CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); @@ -1760,8 +1510,10 @@ TEST_CASE("BuildStore.Metadata") } } { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool); CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) @@ -1776,8 +1528,10 @@ TEST_CASE("BuildStore.Metadata") } std::vector<IoHash> CompressedBlobsHashes; { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (size_t I = 0; I < 5; I++) { IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); @@ -1805,14 +1559,16 @@ TEST_CASE("BuildStore.Metadata") std::vector<IoBuffer> BlobMetaPayloads; { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (const IoHash& BlobHash : CompressedBlobsHashes) { - BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.push_back(MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); } - Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, &WorkerPool); std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool); CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); @@ -1824,8 +1580,10 @@ TEST_CASE("BuildStore.Metadata") } { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool); CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); @@ -1847,14 +1605,16 @@ TEST_CASE("BuildStore.Metadata") for (const IoHash& BlobHash : CompressedBlobsHashes) { BlobMetaPayloads.push_back( - MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}})); + MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}})); BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); } - Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, &WorkerPool); } { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool); CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); @@ -1886,8 +1646,10 @@ TEST_CASE("BuildStore.GC") std::vector<IoHash> CompressedBlobsHashes; std::vector<IoBuffer> BlobMetaPayloads; { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (size_t I = 0; I < 5; I++) { IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); @@ -1900,14 +1662,16 @@ TEST_CASE("BuildStore.GC") } for (const IoHash& BlobHash : CompressedBlobsHashes) { - BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.push_back(MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); } - Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, nullptr); } { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); { GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1), @@ -1967,8 +1731,10 @@ TEST_CASE("BuildStore.SizeLimit") std::vector<IoHash> CompressedBlobsHashes; std::vector<IoBuffer> BlobMetaPayloads; { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); for (size_t I = 0; I < 64; I++) { IoBuffer Blob = CreateSemiRandomBlob(65537 + I * 7); @@ -1981,10 +1747,10 @@ TEST_CASE("BuildStore.SizeLimit") } for (const IoHash& BlobHash : CompressedBlobsHashes) { - BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.push_back(MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); } - Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, nullptr); { for (size_t I = 0; I < 64; I++) @@ -1997,8 +1763,10 @@ TEST_CASE("BuildStore.SizeLimit") } } { - GcManager Gc; - BuildStore Store(Config, Gc); + GcManager Gc; + CidStore BlobStore(Gc); + BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"}); + BuildStore Store(Config, Gc, BlobStore); { GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1), @@ -2023,7 +1791,7 @@ TEST_CASE("BuildStore.SizeLimit") CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash); } } - CHECK(DeletedBlobs == 50); + CHECK(DeletedBlobs == 53); std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 15a1c9650..cacbbd966 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -233,6 +233,28 @@ using namespace std::literals; namespace zen::cache::impl { +static bool +UpdateValueWithRawSizeAndHash(ZenCacheValue& Value) +{ + if ((Value.RawSize == 0) && (Value.RawHash == IoHash::Zero)) + { + if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary) + { + return CompressedBuffer::ValidateCompressedHeader(Value.Value, Value.RawHash, Value.RawSize); + } + else + { + Value.RawSize = Value.Value.GetSize(); + Value.RawHash = IoHash::HashBuffer(Value.Value); + return true; + } + } + else + { + return true; + } +} + class BucketManifestSerializer { using MetaDataIndex = ZenCacheDiskLayer::CacheBucket::MetaDataIndex; @@ -348,11 +370,20 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck Stopwatch Timer; const auto _ = MakeGuard([&] { ZEN_INFO("parsed store manifest '{}' in {}", ManifestPath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - const uint64_t Count = Manifest["Count"sv].AsUInt64(0); + const uint64_t Count = Manifest["Count"sv].AsUInt64(0); + CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView(); + if (KeyArray.Num() != Count) + { + ZEN_WARN("Mismatch in size between 'Keys' ({}) array size and 'Count' ({}) in {}, skipping metadata", + KeyArray.Num(), + Count, + ManifestPath); + return; + } + std::vector<PayloadIndex> KeysIndexes; KeysIndexes.reserve(Count); - CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView(); for (CbFieldView& KeyView : KeyArray) { if (auto It = Index.find(KeyView.AsHash()); It != Index.end()) @@ -367,19 +398,43 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck size_t KeyIndexOffset = 0; CbArrayView TimeStampArray = Manifest["Timestamps"].AsArrayView(); - for (CbFieldView& TimeStampView : TimeStampArray) + if (KeysIndexes.size() != TimeStampArray.Num()) { - const PayloadIndex KeyIndex = KeysIndexes[KeyIndexOffset++]; - if (KeyIndex) + ZEN_WARN("Mismatch in size between 'Keys' ({}) and 'Timestamps' ({}) arrays in {}, skipping timestamps", + KeysIndexes.size(), + TimeStampArray.Num(), + ManifestPath); + } + else + { + for (CbFieldView& TimeStampView : TimeStampArray) { - AccessTimes[KeyIndex] = TimeStampView.AsInt64(); + const PayloadIndex KeyIndex = KeysIndexes[KeyIndexOffset++]; + if (KeyIndex) + { + AccessTimes[KeyIndex] = TimeStampView.AsInt64(); + } } } KeyIndexOffset = 0; CbArrayView RawHashArray = Manifest["RawHash"].AsArrayView(); CbArrayView RawSizeArray = Manifest["RawSize"].AsArrayView(); - if (RawHashArray.Num() == RawSizeArray.Num()) + if (RawHashArray.Num() != KeysIndexes.size()) + { + ZEN_WARN("Mismatch in size between 'Keys' ({}) and 'RawHash' ({}) arrays in {}, skipping meta data", + KeysIndexes.size(), + RawHashArray.Num(), + ManifestPath); + } + else if (RawSizeArray.Num() != KeysIndexes.size()) + { + ZEN_WARN("Mismatch in size between 'Keys' ({}) and 'RawSize' ({}) arrays in {}, skipping meta data", + KeysIndexes.size(), + RawSizeArray.Num(), + ManifestPath); + } + else { auto RawHashIt = RawHashArray.CreateViewIterator(); auto RawSizeIt = RawSizeArray.CreateViewIterator(); @@ -404,10 +459,6 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck RawSizeIt++; } } - else - { - ZEN_WARN("Mismatch in size between 'RawHash' and 'RawSize' arrays in {}, skipping meta data", ManifestPath); - } } Oid @@ -747,6 +798,7 @@ ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager& Gc, m_Configuration.LargeObjectThreshold = Max(m_Configuration.LargeObjectThreshold, IoStoreDDCOverrideSize); } m_Gc.AddGcReferencer(*this); + m_Gc.AddGcStorage(this); } ZenCacheDiskLayer::CacheBucket::~CacheBucket() @@ -761,6 +813,7 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket() { ZEN_ERROR("~CacheBucket() failed with: ", Ex.what()); } + m_Gc.RemoveGcStorage(this); m_Gc.RemoveGcReferencer(*this); } @@ -1286,20 +1339,21 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, struct ZenCacheDiskLayer::CacheBucket::PutBatchHandle { - PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {} + PutBatchHandle(std::vector<ZenCacheDiskLayer::PutResult>& OutResults) : OutResults(OutResults) {} struct Entry { std::vector<IoHash> HashKeyAndReferences; + bool Overwrite; }; - std::vector<IoBuffer> Buffers; - std::vector<Entry> Entries; - std::vector<size_t> EntryResultIndexes; + std::vector<ZenCacheValue> Buffers; + std::vector<Entry> Entries; + std::vector<size_t> EntryResultIndexes; - std::vector<bool>& OutResults; + std::vector<ZenCacheDiskLayer::PutResult>& OutResults; }; ZenCacheDiskLayer::CacheBucket::PutBatchHandle* -ZenCacheDiskLayer::CacheBucket::BeginPutBatch(std::vector<bool>& OutResults) +ZenCacheDiskLayer::CacheBucket::BeginPutBatch(std::vector<ZenCacheDiskLayer::PutResult>& OutResults) { ZEN_TRACE_CPU("Z$::Bucket::BeginPutBatch"); return new PutBatchHandle(OutResults); @@ -1315,23 +1369,40 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept ZEN_ASSERT(Batch); if (!Batch->Buffers.empty()) { - std::vector<uint8_t> EntryFlags; - for (const IoBuffer& Buffer : Batch->Buffers) + ZEN_ASSERT(Batch->Buffers.size() == Batch->Entries.size()); + std::vector<uint8_t> EntryFlags; + std::vector<size_t> BufferToEntryIndexes; + std::vector<IoBuffer> BuffersToCommit; + BuffersToCommit.reserve(Batch->Buffers.size()); + for (size_t Index = 0; Index < Batch->Entries.size(); Index++) { - uint8_t Flags = 0; - if (Buffer.GetContentType() == ZenContentType::kCbObject) + const std::vector<IoHash>& HashKeyAndReferences = Batch->Entries[Index].HashKeyAndReferences; + ZEN_ASSERT(HashKeyAndReferences.size() >= 1); + + ZenCacheValue& Value = Batch->Buffers[Index]; + std::span<const IoHash> ReferenceSpan(HashKeyAndReferences.begin() + 1, HashKeyAndReferences.end()); + PutResult& OutResult = Batch->OutResults[Batch->EntryResultIndexes[Index]]; + OutResult = PutResult{zen::PutStatus::Success}; + if (!ShouldRejectPut(HashKeyAndReferences[0], Value, Batch->Entries[Index].Overwrite, OutResult)) { - Flags |= DiskLocation::kStructured; - } - else if (Buffer.GetContentType() == ZenContentType::kCompressedBinary) - { - Flags |= DiskLocation::kCompressed; + BufferToEntryIndexes.push_back(Index); + BuffersToCommit.push_back(Value.Value); + + uint8_t Flags = 0; + if (Value.Value.GetContentType() == ZenContentType::kCbObject) + { + Flags |= DiskLocation::kStructured; + } + else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary) + { + Flags |= DiskLocation::kCompressed; + } + EntryFlags.push_back(Flags); } - EntryFlags.push_back(Flags); } size_t IndexOffset = 0; - m_BlockStore.WriteChunks(Batch->Buffers, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) { + m_BlockStore.WriteChunks(BuffersToCommit, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) { ZEN_MEMSCOPE(GetCacheDiskTag()); std::vector<DiskIndexEntry> DiskEntries; { @@ -1339,8 +1410,9 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept for (size_t Index = 0; Index < Locations.size(); Index++) { DiskLocation Location(Locations[Index], m_Configuration.PayloadAlignment, EntryFlags[IndexOffset + Index]); - const std::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences; - ZEN_ASSERT(HashKeyAndReferences.size() > 1); + const std::vector<IoHash>& HashKeyAndReferences = + Batch->Entries[BufferToEntryIndexes[IndexOffset + Index]].HashKeyAndReferences; + ZEN_ASSERT(HashKeyAndReferences.size() >= 1); const IoHash HashKey = HashKeyAndReferences[0]; DiskEntries.push_back({.Key = HashKey, .Location = Location}); if (m_TrackedCacheKeys) @@ -1375,12 +1447,6 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept } } m_SlogFile.Append(DiskEntries); - for (size_t Index = 0; Index < Locations.size(); Index++) - { - size_t ResultIndex = Batch->EntryResultIndexes[IndexOffset + Index]; - ZEN_ASSERT(ResultIndex < Batch->OutResults.size()); - Batch->OutResults[ResultIndex] = true; - } IndexOffset += Locations.size(); }); } @@ -1876,30 +1942,128 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal } } -void +bool +ZenCacheDiskLayer::CacheBucket::ShouldRejectPut(const IoHash& HashKey, + ZenCacheValue& InOutValue, + bool Overwrite, + ZenCacheDiskLayer::PutResult& OutPutResult) +{ + const bool CheckExisting = m_Configuration.LimitOverwrites && !Overwrite; + if (CheckExisting) + { + RwLock::SharedLockScope IndexLock(m_IndexLock); + auto It = m_Index.find(HashKey); + if (It != m_Index.end()) + { + const PayloadIndex EntryIndex = It.value(); + m_AccessTimes[EntryIndex] = GcClock::TickCount(); + const DiskLocation Location = m_Payloads[EntryIndex].Location; + + const BucketPayload* Payload = &m_Payloads[EntryIndex]; + if (Payload->MetaData) + { + const BucketMetaData MetaData = m_MetaDatas[Payload->MetaData]; + if (MetaData) + { + IndexLock.ReleaseNow(); + if (!cache::impl::UpdateValueWithRawSizeAndHash(InOutValue)) + { + OutPutResult = PutResult{zen::PutStatus::Fail, "Value provided is of bad format"}; + return true; + } + else if (MetaData.RawSize != InOutValue.RawSize || MetaData.RawHash != InOutValue.RawHash) + { + OutPutResult = PutResult{ + zen::PutStatus::Conflict, + fmt::format("Value exists with different size '{}' or hash '{}'", MetaData.RawSize, MetaData.RawHash)}; + return true; + } + return false; + } + } + + ZenCacheValue ExistingValue; + if (Payload->MemCached) + { + ExistingValue.Value = m_MemCachedPayloads[Payload->MemCached].Payload; + IndexLock.ReleaseNow(); + } + else + { + IndexLock.ReleaseNow(); + + if (Location.IsFlagSet(DiskLocation::kStandaloneFile)) + { + ExistingValue.Value = GetStandaloneCacheValue(Location, HashKey); + } + else + { + ExistingValue.Value = GetInlineCacheValue(Location); + } + } + + if (ExistingValue.Value) + { + if (cache::impl::UpdateValueWithRawSizeAndHash(ExistingValue)) + { + if (!cache::impl::UpdateValueWithRawSizeAndHash(InOutValue)) + { + OutPutResult = PutResult{zen::PutStatus::Fail, "Value provided is of bad format"}; + return true; + } + + if (ExistingValue.RawSize != InOutValue.RawSize || ExistingValue.RawHash != InOutValue.RawHash) + { + OutPutResult = PutResult{zen::PutStatus::Conflict, + fmt::format("Value exists with different size '{}' or hash '{}'", + ExistingValue.RawSize, + ExistingValue.RawHash)}; + return true; + } + } + } + } + } + return false; +} + +ZenCacheDiskLayer::PutResult ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, + bool Overwrite, PutBatchHandle* OptionalBatchHandle) { ZEN_TRACE_CPU("Z$::Bucket::Put"); metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size()); + PutResult Result{zen::PutStatus::Success}; + if (Value.Value.Size() >= m_Configuration.LargeObjectThreshold) { - PutStandaloneCacheValue(HashKey, Value, References); + ZenCacheValue AcceptedValue = Value; + if (ShouldRejectPut(HashKey, AcceptedValue, Overwrite, Result)) + { + if (OptionalBatchHandle) + { + OptionalBatchHandle->OutResults.push_back(Result); + } + return Result; + } + PutStandaloneCacheValue(HashKey, AcceptedValue, References); if (OptionalBatchHandle) { - OptionalBatchHandle->OutResults.push_back(true); + OptionalBatchHandle->OutResults.push_back({zen::PutStatus::Success}); } } else { - PutInlineCacheValue(HashKey, Value, References, OptionalBatchHandle); + Result = PutInlineCacheValue(HashKey, Value, References, Overwrite, OptionalBatchHandle); } m_DiskWriteCount++; + return Result; } uint64_t @@ -2425,7 +2589,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) ZenCacheDiskLayer::BucketStats ZenCacheDiskLayer::CacheBucket::Stats() { - GcStorageSize Size = StorageSize(); + CacheStoreSize Size = TotalSize(); return ZenCacheDiskLayer::BucketStats{.DiskSize = Size.DiskSize, .MemorySize = Size.MemorySize, .DiskHitCount = m_DiskHitCount, @@ -2748,38 +2912,49 @@ ZenCacheDiskLayer::CacheBucket::GetMetaData(RwLock::SharedLockScope&, const Buck return {}; } -void +ZenCacheDiskLayer::PutResult ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, + bool Overwrite, PutBatchHandle* OptionalBatchHandle) { ZEN_TRACE_CPU("Z$::Bucket::PutInlineCacheValue"); + PutResult Result{zen::PutStatus::Success}; if (OptionalBatchHandle != nullptr) { - OptionalBatchHandle->Buffers.push_back(Value.Value); + OptionalBatchHandle->Buffers.push_back(Value); OptionalBatchHandle->Entries.push_back({}); OptionalBatchHandle->EntryResultIndexes.push_back(OptionalBatchHandle->OutResults.size()); - OptionalBatchHandle->OutResults.push_back(false); + OptionalBatchHandle->OutResults.push_back(PutResult{zen::PutStatus::Fail}); + PutBatchHandle::Entry& CurrentEntry = OptionalBatchHandle->Entries.back(); + CurrentEntry.Overwrite = Overwrite; std::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences; - HashKeyAndReferences.reserve(1 + HashKeyAndReferences.size()); + HashKeyAndReferences.reserve(1 + References.size()); HashKeyAndReferences.push_back(HashKey); - HashKeyAndReferences.insert(HashKeyAndReferences.end(), HashKeyAndReferences.begin(), HashKeyAndReferences.end()); - return; + HashKeyAndReferences.insert(HashKeyAndReferences.end(), References.begin(), References.end()); + return Result; } + + ZenCacheValue AcceptedValue = Value; + if (ShouldRejectPut(HashKey, AcceptedValue, Overwrite, Result)) + { + return Result; + } + uint8_t EntryFlags = 0; - if (Value.Value.GetContentType() == ZenContentType::kCbObject) + if (AcceptedValue.Value.GetContentType() == ZenContentType::kCbObject) { EntryFlags |= DiskLocation::kStructured; } - else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary) + else if (AcceptedValue.Value.GetContentType() == ZenContentType::kCompressedBinary) { EntryFlags |= DiskLocation::kCompressed; } - m_BlockStore.WriteChunk(Value.Value.Data(), - Value.Value.Size(), + m_BlockStore.WriteChunk(AcceptedValue.Value.Data(), + AcceptedValue.Value.Size(), m_Configuration.PayloadAlignment, [&](const BlockStoreLocation& BlockStoreLocation) { ZEN_MEMSCOPE(GetCacheDiskTag()); @@ -2816,6 +2991,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, } m_SlogFile.Append({.Key = HashKey, .Location = Location}); }); + return Result; } std::string @@ -3752,7 +3928,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) struct ZenCacheDiskLayer::PutBatchHandle { - PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {} + PutBatchHandle(std::vector<ZenCacheDiskLayer::PutResult>& OutResults) : OutResults(OutResults) {} struct BucketHandle { CacheBucket* Bucket; @@ -3811,13 +3987,13 @@ struct ZenCacheDiskLayer::PutBatchHandle return NewBucketHandle; } - RwLock Lock; - std::vector<BucketHandle> BucketHandles; - std::vector<bool>& OutResults; + RwLock Lock; + std::vector<BucketHandle> BucketHandles; + std::vector<ZenCacheDiskLayer::PutResult>& OutResults; }; ZenCacheDiskLayer::PutBatchHandle* -ZenCacheDiskLayer::BeginPutBatch(std::vector<bool>& OutResults) +ZenCacheDiskLayer::BeginPutBatch(std::vector<PutResult>& OutResults) { return new PutBatchHandle(OutResults); } @@ -3954,21 +4130,23 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, GetBatc } } -void +ZenCacheDiskLayer::PutResult ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, + bool Overwrite, PutBatchHandle* OptionalBatchHandle) { ZEN_TRACE_CPU("Z$::Put"); - + PutResult RetVal = {zen::PutStatus::Fail}; if (CacheBucket* Bucket = GetOrCreateBucket(InBucket); Bucket != nullptr) { CacheBucket::PutBatchHandle* BucketBatchHandle = OptionalBatchHandle == nullptr ? nullptr : OptionalBatchHandle->GetHandle(Bucket); - Bucket->Put(HashKey, Value, References, BucketBatchHandle); + RetVal = Bucket->Put(HashKey, Value, References, Overwrite, BucketBatchHandle); TryMemCacheTrim(); } + return RetVal; } void @@ -4241,8 +4419,9 @@ ZenCacheDiskLayer::Flush() } } +#if ZEN_WITH_TESTS void -ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx) +ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) { ZEN_TRACE_CPU("Z$::ScrubStorage"); @@ -4253,13 +4432,13 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx) for (auto& Kv : m_Buckets) { -#if 1 +# if 1 Results.push_back(Ctx.ThreadPool().EnqueueTask( std::packaged_task<void()>{[this, Bucket = Kv.second.get(), &Ctx] { Bucket->ScrubStorage(Ctx); }})); -#else +# else CacheBucket& Bucket = *Kv.second; Bucket.ScrubStorage(Ctx); -#endif +# endif } for (auto& Result : Results) @@ -4275,16 +4454,17 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx) } } } +#endif // ZEN_WITH_TESTS -GcStorageSize -ZenCacheDiskLayer::StorageSize() const +CacheStoreSize +ZenCacheDiskLayer::TotalSize() const { - GcStorageSize StorageSize{}; + CacheStoreSize StorageSize{}; RwLock::SharedLockScope _(m_Lock); for (auto& Kv : m_Buckets) { - GcStorageSize BucketSize = Kv.second->StorageSize(); + CacheStoreSize BucketSize = Kv.second->TotalSize(); StorageSize.DiskSize += BucketSize.DiskSize; StorageSize.MemorySize += BucketSize.MemorySize; } @@ -4295,7 +4475,7 @@ ZenCacheDiskLayer::StorageSize() const ZenCacheDiskLayer::DiskStats ZenCacheDiskLayer::Stats() const { - GcStorageSize Size = StorageSize(); + CacheStoreSize Size = TotalSize(); ZenCacheDiskLayer::DiskStats Stats = {.DiskSize = Size.DiskSize, .MemorySize = Size.MemorySize}; { RwLock::SharedLockScope _(m_Lock); @@ -4319,7 +4499,7 @@ ZenCacheDiskLayer::GetInfo() const { Info.BucketNames.push_back(Kv.first); Info.EntryCount += Kv.second->EntryCount(); - GcStorageSize BucketSize = Kv.second->StorageSize(); + CacheStoreSize BucketSize = Kv.second->TotalSize(); Info.StorageSize.DiskSize += BucketSize.DiskSize; Info.StorageSize.MemorySize += BucketSize.MemorySize; } @@ -4334,7 +4514,7 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end()) { - return ZenCacheDiskLayer::BucketInfo{.EntryCount = It->second->EntryCount(), .StorageSize = It->second->StorageSize()}; + return ZenCacheDiskLayer::BucketInfo{.EntryCount = It->second->EntryCount(), .StorageSize = It->second->TotalSize()}; } return {}; } diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index de4b0a37c..ff21d1ede 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -153,13 +153,13 @@ CacheRpcHandler::CacheRpcHandler(LoggerRef InLog, CacheStats& InCacheStats, UpstreamCacheClient& InUpstreamCache, ZenCacheStore& InCacheStore, - CidStore& InCidStore, + GetCidStoreFunc&& InGetCidStore, const DiskWriteBlocker* InDiskWriteBlocker) : m_Log(InLog) , m_CacheStats(InCacheStats) , m_UpstreamCache(InUpstreamCache) , m_CacheStore(InCacheStore) -, m_CidStore(InCidStore) +, m_GetCidStore(std::move(InGetCidStore)) , m_DiskWriteBlocker(InDiskWriteBlocker) { } @@ -174,6 +174,12 @@ CacheRpcHandler::AreDiskWritesAllowed() const return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); } +CidStore& +CacheRpcHandler::GetCidStore(std::string_view Namespace) +{ + return m_GetCidStore(Namespace); +} + CacheRpcHandler::RpcResponseCode CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context, std::string_view UriNamespace, @@ -334,13 +340,13 @@ CacheRpcHandler::HandleRpcPutCacheRecords(const CacheRequestContext& Context, co .Policy = std::move(Policy), .Context = Context}; - PutResult Result = PutCacheRecord(PutRequest, &BatchRequest); + PutStatus Result = PutCacheRecord(PutRequest, &BatchRequest); - if (Result == PutResult::Invalid) + if (Result == PutStatus::Invalid) { return CbPackage{}; } - Results.push_back(Result == PutResult::Success); + Results.push_back(Result == PutStatus::Success); } if (Results.empty()) { @@ -360,7 +366,7 @@ CacheRpcHandler::HandleRpcPutCacheRecords(const CacheRequestContext& Context, co return RpcResponse; } -PutResult +PutStatus CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Package) { CbObjectView Record = Request.RecordObject; @@ -381,9 +387,12 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag Stopwatch Timer; + CidStore& ChunkStore = m_GetCidStore(Request.Namespace); + Request.RecordObject.IterateAttachments([this, &Request, Package, + &ChunkStore, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, @@ -412,7 +421,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag Count.Invalid++; } } - else if (m_CidStore.ContainsChunk(ValueHash)) + else if (ChunkStore.ContainsChunk(ValueHash)) { ValidAttachments.emplace_back(ValueHash); Count.Valid++; @@ -422,19 +431,33 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag if (Count.Invalid > 0) { - return PutResult::Invalid; + return PutStatus::Invalid; } ZenCacheValue CacheValue; CacheValue.Value = IoBuffer(Record.GetSize()); Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize())); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(Request.Context, Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue, ReferencedAttachments, nullptr); + bool Overwrite = EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::StoreLocal) && + !EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::QueryLocal); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Request.Context, + Request.Namespace, + Request.Key.Bucket, + Request.Key.Hash, + CacheValue, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return PutResult.Status; + } m_CacheStats.WriteCount++; if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (size_t Index = 0; Index < InsertResults.size(); Index++) { if (InsertResults[Index].New) @@ -461,12 +484,14 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag if (HasUpstream && EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::StoreRemote) && !IsPartialRecord) { - m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .Namespace = Request.Namespace, - .Key = Request.Key, - .ValueContentIds = std::move(ValidAttachments)}); + m_UpstreamCache.EnqueueUpstream( + {.Type = ZenContentType::kCbPackage, + .Namespace = Request.Namespace, + .Key = Request.Key, + .ValueContentIds = std::move(ValidAttachments)}, + [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); } - return PutResult::Success; + return PutStatus::Success; } CbPackage @@ -507,6 +532,8 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb return CbPackage{}; } + CidStore& ChunkStore = m_GetCidStore(Namespace.value()); + const bool HasUpstream = m_UpstreamCache.IsActive(); eastl::fixed_vector<RecordRequestData, 16> Requests; @@ -606,7 +633,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else if (EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData)) { - if (m_CidStore.ContainsChunk(Value.ContentId)) + if (ChunkStore.ContainsChunk(Value.ContentId)) { Value.Exists = true; } @@ -627,7 +654,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Value.ContentId)) + if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Value.ContentId)) { if (Chunk.GetSize() > 0) { @@ -651,7 +678,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } if (!RequestValueIndexes.empty()) { - m_CidStore.IterateChunks( + ChunkStore.IterateChunks( CidHashes, [this, &Request, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool { try @@ -744,7 +771,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } } - const auto OnCacheRecordGetComplete = [this, Namespace, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) { + const auto OnCacheRecordGetComplete = [this, Namespace, &ChunkStore, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) { if (!Params.Record) { return; @@ -765,18 +792,24 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal) && AreDiskWritesAllowed(); if (StoreLocal) { + bool Overwrite = !EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::QueryLocal); std::vector<IoHash> ReferencedAttachments; ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) { const IoHash ValueHash = HashView.AsHash(); ReferencedAttachments.push_back(ValueHash); }); - m_CacheStore.Put(Context, - *Namespace, - Key.Bucket, - Key.Hash, - {.Value = {Request.RecordCacheValue}}, - ReferencedAttachments, - nullptr); + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Context, + *Namespace, + Key.Bucket, + Key.Hash, + {.Value = {Request.RecordCacheValue}}, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return; + } m_CacheStats.WriteCount++; } ParseValues(Request); @@ -807,7 +840,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb Value.Exists = true; if (StoreLocal) { - m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash()); + ChunkStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash()); } if (!EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData)) { @@ -924,10 +957,12 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con const bool HasUpstream = m_UpstreamCache.IsActive(); CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); - std::vector<bool> BatchResults; - eastl::fixed_vector<size_t, 32> BatchResultIndexes; - eastl::fixed_vector<bool, 32> Results; - eastl::fixed_vector<CacheKey, 32> UpstreamCacheKeys; + CidStore& ChunkStore = m_GetCidStore(Namespace.value()); + + std::vector<ZenCacheStore::PutResult> BatchResults; + eastl::fixed_vector<size_t, 32> BatchResultIndexes; + eastl::fixed_vector<ZenCacheStore::PutResult, 32> Results; + eastl::fixed_vector<CacheKey, 32> UpstreamCacheKeys; uint64_t RequestCount = RequestsArray.Num(); { @@ -977,34 +1012,39 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) { - IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); + bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal); + IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); Value.SetContentType(ZenContentType::kCompressedBinary); if (RawSize == 0) { RawSize = Chunk.DecodeRawSize(); } - m_CacheStore.Put(Context, - *Namespace, - Key.Bucket, - Key.Hash, - {.Value = Value, .RawSize = RawSize, .RawHash = RawHash}, - {}, - Batch.get()); - m_CacheStats.WriteCount++; + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Context, + *Namespace, + Key.Bucket, + Key.Hash, + {.Value = Value, .RawSize = RawSize, .RawHash = RawHash}, + {}, + Overwrite, + Batch.get()); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + } if (Batch) { BatchResultIndexes.push_back(Results.size()); - Results.push_back(false); + Results.push_back({zen::PutStatus::Fail}); } else { - Results.push_back(true); + Results.push_back(PutResult); } TransferredSize = Chunk.GetCompressedSize(); } else { - Results.push_back(true); + Results.push_back({zen::PutStatus::Success}); } Valid = true; } @@ -1020,12 +1060,12 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con if (m_CacheStore.Get(Context, *Namespace, Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType())) { - Results.push_back(true); + Results.push_back({zen::PutStatus::Success}); Valid = true; } else { - Results.push_back(false); + Results.push_back({zen::PutStatus::Fail, fmt::format("Missing attachment with raw hash {}", RawHash)}); } } // We do not search the Upstream. No data in a put means the caller is probing for whether they need to do a heavy put. @@ -1060,27 +1100,49 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con { size_t BatchResultIndex = BatchResultIndexes[Index]; ZEN_ASSERT(BatchResultIndex < Results.size()); - ZEN_ASSERT(Results[BatchResultIndex] == false); + ZEN_ASSERT(Results[BatchResultIndex].Status != zen::PutStatus::Success); Results[BatchResultIndex] = BatchResults[Index]; } for (std::size_t Index = 0; Index < Results.size(); Index++) { - if (Results[Index] && UpstreamCacheKeys[Index] != CacheKey::Empty) + if ((Results[Index].Status == zen::PutStatus::Success) && UpstreamCacheKeys[Index] != CacheKey::Empty) { m_UpstreamCache.EnqueueUpstream( - {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]}); + {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]}, + [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); } } { ZEN_TRACE_CPU("Z$::RpcPutCacheValues::Response"); CbObjectWriter ResponseObject{1024}; ResponseObject.BeginArray("Result"sv); - for (bool Value : Results) + bool bAnyErrors = false; + for (const ZenCacheStore::PutResult& Value : Results) { - ResponseObject.AddBool(Value); + if (Value.Status == zen::PutStatus::Success) + { + ResponseObject.AddBool(true); + } + else + { + bAnyErrors = true; + ResponseObject.AddBool(false); + } } ResponseObject.EndArray(); + if (bAnyErrors) + { + ResponseObject.BeginArray("ErrorMessages"sv); + for (const ZenCacheStore::PutResult& Value : Results) + { + if (Value.Status != zen::PutStatus::Success) + { + ResponseObject.AddString(Value.Message); + } + } + ResponseObject.EndArray(); + } CbPackage RpcResponse; RpcResponse.SetObject(ResponseObject.Save()); @@ -1239,6 +1301,7 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO const bool HasData = IsCompressedBinary(Params.Value.GetContentType()); const bool SkipData = EnumHasAllFlags(Request.Policy, CachePolicy::SkipData); const bool StoreData = EnumHasAllFlags(Request.Policy, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); + const bool Overwrite = StoreData && !EnumHasAllFlags(Request.Policy, CachePolicy::QueryLocal); const bool IsHit = SkipData || HasData; if (IsHit) { @@ -1249,14 +1312,19 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO if (HasData && StoreData) { - m_CacheStore.Put(Context, - *Namespace, - Request.Key.Bucket, - Request.Key.Hash, - ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash}, - {}, - nullptr); - m_CacheStats.WriteCount++; + ZenCacheStore::PutResult PutResult = m_CacheStore.Put( + Context, + *Namespace, + Request.Key.Bucket, + Request.Key.Hash, + ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash}, + {}, + Overwrite, + nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + } } ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", @@ -1494,6 +1562,8 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, using namespace cache::detail; const bool HasUpstream = m_UpstreamCache.IsActive(); + CidStore& ChunkStore = m_GetCidStore(Namespace); + // TODO: BatchGet records? std::vector<CacheKeyRequest*> UpstreamRecordRequests; for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex) @@ -1527,36 +1597,48 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, if (!UpstreamRecordRequests.empty()) { - const auto OnCacheRecordGetComplete = [this, Namespace, &RecordKeys, &Records, &RecordRequests, Context]( - CacheRecordGetCompleteParams&& Params) { - if (!Params.Record) - { - return; - } - CacheKeyRequest& RecordKey = Params.Request; - size_t RecordIndex = std::distance(RecordKeys.data(), &RecordKey); - RecordRequests[RecordIndex]->ElapsedTimeUs += static_cast<uint64_t>(Params.ElapsedSeconds * 1000000.0); - RecordBody& Record = Records[RecordIndex]; - - const CacheKey& Key = RecordKey.Key; - Record.Exists = true; - CbObject ObjectBuffer = CbObject::Clone(Params.Record); - Record.CacheValue = ObjectBuffer.GetBuffer().AsIoBuffer(); - Record.CacheValue.SetContentType(ZenContentType::kCbObject); - Record.Source = Params.Source; - - bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); - if (StoreLocal) - { - std::vector<IoHash> ReferencedAttachments; - ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) { - const IoHash ValueHash = HashView.AsHash(); - ReferencedAttachments.push_back(ValueHash); - }); - m_CacheStore.Put(Context, Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}, ReferencedAttachments, nullptr); - m_CacheStats.WriteCount++; - } - }; + const auto OnCacheRecordGetComplete = + [this, Namespace, &RecordKeys, &Records, &RecordRequests, Context](CacheRecordGetCompleteParams&& Params) { + if (!Params.Record) + { + return; + } + CacheKeyRequest& RecordKey = Params.Request; + size_t RecordIndex = std::distance(RecordKeys.data(), &RecordKey); + RecordRequests[RecordIndex]->ElapsedTimeUs += static_cast<uint64_t>(Params.ElapsedSeconds * 1000000.0); + RecordBody& Record = Records[RecordIndex]; + + const CacheKey& Key = RecordKey.Key; + Record.Exists = true; + CbObject ObjectBuffer = CbObject::Clone(Params.Record); + Record.CacheValue = ObjectBuffer.GetBuffer().AsIoBuffer(); + Record.CacheValue.SetContentType(ZenContentType::kCbObject); + Record.Source = Params.Source; + + bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); + if (StoreLocal) + { + bool Overwrite = !EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal); + std::vector<IoHash> ReferencedAttachments; + ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) { + const IoHash ValueHash = HashView.AsHash(); + ReferencedAttachments.push_back(ValueHash); + }); + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Context, + Namespace, + Key.Bucket, + Key.Hash, + {.Value = Record.CacheValue}, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return; + } + m_CacheStats.WriteCount++; + } + }; m_UpstreamCache.GetCacheRecords(Namespace, UpstreamRecordRequests, std::move(OnCacheRecordGetComplete)); } @@ -1620,12 +1702,12 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, { if (EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData) && Request->RawSizeKnown) { - if (m_CidStore.ContainsChunk(Request->Key->ChunkId)) + if (ChunkStore.ContainsChunk(Request->Key->ChunkId)) { Request->Exists = true; } } - else if (IoBuffer Payload = m_CidStore.FindChunkByCid(Request->Key->ChunkId)) + else if (IoBuffer Payload = ChunkStore.FindChunkByCid(Request->Key->ChunkId)) { if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData)) { @@ -1758,6 +1840,8 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, return; } + CidStore& ChunkStore = m_GetCidStore(Namespace); + CacheChunkRequest& Key = Params.Request; size_t RequestIndex = std::distance(RequestKeys.data(), &Key); ChunkRequest& Request = Requests[RequestIndex]; @@ -1774,20 +1858,26 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, bool StoreLocal = EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); if (StoreLocal) { + bool Overwrite = !EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::QueryLocal); if (Request.IsRecordRequest) { - m_CidStore.AddChunk(Params.Value, Params.RawHash); + ChunkStore.AddChunk(Params.Value, Params.RawHash); } else { - m_CacheStore.Put(Context, - Namespace, - Key.Key.Bucket, - Key.Key.Hash, - {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash}, - {}, - nullptr); - m_CacheStats.WriteCount++; + ZenCacheStore::PutResult PutResult = + m_CacheStore.Put(Context, + Namespace, + Key.Key.Bucket, + Key.Key.Hash, + {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash}, + {}, + Overwrite, + nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + } } } if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData)) diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index d956384ca..1f2d6c37f 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -139,13 +139,10 @@ ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const st CreateDirectories(m_RootDir); m_DiskLayer.DiscoverBuckets(); - - m_Gc.AddGcStorage(this); } ZenCacheNamespace::~ZenCacheNamespace() { - m_Gc.RemoveGcStorage(this); } struct ZenCacheNamespace::PutBatchHandle @@ -154,7 +151,7 @@ struct ZenCacheNamespace::PutBatchHandle }; ZenCacheNamespace::PutBatchHandle* -ZenCacheNamespace::BeginPutBatch(std::vector<bool>& OutResult) +ZenCacheNamespace::BeginPutBatch(std::vector<PutResult>& OutResult) { ZenCacheNamespace::PutBatchHandle* Handle = new ZenCacheNamespace::PutBatchHandle; Handle->DiskLayerHandle = m_DiskLayer.BeginPutBatch(OutResult); @@ -252,11 +249,12 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, GetBatc return; } -void +ZenCacheNamespace::PutResult ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, + bool Overwrite, PutBatchHandle* OptionalBatchHandle) { ZEN_TRACE_CPU(OptionalBatchHandle ? "Z$::Namespace::Put(Batched)" : "Z$::Namespace::Put"); @@ -268,8 +266,12 @@ ZenCacheNamespace::Put(std::string_view InBucket, ZEN_ASSERT(Value.Value.Size()); ZenCacheDiskLayer::PutBatchHandle* DiskLayerBatchHandle = OptionalBatchHandle ? OptionalBatchHandle->DiskLayerHandle : nullptr; - m_DiskLayer.Put(InBucket, HashKey, Value, References, DiskLayerBatchHandle); - m_WriteCount++; + PutResult RetVal = m_DiskLayer.Put(InBucket, HashKey, Value, References, Overwrite, DiskLayerBatchHandle); + if (RetVal.Status == zen::PutStatus::Success) + { + m_WriteCount++; + } + return RetVal; } bool @@ -297,7 +299,6 @@ ZenCacheNamespace::EnumerateBucketContents(std::string_view std::function<void()> ZenCacheNamespace::Drop() { - m_Gc.RemoveGcStorage(this); return m_DiskLayer.Drop(); } @@ -307,25 +308,19 @@ ZenCacheNamespace::Flush() m_DiskLayer.Flush(); } +#if ZEN_WITH_TESTS void -ZenCacheNamespace::ScrubStorage(ScrubContext& Ctx) +ZenCacheNamespace::Scrub(ScrubContext& Ctx) { - if (m_LastScrubTime == Ctx.ScrubTimestamp()) - { - return; - } - ZEN_INFO("scrubbing '{}'", m_RootDir); - - m_LastScrubTime = Ctx.ScrubTimestamp(); - - m_DiskLayer.ScrubStorage(Ctx); + m_DiskLayer.Scrub(Ctx); } +#endif // ZEN_WITH_TESTS -GcStorageSize -ZenCacheNamespace::StorageSize() const +CacheStoreSize +ZenCacheNamespace::TotalSize() const { - return m_DiskLayer.StorageSize(); + return m_DiskLayer.TotalSize(); } ZenCacheNamespace::Info @@ -557,7 +552,7 @@ ZenCacheStore::LogWorker() } } -ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<bool>& OutResult) +ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<PutResult>& OutResult) : m_CacheStore(CacheStore) { ZEN_MEMSCOPE(GetCacheStoreTag()); @@ -720,13 +715,14 @@ ZenCacheStore::Get(const CacheRequestContext& Context, m_MissCount++; } -void +ZenCacheStore::PutResult ZenCacheStore::Put(const CacheRequestContext& Context, std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, + bool Overwrite, PutBatch* OptionalBatchHandle) { // Ad hoc rejection of known bad usage patterns for DDC bucket names @@ -734,7 +730,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context, if (IsKnownBadBucketName(Bucket)) { m_RejectedWriteCount++; - return; + return PutResult{zen::PutStatus::Invalid, "Bad bucket name"}; } ZEN_MEMSCOPE(GetCacheStoreTag()); @@ -764,9 +760,16 @@ ZenCacheStore::Put(const CacheRequestContext& Context, if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store) { ZenCacheNamespace::PutBatchHandle* BatchHandle = OptionalBatchHandle ? OptionalBatchHandle->m_NamespaceBatchHandle : nullptr; - Store->Put(Bucket, HashKey, Value, References, BatchHandle); - m_WriteCount++; - return; + PutResult RetVal = Store->Put(Bucket, HashKey, Value, References, Overwrite, BatchHandle); + if (RetVal.Status == zen::PutStatus::Success) + { + m_WriteCount++; + } + else + { + m_RejectedWriteCount++; + } + return RetVal; } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put [{}] bucket '{}', key '{}'", @@ -774,6 +777,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context, Namespace, Bucket, HashKey.ToHexString()); + return PutResult{zen::PutStatus::Fail, fmt::format("Unknown namespace '{}'", Namespace)}; } bool @@ -822,11 +826,13 @@ ZenCacheStore::Flush() IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Flush(); }); } +#if ZEN_WITH_TESTS void -ZenCacheStore::ScrubStorage(ScrubContext& Ctx) +ZenCacheStore::Scrub(ScrubContext& Ctx) { - IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.ScrubStorage(Ctx); }); + IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); }); } +#endif // ZEN_WITH_TESTS CacheValueDetails ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter, @@ -951,12 +957,12 @@ ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Names } } -GcStorageSize -ZenCacheStore::StorageSize() const +CacheStoreSize +ZenCacheStore::TotalSize() const { - GcStorageSize Size; + CacheStoreSize Size; IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { - GcStorageSize StoreSize = Store.StorageSize(); + CacheStoreSize StoreSize = Store.TotalSize(); Size.MemorySize += StoreSize.MemorySize; Size.DiskSize += StoreSize.DiskSize; }); @@ -1026,7 +1032,7 @@ ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig) ZenCacheStore::Info ZenCacheStore::GetInfo() const { - ZenCacheStore::Info Info = {.Config = m_Configuration, .StorageSize = StorageSize()}; + ZenCacheStore::Info Info = {.Config = m_Configuration, .StorageSize = TotalSize()}; IterateNamespaces([&Info](std::string_view NamespaceName, ZenCacheNamespace& Namespace) { Info.NamespaceNames.push_back(std::string(NamespaceName)); @@ -1378,7 +1384,7 @@ TEST_CASE("cachestore.store") Value.Value = Obj.GetBuffer().AsIoBuffer(); Value.Value.SetContentType(ZenContentType::kCbObject); - Zcs.Put("test_bucket"sv, Key, Value, {}); + Zcs.Put("test_bucket"sv, Key, Value, {}, false); } for (int i = 0; i < kIterationCount; ++i) @@ -1414,7 +1420,7 @@ TEST_CASE("cachestore.size") const size_t Count = 16; ScopedTemporaryDirectory TempDir; - GcStorageSize CacheSize; + CacheStoreSize CacheSize; { GcManager Gc; @@ -1432,10 +1438,10 @@ TEST_CASE("cachestore.size") const size_t Bucket = Key % 4; std::string BucketName = fmt::format("test_bucket-{}", Bucket); IoHash Hash = IoHash::HashBuffer(&Key, sizeof(uint32_t)); - Zcs.Put(BucketName, Hash, ZenCacheValue{.Value = Buffer}, {}); + Zcs.Put(BucketName, Hash, ZenCacheValue{.Value = Buffer}, {}, false); Keys.push_back({BucketName, Hash}); } - CacheSize = Zcs.StorageSize(); + CacheSize = Zcs.TotalSize(); CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize); CHECK_EQ(0, CacheSize.MemorySize); @@ -1445,7 +1451,7 @@ TEST_CASE("cachestore.size") Zcs.Get(Key.first, Key.second, _); } - CacheSize = Zcs.StorageSize(); + CacheSize = Zcs.TotalSize(); CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize); CHECK_LE(CacheValue.GetSize() * Count, CacheSize.MemorySize); } @@ -1454,7 +1460,7 @@ TEST_CASE("cachestore.size") GcManager Gc; ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {}); - const GcStorageSize SerializedSize = Zcs.StorageSize(); + const CacheStoreSize SerializedSize = Zcs.TotalSize(); CHECK_EQ(SerializedSize.MemorySize, 0); CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize); @@ -1462,8 +1468,8 @@ TEST_CASE("cachestore.size") { Zcs.DropBucket(fmt::format("test_bucket-{}", Bucket)); } - CHECK_EQ(0, Zcs.StorageSize().DiskSize); - CHECK_EQ(0, Zcs.StorageSize().MemorySize); + CHECK_EQ(0, Zcs.TotalSize().DiskSize); + CHECK_EQ(0, Zcs.TotalSize().MemorySize); } } @@ -1472,7 +1478,7 @@ TEST_CASE("cachestore.size") const size_t Count = 16; ScopedTemporaryDirectory TempDir; - GcStorageSize CacheSize; + CacheStoreSize CacheSize; { GcManager Gc; @@ -1486,10 +1492,10 @@ TEST_CASE("cachestore.size") for (size_t Key = 0; Key < Count; ++Key) { const size_t Bucket = Key % 4; - Zcs.Put(fmt::format("test_bucket-{}", Bucket), IoHash::HashBuffer(&Key, sizeof(uint32_t)), {.Value = Buffer}, {}); + Zcs.Put(fmt::format("test_bucket-{}", Bucket), IoHash::HashBuffer(&Key, sizeof(uint32_t)), {.Value = Buffer}, {}, false); } - CacheSize = Zcs.StorageSize(); + CacheSize = Zcs.TotalSize(); CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize); CHECK_EQ(0, CacheSize.MemorySize); } @@ -1498,7 +1504,7 @@ TEST_CASE("cachestore.size") GcManager Gc; ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {}); - const GcStorageSize SerializedSize = Zcs.StorageSize(); + const CacheStoreSize SerializedSize = Zcs.TotalSize(); CHECK_EQ(SerializedSize.MemorySize, 0); CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize); @@ -1506,7 +1512,7 @@ TEST_CASE("cachestore.size") { Zcs.DropBucket(fmt::format("test_bucket-{}", Bucket)); } - CHECK_EQ(0, Zcs.StorageSize().DiskSize); + CHECK_EQ(0, Zcs.TotalSize().DiskSize); } } } @@ -1569,7 +1575,7 @@ TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true)) for (const auto& Chunk : Chunks) { ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, &Chunk]() { - Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {}); + Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {}, false); WorkCompleted.fetch_add(1); }); } @@ -1599,7 +1605,7 @@ TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true)) GcChunkHashes.swap(RemainingChunkHashes); }; - const uint64_t TotalSize = Zcs.StorageSize().DiskSize; + const uint64_t TotalSize = Zcs.TotalSize().DiskSize; CHECK_LE(kChunkSize * Chunks.size(), TotalSize); { @@ -1650,7 +1656,7 @@ TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true)) for (const auto& Chunk : NewChunks) { ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, Chunk, &AddedChunkCount]() { - Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {}); + Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {}, false); AddedChunkCount.fetch_add(1); WorkCompleted.fetch_add(1); }); @@ -1755,14 +1761,14 @@ TEST_CASE("cachestore.namespaces") Buffer.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue = {.Value = Buffer}; - Zcs.Put(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue, {}); + Zcs.Put(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue, {}, false); ZenCacheValue GetValue; CHECK(Zcs.Get(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue)); CHECK(!Zcs.Get(Context, CustomNamespace, Bucket, Key1, GetValue)); // This should just be dropped as we don't allow creating of namespaces on the fly - Zcs.Put(Context, CustomNamespace, Bucket, Key1, PutValue, {}); + Zcs.Put(Context, CustomNamespace, Bucket, Key1, PutValue, {}, false); CHECK(!Zcs.Get(Context, CustomNamespace, Bucket, Key1, GetValue)); } @@ -1778,7 +1784,7 @@ TEST_CASE("cachestore.namespaces") IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer(); Buffer2.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue2 = {.Value = Buffer2}; - Zcs.Put(Context, CustomNamespace, Bucket, Key2, PutValue2, {}); + Zcs.Put(Context, CustomNamespace, Bucket, Key2, PutValue2, {}, false); ZenCacheValue GetValue; CHECK(!Zcs.Get(Context, ZenCacheStore::DefaultNamespace, Bucket, Key2, GetValue)); @@ -1820,7 +1826,7 @@ TEST_CASE("cachestore.drop.bucket") Buffer.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue = {.Value = Buffer}; - Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {}); + Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {}, false); return Key; }; auto GetValue = [&Context](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) { @@ -1893,7 +1899,7 @@ TEST_CASE("cachestore.drop.namespace") Buffer.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue = {.Value = Buffer}; - Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {}); + Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {}, false); return Key; }; auto GetValue = [&Context](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) { @@ -1957,8 +1963,6 @@ TEST_CASE("cachestore.blocked.disklayer.put") { ScopedTemporaryDirectory TempDir; - GcStorageSize CacheSize; - const auto CreateCacheValue = [](size_t Size) -> CbObject { std::vector<uint8_t> Buf; Buf.resize(Size, Size & 0xff); @@ -1979,7 +1983,7 @@ TEST_CASE("cachestore.blocked.disklayer.put") size_t Key = Buffer.Size(); IoHash HashKey = IoHash::HashBuffer(&Key, sizeof(uint32_t)); - Zcs.Put("test_bucket", HashKey, {.Value = Buffer}, {}); + Zcs.Put("test_bucket", HashKey, {.Value = Buffer}, {}, false); ZenCacheValue BufferGet; CHECK(Zcs.Get("test_bucket", HashKey, BufferGet)); @@ -1989,7 +1993,7 @@ TEST_CASE("cachestore.blocked.disklayer.put") Buffer2.SetContentType(ZenContentType::kCbObject); // We should be able to overwrite even if the file is open for read - Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}, {}); + Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}, {}, false); MemoryView OldView = BufferGet.Value.GetView(); @@ -2080,7 +2084,7 @@ TEST_CASE("cachestore.scrub") AttachmentHashes.push_back(Attachment.DecodeRawHash()); CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back()); } - Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes); + Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes, false); } }; @@ -2094,8 +2098,8 @@ TEST_CASE("cachestore.scrub") WorkerThreadPool ThreadPool{1}; ScrubContext ScrubCtx{ThreadPool}; - Zcs.ScrubStorage(ScrubCtx); - CidStore.ScrubStorage(ScrubCtx); + Zcs.Scrub(ScrubCtx); + CidStore.Scrub(ScrubCtx); CHECK(ScrubCtx.ScrubbedChunks() == (StructuredCids.size() + StructuredCids.size() * AttachmentSizes.size()) + UnstructuredCids.size()); CHECK(ScrubCtx.BadCids().GetSize() == 0); } @@ -2129,7 +2133,8 @@ TEST_CASE("cachestore.newgc.basics") {.Value = Record.second, .RawSize = Record.second.GetSize(), .RawHash = IoHash::HashBuffer(Record.second.GetData(), Record.second.GetSize())}, - AttachmentKeys); + AttachmentKeys, + false); for (const auto& Attachment : Attachments) { CidStore.AddChunk(Attachment.second.GetCompressed().Flatten().AsIoBuffer(), Attachment.second.DecodeRawHash()); @@ -2145,7 +2150,8 @@ TEST_CASE("cachestore.newgc.basics") {.Value = CacheValue.second, .RawSize = CacheValue.second.GetSize(), .RawHash = IoHash::HashBuffer(CacheValue.second.GetData(), CacheValue.second.GetSize())}, - {}); + {}, + false); CacheEntries.insert({Key, CacheEntry{CacheValue.second, {}}}); return Key; }; diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 460f0e10d..6b89beb3d 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -73,9 +73,12 @@ public: WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit) override; virtual void Flush() override; - virtual void ScrubStorage(ScrubContext& Ctx) override; virtual CidStoreSize TotalSize() const override; +#if ZEN_WITH_TESTS + virtual void Scrub(ScrubContext& Ctx) override; +#endif // ZEN_WITH_TESTS + private: CasContainerStrategy m_TinyStrategy; CasContainerStrategy m_SmallStrategy; @@ -195,7 +198,7 @@ CasImpl::OpenOrCreateManifest() } else { - ZEN_WARN("Store manifest validation failed: {:#x}, will generate new manifest to recover", uint32_t(ValidationResult)); + ZEN_WARN("Store manifest validation failed: {}, will generate new manifest to recover", ToString(ValidationResult)); } if (ManifestIsOk) @@ -463,24 +466,19 @@ CasImpl::Flush() m_LargeStrategy.Flush(); } +#if ZEN_WITH_TESTS void -CasImpl::ScrubStorage(ScrubContext& Ctx) +CasImpl::Scrub(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetCasTag()); ZEN_TRACE_CPU("Cas::ScrubStorage"); - if (m_LastScrubTime == Ctx.ScrubTimestamp()) - { - return; - } - - m_LastScrubTime = Ctx.ScrubTimestamp(); - m_SmallStrategy.ScrubStorage(Ctx); m_TinyStrategy.ScrubStorage(Ctx); m_LargeStrategy.ScrubStorage(Ctx); } +#endif // ZEN_WITH_TESTS CidStoreSize CasImpl::TotalSize() const @@ -523,7 +521,7 @@ TEST_CASE("CasStore") WorkerThreadPool ThreadPool{1}; ScrubContext Ctx{ThreadPool}; - Store->ScrubStorage(Ctx); + Store->Scrub(Ctx); IoBuffer Value1{16}; memcpy(Value1.MutableData(), "1234567890123456", 16); diff --git a/src/zenstore/cas.h b/src/zenstore/cas.h index e279dd2cc..0f6e2ba9d 100644 --- a/src/zenstore/cas.h +++ b/src/zenstore/cas.h @@ -50,12 +50,13 @@ public: WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit) = 0; virtual void Flush() = 0; - virtual void ScrubStorage(ScrubContext& Ctx) = 0; virtual CidStoreSize TotalSize() const = 0; +#if ZEN_WITH_TESTS + virtual void Scrub(ScrubContext& Ctx) = 0; +#endif // ZEN_WITH_TESTS protected: CidStoreConfiguration m_Config; - uint64_t m_LastScrubTime = 0; }; ZENCORE_API std::unique_ptr<CasStore> CreateCasStore(GcManager& Gc); diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp index 2ab769d04..ae1b59dc0 100644 --- a/src/zenstore/cidstore.cpp +++ b/src/zenstore/cidstore.cpp @@ -127,17 +127,9 @@ struct CidStore::Impl void Flush() { m_CasStore.Flush(); } - void ScrubStorage(ScrubContext& Ctx) - { - if (Ctx.ScrubTimestamp() == m_LastScrubTime) - { - return; - } - - m_LastScrubTime = Ctx.ScrubTimestamp(); - - m_CasStore.ScrubStorage(Ctx); - } +#if ZEN_WITH_TESTS + void Scrub(ScrubContext& Ctx) { m_CasStore.Scrub(Ctx); } +#endif // ZEN_WITH_TESTS CidStoreStats Stats() { @@ -236,11 +228,13 @@ CidStore::Flush() m_Impl->Flush(); } +#if ZEN_WITH_TESTS void -CidStore::ScrubStorage(ScrubContext& Ctx) +CidStore::Scrub(ScrubContext& Ctx) { - m_Impl->ScrubStorage(Ctx); + m_Impl->Scrub(Ctx); } +#endif // ZEN_WITH_TESTS CidStoreSize CidStore::TotalSize() const diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index 15e4cbf81..32c256a42 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -68,10 +68,10 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore void Flush(); // GcStorage - virtual void ScrubStorage(ScrubContext& ScrubCtx) override; virtual GcStorageSize StorageSize() const override; + // GcReferenceStore virtual std::string GetGcName(GcCtx& Ctx) override; virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override; diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h index adf48dc26..87b7dd812 100644 --- a/src/zenstore/include/zenstore/buildstore/buildstore.h +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -6,9 +6,8 @@ #include <zencore/iohash.h> #include <zenstore/accesstime.h> #include <zenstore/caslog.h> +#include <zenstore/cidstore.h> #include <zenstore/gc.h> -#include "../compactcas.h" -#include "../filecas.h" ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> @@ -19,18 +18,13 @@ namespace zen { struct BuildStoreConfig { std::filesystem::path RootDirectory; - uint32_t SmallBlobBlockStoreMaxBlockSize = 256 * 1024 * 1024; - uint64_t SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024; - uint32_t SmallBlobBlockStoreAlignement = 16; - uint32_t MetadataBlockStoreMaxBlockSize = 64 * 1024 * 1024; - uint32_t MetadataBlockStoreAlignement = 8; - uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB }; -class BuildStore : public GcReferencer, public GcReferenceLocker, public GcStorage +class BuildStore : public GcReferencer, public GcReferenceLocker { public: - explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc); + explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc, CidStore& BlobStore); virtual ~BuildStore(); void PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload); @@ -44,7 +38,7 @@ public: std::vector<BlobExistsResult> BlobsExists(std::span<const IoHash> BlobHashes); - void PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas); + void PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas, WorkerThreadPool* OptionalWorkerPool); std::vector<IoBuffer> GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool); void Flush(); @@ -52,10 +46,8 @@ public: struct StorageStats { uint64_t EntryCount = 0; - uint64_t LargeBlobCount = 0; - uint64_t LargeBlobBytes = 0; - uint64_t SmallBlobCount = 0; - uint64_t SmallBlobBytes = 0; + uint64_t BlobCount = 0; + uint64_t BlobBytes = 0; uint64_t MetadataCount = 0; uint64_t MetadataByteCount = 0; }; @@ -86,23 +78,18 @@ private: //////// GcReferenceLocker virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; - //////// GcStorage - virtual void ScrubStorage(ScrubContext& ScrubCtx) override; - virtual GcStorageSize StorageSize() const override; - #pragma pack(push) #pragma pack(1) struct PayloadEntry { PayloadEntry() {} - PayloadEntry(uint64_t Flags, uint64_t Size) + PayloadEntry(uint8_t Flags, uint64_t Size) { ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size); - ZEN_ASSERT((Flags & (kTombStone | kStandalone)) == Flags); + ZEN_ASSERT((Flags & (kTombStone)) == Flags); FlagsAndSize = (Size << 8) | Flags; } - static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value - static const uint8_t kStandalone = 0x20u; // This payload is stored as a standalone value + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value uint64_t FlagsAndSize = 0; uint64_t GetSize() const { return FlagsAndSize >> 8; } @@ -126,27 +113,47 @@ private: struct MetadataEntry { - BlockStoreLocation Location; // 12 bytes + IoHash MetadataHash; // 20 bytes + + MetadataEntry() {} - ZenContentType ContentType = ZenContentType::kCOUNT; // 1 byte - static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value - uint8_t Flags = 0; // 1 byte + MetadataEntry(const IoHash& Hash, uint64_t Size, ZenContentType ContentType, uint8_t Flags) + { + ZEN_ASSERT((Size & 0x0000ffffffffffffu) == Size); + ZEN_ASSERT((Flags & kTombStone) == Flags); + FlagsContentTypeAndSize = (Size << 16) | ((uint64_t)ContentType << 8) | Flags; + MetadataHash = Hash; + } + + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value + + uint64_t GetSize() const { return FlagsContentTypeAndSize >> 16; } + void SetSize(uint64_t Size) + { + ZEN_ASSERT((Size & 0x0000ffffffffffffu) == Size); + FlagsContentTypeAndSize = (Size << 16) | (FlagsContentTypeAndSize & 0xffff); + } + + uint8_t GetFlags() const { return uint8_t(FlagsContentTypeAndSize & 0xff); } + void AddFlag(uint8_t Flag) { FlagsContentTypeAndSize |= Flag; } + void SetFlags(uint8_t Flags) { FlagsContentTypeAndSize = (FlagsContentTypeAndSize & 0xffffffffffffff00u) | Flags; } + + ZenContentType GetContentType() const { return ZenContentType((FlagsContentTypeAndSize >> 8) & 0xff); } + void SetContentType(ZenContentType ContentType) + { + FlagsContentTypeAndSize = (FlagsContentTypeAndSize & 0xffffffffffff00ffu) | (uint16_t(ContentType) << 8); + } - uint8_t Reserved1 = 0; - uint8_t Reserved2 = 0; + uint64_t FlagsContentTypeAndSize = ((uint64_t)ZenContentType::kCOUNT << 8); }; - static_assert(sizeof(MetadataEntry) == 16); + static_assert(sizeof(MetadataEntry) == 28); struct MetadataDiskEntry { - MetadataEntry Entry; // 16 bytes + MetadataEntry Entry; // 28 bytes IoHash BlobHash; // 20 bytes - uint8_t Reserved1 = 0; - uint8_t Reserved2 = 0; - uint8_t Reserved3 = 0; - uint8_t Reserved4 = 0; }; - static_assert(sizeof(MetadataDiskEntry) == 40); + static_assert(sizeof(MetadataDiskEntry) == 48); #pragma pack(pop) @@ -206,17 +213,15 @@ private: std::vector<BlobEntry> m_BlobEntries; tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> m_BlobLookup; - FileCasStrategy m_LargeBlobStore; - CasContainerStrategy m_SmallBlobStore; - BlockStore m_MetadataBlockStore; + CidStore& m_BlobStore; TCasLogFile<PayloadDiskEntry> m_PayloadlogFile; TCasLogFile<MetadataDiskEntry> m_MetadatalogFile; uint64_t m_BlobLogFlushPosition = 0; uint64_t m_MetaLogFlushPosition = 0; - std::unique_ptr<HashSet> m_TrackedCacheKeys; - std::atomic<uint64_t> m_LastAccessTimeUpdateCount; + std::unique_ptr<std::vector<IoHash>> m_TrackedBlobKeys; + std::atomic<uint64_t> m_LastAccessTimeUpdateCount; friend class BuildStoreGcReferenceChecker; friend class BuildStoreGcReferencePruner; diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 3cd2d6423..49c52f847 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -106,6 +106,12 @@ static_assert(sizeof(DiskIndexEntry) == 32); ////////////////////////////////////////////////////////////////////////// +struct CacheStoreSize +{ + uint64_t DiskSize = 0; + uint64_t MemorySize = 0; +}; + class ZenCacheDiskLayer { public: @@ -115,6 +121,7 @@ public: uint32_t PayloadAlignment = 1u << 4; uint64_t MemCacheSizeThreshold = 1 * 1024; uint64_t LargeObjectThreshold = 128 * 1024; + bool LimitOverwrites = false; }; struct Configuration @@ -130,8 +137,8 @@ public: struct BucketInfo { - uint64_t EntryCount = 0; - GcStorageSize StorageSize; + uint64_t EntryCount = 0; + CacheStoreSize StorageSize; }; struct Info @@ -140,7 +147,7 @@ public: Configuration Config; std::vector<std::string> BucketNames; uint64_t EntryCount = 0; - GcStorageSize StorageSize; + CacheStoreSize StorageSize; }; struct BucketStats @@ -170,6 +177,12 @@ public: uint64_t MemorySize; }; + struct PutResult + { + zen::PutStatus Status; + std::string Message; + }; + explicit ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config); ~ZenCacheDiskLayer(); @@ -180,22 +193,22 @@ public: void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& BatchHandle); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult); + PutBatchHandle* BeginPutBatch(std::vector<PutResult>& OutResult); void EndPutBatch(PutBatchHandle* Batch) noexcept; - void Put(std::string_view Bucket, + PutResult Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, + bool Overwrite, PutBatchHandle* OptionalBatchHandle); std::function<void()> Drop(); std::function<void()> DropBucket(std::string_view Bucket); void Flush(); - void ScrubStorage(ScrubContext& Ctx); - void DiscoverBuckets(); - GcStorageSize StorageSize() const; - DiskStats Stats() const; + void DiscoverBuckets(); + CacheStoreSize TotalSize() const; + DiskStats Stats() const; Info GetInfo() const; std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const; @@ -212,6 +225,7 @@ public: #if ZEN_WITH_TESTS void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time); + void Scrub(ScrubContext& Ctx); #endif // ZEN_WITH_TESTS bool GetContentStats(std::string_view BucketName, CacheContentStats& OutContentStats) const; @@ -219,7 +233,7 @@ public: /** A cache bucket manages a single directory containing metadata and data for that bucket */ - struct CacheBucket : public GcReferencer + struct CacheBucket : public GcReferencer, public GcStorage { CacheBucket(GcManager& Gc, std::atomic_uint64_t& OuterCacheMemoryUsage, @@ -236,13 +250,22 @@ public: void Get(const IoHash& HashKey, GetBatchHandle& BatchHandle); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult); - void EndPutBatch(PutBatchHandle* Batch) noexcept; - void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, PutBatchHandle* OptionalBatchHandle); - uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime); - std::function<void()> Drop(); - void Flush(); - void ScrubStorage(ScrubContext& Ctx); + PutBatchHandle* BeginPutBatch(std::vector<ZenCacheDiskLayer::PutResult>& OutResult); + void EndPutBatch(PutBatchHandle* Batch) noexcept; + PutResult Put(const IoHash& HashKey, + const ZenCacheValue& Value, + std::span<IoHash> References, + bool Overwrite, + PutBatchHandle* OptionalBatchHandle); + uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime); + std::function<void()> Drop(); + void Flush(); + inline CacheStoreSize TotalSize() const + { + return {.DiskSize = m_StandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(), + .MemorySize = m_MemCachedSize.load(std::memory_order::relaxed)}; + } + RwLock::SharedLockScope GetGcReferencerLock(); struct ReferencesStats @@ -265,11 +288,6 @@ public: std::span<const std::size_t> ChunkIndexes, std::vector<IoHash>& OutReferences) const; - inline GcStorageSize StorageSize() const - { - return {.DiskSize = m_StandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(), - .MemorySize = m_MemCachedSize.load(std::memory_order::relaxed)}; - } uint64_t EntryCount() const; BucketStats Stats(); @@ -281,6 +299,20 @@ public: void SetAccessTime(const IoHash& HashKey, GcClock::TimePoint Time); #endif // ZEN_WITH_TESTS + // GcReferencer + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; + virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; + + // GcStorage + virtual void ScrubStorage(ScrubContext& Ctx) override; + virtual GcStorageSize StorageSize() const override + { + CacheStoreSize Size = TotalSize(); + return {.DiskSize = Size.DiskSize, .MemorySize = Size.MemorySize}; + } + private: #pragma pack(push) #pragma pack(1) @@ -379,19 +411,16 @@ public: std::atomic_uint64_t m_StandaloneSize{}; std::atomic_uint64_t m_MemCachedSize{}; - virtual std::string GetGcName(GcCtx& Ctx) override; - virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; - - void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const; - void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References); - IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const; - void PutInlineCacheValue(const IoHash& HashKey, - const ZenCacheValue& Value, - std::span<IoHash> References, - PutBatchHandle* OptionalBatchHandle = nullptr); - IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const; + void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const; + bool ShouldRejectPut(const IoHash& HashKey, ZenCacheValue& InOutValue, bool Overwrite, ZenCacheDiskLayer::PutResult& OutPutResult); + void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References); + IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const; + PutResult PutInlineCacheValue(const IoHash& HashKey, + const ZenCacheValue& Value, + std::span<IoHash> References, + bool Overwrite, + PutBatchHandle* OptionalBatchHandle = nullptr); + IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const; CacheValueDetails::ValueDetails GetValueDetails(RwLock::SharedLockScope&, const IoHash& Key, PayloadIndex Index) const; void SetMetaData(RwLock::ExclusiveLockScope&, diff --git a/src/zenstore/include/zenstore/cache/cacherpc.h b/src/zenstore/include/zenstore/cache/cacherpc.h index da8cf69fe..80340d72c 100644 --- a/src/zenstore/include/zenstore/cache/cacherpc.h +++ b/src/zenstore/include/zenstore/cache/cacherpc.h @@ -4,6 +4,7 @@ #include <zencore/iobuffer.h> #include <zencore/logging.h> +#include <zenstore/cache/cacheshared.h> #include <zenutil/cache/cache.h> #include <atomic> @@ -56,13 +57,6 @@ struct CacheStats std::atomic_uint64_t RpcChunkBatchRequests{}; }; -enum class PutResult -{ - Success, - Fail, - Invalid, -}; - /** Recognize both kBinary and kCompressedBinary as kCompressedBinary for structured cache value keys. We need this until the content type is preserved for kCompressedBinary when passing to and from upstream servers. @@ -76,11 +70,13 @@ IsCompressedBinary(ZenContentType Type) struct CacheRpcHandler { + typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc; + CacheRpcHandler(LoggerRef InLog, CacheStats& InCacheStats, UpstreamCacheClient& InUpstreamCache, ZenCacheStore& InCacheStore, - CidStore& InCidStore, + GetCidStoreFunc&& InGetCidStore, const DiskWriteBlocker* InDiskWriteBlocker); ~CacheRpcHandler(); @@ -100,6 +96,8 @@ struct CacheRpcHandler int& OutTargetProcessId, CbPackage& OutPackage); + CidStore& GetCidStore(std::string_view Namespace); + private: CbPackage HandleRpcPutCacheRecords(const CacheRequestContext& Context, const CbPackage& BatchRequest); CbPackage HandleRpcGetCacheRecords(const CacheRequestContext& Context, CbObjectView BatchRequest); @@ -107,7 +105,7 @@ private: CbPackage HandleRpcGetCacheValues(const CacheRequestContext& Context, CbObjectView BatchRequest); CbPackage HandleRpcGetCacheChunks(const CacheRequestContext& Context, RpcAcceptOptions AcceptOptions, CbObjectView BatchRequest); - PutResult PutCacheRecord(PutRequestData& Request, const CbPackage* Package); + PutStatus PutCacheRecord(PutRequestData& Request, const CbPackage* Package); /** HandleRpcGetCacheChunks Helper: Parse the Body object into RecordValue Requests and Value Requests. */ bool ParseGetCacheChunksRequest(std::string& Namespace, @@ -148,7 +146,7 @@ private: CacheStats& m_CacheStats; UpstreamCacheClient& m_UpstreamCache; ZenCacheStore& m_CacheStore; - CidStore& m_CidStore; + GetCidStoreFunc m_GetCidStore; const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; bool AreDiskWritesAllowed() const; diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index ef1b803de..8f40ae727 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -69,6 +69,14 @@ struct CacheContentStats std::vector<IoHash> Attachments; }; +enum class PutStatus +{ + Success, + Fail, + Conflict, + Invalid, +}; + bool IsKnownBadBucketName(std::string_view BucketName); bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer); diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 48fc17960..c51d7312c 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -49,7 +49,7 @@ class JobQueue; */ -class ZenCacheNamespace final : public GcStorage +class ZenCacheNamespace final { public: struct Configuration @@ -78,24 +78,27 @@ public: ZenCacheDiskLayer::DiskStats DiskStats; }; + using PutResult = ZenCacheDiskLayer::PutResult; + ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config); ~ZenCacheNamespace(); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResults); + PutBatchHandle* BeginPutBatch(std::vector<PutResult>& OutResults); void EndPutBatch(PutBatchHandle* Batch) noexcept; struct GetBatchHandle; GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResults); void EndGetBatch(GetBatchHandle* Batch) noexcept; - bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); - void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& OptionalBatchHandle); - void Put(std::string_view Bucket, - const IoHash& HashKey, - const ZenCacheValue& Value, - std::span<IoHash> References, - PutBatchHandle* OptionalBatchHandle = nullptr); + bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); + void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& OptionalBatchHandle); + PutResult Put(std::string_view Bucket, + const IoHash& HashKey, + const ZenCacheValue& Value, + std::span<IoHash> References, + bool Overwrite, + PutBatchHandle* OptionalBatchHandle = nullptr); bool DropBucket(std::string_view Bucket); void EnumerateBucketContents(std::string_view Bucket, @@ -104,10 +107,7 @@ public: std::function<void()> Drop(); void Flush(); - // GcStorage - virtual void ScrubStorage(ScrubContext& ScrubCtx) override; - virtual GcStorageSize StorageSize() const override; - + CacheStoreSize TotalSize() const; Configuration GetConfig() const { return m_Configuration; } Info GetInfo() const; std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const; @@ -124,6 +124,7 @@ public: #if ZEN_WITH_TESTS void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time); + void Scrub(ScrubContext& ScrubCtx); #endif // ZEN_WITH_TESTS private: @@ -137,7 +138,6 @@ private: std::atomic<uint64_t> m_WriteCount{}; metrics::RequestStats m_PutOps; metrics::RequestStats m_GetOps; - uint64_t m_LastScrubTime = 0; ZenCacheNamespace(const ZenCacheNamespace&) = delete; ZenCacheNamespace& operator=(const ZenCacheNamespace&) = delete; @@ -175,7 +175,7 @@ public: Configuration Config; std::vector<std::string> NamespaceNames; uint64_t DiskEntryCount = 0; - GcStorageSize StorageSize; + CacheStoreSize StorageSize; }; struct NamedNamespaceStats @@ -196,6 +196,8 @@ public: std::vector<NamedNamespaceStats> NamespaceStats; }; + using PutResult = ZenCacheNamespace::PutResult; + ZenCacheStore(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& BasePath, @@ -206,7 +208,7 @@ public: class PutBatch { public: - PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector<bool>& OutResult); + PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector<PutResult>& OutResult); ~PutBatch(); private: @@ -243,24 +245,24 @@ public: const IoHash& HashKey, GetBatch& BatchHandle); - void Put(const CacheRequestContext& Context, - std::string_view Namespace, - std::string_view Bucket, - const IoHash& HashKey, - const ZenCacheValue& Value, - std::span<IoHash> References, - PutBatch* OptionalBatchHandle = nullptr); + PutResult Put(const CacheRequestContext& Context, + std::string_view Namespace, + std::string_view Bucket, + const IoHash& HashKey, + const ZenCacheValue& Value, + std::span<IoHash> References, + bool Overwrite, + PutBatch* OptionalBatchHandle = nullptr); bool DropBucket(std::string_view Namespace, std::string_view Bucket); bool DropNamespace(std::string_view Namespace); void Flush(); - void ScrubStorage(ScrubContext& Ctx); CacheValueDetails GetValueDetails(const std::string_view NamespaceFilter, const std::string_view BucketFilter, const std::string_view ValueFilter) const; - GcStorageSize StorageSize() const; + CacheStoreSize TotalSize() const; CacheStoreStats Stats(bool IncludeNamespaceStats = true); Configuration GetConfiguration() const { return m_Configuration; } @@ -290,6 +292,10 @@ public: bool GetContentStats(std::string_view Namespace, std::string_view BucketName, CacheContentStats& OutContentStats) const; +#if ZEN_WITH_TESTS + void Scrub(ScrubContext& Ctx); +#endif // ZEN_WITH_TESTS + private: const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const; ZenCacheNamespace* GetNamespace(std::string_view Namespace); diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h index 152031c3a..c3993c028 100644 --- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h +++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h @@ -113,7 +113,7 @@ public: std::span<CacheChunkRequest*> CacheChunkRequests, OnCacheChunksGetComplete&& OnComplete) = 0; - virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) = 0; + virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) = 0; }; } // namespace zen diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h index b3d00fec0..8918b119f 100644 --- a/src/zenstore/include/zenstore/cidstore.h +++ b/src/zenstore/include/zenstore/cidstore.h @@ -87,12 +87,15 @@ public: bool ContainsChunk(const IoHash& DecompressedId); void FilterChunks(HashKeySet& InOutChunks); void Flush(); - void ScrubStorage(ScrubContext& Ctx); CidStoreSize TotalSize() const; CidStoreStats Stats() const; virtual void ReportMetrics(StatsMetrics& Statsd) override; +#if ZEN_WITH_TESTS + void Scrub(ScrubContext& Ctx); +#endif // ZEN_WITH_TESTS + private: struct Impl; std::unique_ptr<CasStore> m_CasStore; diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp index 88238effd..2171f4d62 100644 --- a/src/zenutil/buildstoragecache.cpp +++ b/src/zenutil/buildstoragecache.cpp @@ -378,6 +378,13 @@ private: m_Stats.TotalBytesRead += Result.DownloadedBytes; m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0); m_Stats.TotalRequestCount++; + SetAtomicMax(m_Stats.PeakSentBytes, Result.UploadedBytes); + SetAtomicMax(m_Stats.PeakReceivedBytes, Result.DownloadedBytes); + if (Result.ElapsedSeconds > 0.0) + { + uint64_t BytesPerSec = uint64_t((Result.UploadedBytes + Result.DownloadedBytes) / Result.ElapsedSeconds); + SetAtomicMax(m_Stats.PeakBytesPerSec, BytesPerSec); + } } HttpClient& m_HttpClient; diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index c2cc5ab3c..f75fe403f 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -40,19 +40,23 @@ public: ZEN_TRACE_CPU("FileBuildStorage::ListNamespaces"); ZEN_UNUSED(bRecursive); - SimulateLatency(0, 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); CbObjectWriter Writer; Writer.BeginArray("results"); { } Writer.EndArray(); // results - Writer.Save(); - SimulateLatency(Writer.GetSaveSize(), 0); + + Writer.Finalize(); + ReceivedBytes = Writer.GetSaveSize(); return Writer.Save(); } @@ -61,11 +65,14 @@ public: ZEN_TRACE_CPU("FileBuildStorage::ListBuilds"); ZEN_UNUSED(Query); - SimulateLatency(Query.GetSize(), 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = Query.GetSize(); + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BuildFolder = GetBuildsFolder(); DirectoryContent Content; @@ -88,19 +95,23 @@ public: } } Writer.EndArray(); // results - Writer.Save(); - SimulateLatency(Writer.GetSaveSize(), 0); + + Writer.Finalize(); + ReceivedBytes = Writer.GetSaveSize(); return Writer.Save(); } virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override { ZEN_TRACE_CPU("FileBuildStorage::PutBuild"); - SimulateLatency(MetaData.GetSize(), 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = MetaData.GetSize(); + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); CbObjectWriter BuildObject; BuildObject.AddObject("metadata", MetaData); @@ -109,35 +120,41 @@ public: CbObjectWriter BuildResponse; BuildResponse.AddInteger("chunkSize"sv, 32u * 1024u * 1024u); - BuildResponse.Save(); - - SimulateLatency(0, BuildResponse.GetSaveSize()); + BuildResponse.Finalize(); + ReceivedBytes = BuildResponse.GetSaveSize(); return BuildResponse.Save(); } virtual CbObject GetBuild(const Oid& BuildId) override { ZEN_TRACE_CPU("FileBuildStorage::GetBuild"); - SimulateLatency(0, 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); CbObject Build = ReadBuild(BuildId); - SimulateLatency(0, Build.GetSize()); + ReceivedBytes = Build.GetSize(); return Build; } virtual void FinalizeBuild(const Oid& BuildId) override { ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuild"); - SimulateLatency(0, 0); - Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; - ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); } virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid& BuildId, @@ -146,10 +163,14 @@ public: const CbObject& MetaData) override { ZEN_TRACE_CPU("FileBuildStorage::PutBuildPart"); - SimulateLatency(MetaData.GetSize(), 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = MetaData.GetSize(); + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId); CreateDirectories(BuildPartDataPath.parent_path()); @@ -184,7 +205,7 @@ public: std::vector<IoHash> NeededAttachments = GetNeededAttachments(MetaData); - SimulateLatency(0, sizeof(IoHash) * NeededAttachments.size()); + ReceivedBytes = sizeof(IoHash) * NeededAttachments.size(); return std::make_pair(RawHash, std::move(NeededAttachments)); } @@ -192,22 +213,24 @@ public: virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override { ZEN_TRACE_CPU("FileBuildStorage::GetBuildPart"); - SimulateLatency(0, 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId); IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten(); - m_Stats.TotalBytesRead += Payload.GetSize(); ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None); CbObject BuildPartObject = CbObject(SharedBuffer(Payload)); - SimulateLatency(0, BuildPartObject.GetSize()); + ReceivedBytes = BuildPartObject.GetSize(); return BuildPartObject; } @@ -215,15 +238,18 @@ public: virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override { ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuildPart"); - SimulateLatency(0, 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId); IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten(); - m_Stats.TotalBytesRead += Payload.GetSize(); + IoHash RawHash = IoHash::HashBuffer(Payload.GetView()); if (RawHash != PartHash) { @@ -234,7 +260,7 @@ public: CbObject BuildPartObject = CbObject(SharedBuffer(Payload)); std::vector<IoHash> NeededAttachments(GetNeededAttachments(BuildPartObject)); - SimulateLatency(0, NeededAttachments.size() * sizeof(IoHash)); + ReceivedBytes = NeededAttachments.size() * sizeof(IoHash); return NeededAttachments; } @@ -247,13 +273,14 @@ public: ZEN_TRACE_CPU("FileBuildStorage::PutBuildBlob"); ZEN_UNUSED(BuildId); ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); - SimulateLatency(Payload.GetSize(), 0); ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, Payload)); - Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = Payload.GetSize(); + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); if (!IsFile(BlockPath)) @@ -261,8 +288,8 @@ public: CreateDirectories(BlockPath.parent_path()); TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView()); } - m_Stats.TotalBytesWritten += Payload.GetSize(); - SimulateLatency(0, 0); + + ReceivedBytes = Payload.GetSize(); } virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId, @@ -275,10 +302,15 @@ public: ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob"); ZEN_UNUSED(BuildId); ZEN_UNUSED(ContentType); - SimulateLatency(0, 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); if (!IsFile(BlockPath)) @@ -314,7 +346,15 @@ public: WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() { ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob_Work"); IoBuffer PartPayload = Workload->Transmitter(Offset, Size); - SimulateLatency(PartPayload.GetSize(), 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = PartPayload.GetSize(); + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); std::error_code Ec; Workload->TempFile.Write(PartPayload, Offset, Ec); @@ -325,8 +365,7 @@ public: Ec.message(), Ec.value())); } - uint64_t BytesWritten = PartPayload.GetSize(); - m_Stats.TotalBytesWritten += BytesWritten; + const bool IsLastPart = Workload->PartsLeft.fetch_sub(1) == 1; if (IsLastPart) { @@ -342,18 +381,14 @@ public: Ec.value())); } } - Workload->OnSentBytes(BytesWritten, IsLastPart); - SimulateLatency(0, 0); + Workload->OnSentBytes(SentBytes, IsLastPart); }); Offset += Size; } Workload->PartsLeft.store(WorkItems.size()); - - SimulateLatency(0, 0); return WorkItems; } - SimulateLatency(0, 0); return {}; } @@ -361,10 +396,15 @@ public: { ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlob"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); if (IsFile(BlockPath)) @@ -382,11 +422,9 @@ public: ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload)))); } Payload.SetContentType(ZenContentType::kCompressedBinary); - m_Stats.TotalBytesRead += Payload.GetSize(); - SimulateLatency(0, Payload.GetSize()); + ReceivedBytes = Payload.GetSize(); return Payload; } - SimulateLatency(0, 0); return IoBuffer{}; } @@ -398,10 +436,15 @@ public: { ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); if (IsFile(BlockPath)) @@ -429,22 +472,29 @@ public: uint64_t Size = Min(ChunkSize, BlobSize - Offset); WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() { ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob_Work"); - SimulateLatency(0, 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); + IoBuffer PartPayload(Size); Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset); - m_Stats.TotalBytesRead += PartPayload.GetSize(); + ReceivedBytes = PartPayload.GetSize(); Workload->OnReceive(Offset, PartPayload); uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size); if (ByteRemaning == Size) { Workload->OnComplete(); } - SimulateLatency(Size, PartPayload.GetSize()); }); Offset += Size; } - SimulateLatency(0, 0); return WorkItems; } return {}; @@ -455,18 +505,19 @@ public: ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata"); ZEN_UNUSED(BuildId); - SimulateLatency(MetaData.GetSize(), 0); + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = MetaData.GetSize(); + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); const std::filesystem::path BlockMetaDataPath = GetBlobMetadataPath(BlockRawHash); CreateDirectories(BlockMetaDataPath.parent_path()); TemporaryFile::SafeWriteFile(BlockMetaDataPath, MetaData.GetView()); - m_Stats.TotalBytesWritten += MetaData.GetSize(); WriteAsJson(BlockMetaDataPath, MetaData); - SimulateLatency(0, 0); return true; } @@ -474,10 +525,15 @@ public: { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); - SimulateLatency(sizeof(BuildId), 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); uint64_t FoundCount = 0; @@ -495,8 +551,6 @@ public: { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); - m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); - CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); Writer.AddObject(BlockObject); FoundCount++; @@ -508,19 +562,25 @@ public: } } Writer.EndArray(); // blocks - CbObject Result = Writer.Save(); - SimulateLatency(0, Result.GetSize()); - return Result; + + Writer.Finalize(); + ReceivedBytes = Writer.GetSaveSize(); + return Writer.Save(); } virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override { ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata"); ZEN_UNUSED(BuildId); - SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(); + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - m_Stats.TotalRequestCount++; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); CbObjectWriter Writer; Writer.BeginArray("blocks"); @@ -532,22 +592,29 @@ public: { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); - m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); - CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); Writer.AddObject(BlockObject); } } Writer.EndArray(); // blocks - CbObject Result = Writer.Save(); - SimulateLatency(0, Result.GetSize()); - return Result; + Writer.Finalize(); + ReceivedBytes = Writer.GetSaveSize(); + return Writer.Save(); } virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map<std::string, double>& FloatStats) override { + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = 0; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); + CbObjectWriter Request; Request.BeginObject("floatStats"sv); for (auto It : FloatStats) @@ -555,17 +622,16 @@ public: Request.AddFloat(It.first, It.second); } Request.EndObject(); - CbObject Payload = Request.Save(); - - SimulateLatency(Payload.GetSize(), 0); + Request.Finalize(); + SentBytes = Request.GetSaveSize(); const std::filesystem::path BuildPartStatsDataPath = GetBuildPartStatsPath(BuildId, BuildPartId); CreateDirectories(BuildPartStatsDataPath.parent_path()); + CbObject Payload = Request.Save(); + TemporaryFile::SafeWriteFile(BuildPartStatsDataPath, Payload.GetView()); WriteAsJson(BuildPartStatsDataPath, Payload); - - SimulateLatency(0, 0); } protected: @@ -629,7 +695,6 @@ protected: const std::filesystem::path BuildDataPath = GetBuildPath(BuildId); CreateDirectories(BuildDataPath.parent_path()); TemporaryFile::SafeWriteFile(BuildDataPath, Data.GetView()); - m_Stats.TotalBytesWritten += Data.GetSize(); WriteAsJson(BuildDataPath, Data); } @@ -646,7 +711,6 @@ protected: Content.ErrorCode.value())); } IoBuffer Payload = Content.Flatten(); - m_Stats.TotalBytesRead += Payload.GetSize(); ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None); CbObject BuildObject = CbObject(SharedBuffer(Payload)); return BuildObject; @@ -704,6 +768,22 @@ protected: } private: + void AddStatistic(Stopwatch& ExecutionTimer, uint64_t UploadedBytes, uint64_t DownloadedBytes) + { + const uint64_t ElapsedUs = ExecutionTimer.GetElapsedTimeUs(); + m_Stats.TotalBytesWritten += UploadedBytes; + m_Stats.TotalBytesRead += DownloadedBytes; + m_Stats.TotalRequestTimeUs += ElapsedUs; + m_Stats.TotalRequestCount++; + SetAtomicMax(m_Stats.PeakSentBytes, UploadedBytes); + SetAtomicMax(m_Stats.PeakReceivedBytes, DownloadedBytes); + if (ElapsedUs > 0) + { + uint64_t BytesPerSec = ((UploadedBytes + DownloadedBytes) * 1000000u) / ElapsedUs; + SetAtomicMax(m_Stats.PeakBytesPerSec, BytesPerSec); + } + } + const std::filesystem::path m_StoragePath; BuildStorage::Statistics& m_Stats; const bool m_EnableJsonOutput = false; diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index f49d4b42a..46ecd0a11 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -21,6 +21,9 @@ public: std::atomic<uint64_t> TotalRequestCount = 0; std::atomic<uint64_t> TotalRequestTimeUs = 0; std::atomic<uint64_t> TotalExecutionTimeUs = 0; + std::atomic<uint64_t> PeakSentBytes = 0; + std::atomic<uint64_t> PeakReceivedBytes = 0; + std::atomic<uint64_t> PeakBytesPerSec = 0; }; virtual ~BuildStorage() {} diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h index e1fb73fd4..a0690a16a 100644 --- a/src/zenutil/include/zenutil/buildstoragecache.h +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -22,6 +22,9 @@ public: std::atomic<uint64_t> TotalRequestCount = 0; std::atomic<uint64_t> TotalRequestTimeUs = 0; std::atomic<uint64_t> TotalExecutionTimeUs = 0; + std::atomic<uint64_t> PeakSentBytes = 0; + std::atomic<uint64_t> PeakReceivedBytes = 0; + std::atomic<uint64_t> PeakBytesPerSec = 0; }; virtual ~BuildStorageCache() {} diff --git a/src/zenutil/include/zenutil/wildcard.h b/src/zenutil/include/zenutil/wildcard.h new file mode 100644 index 000000000..9f402e100 --- /dev/null +++ b/src/zenutil/include/zenutil/wildcard.h @@ -0,0 +1,13 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <string_view> + +namespace zen { + +bool MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive); + +void wildcard_forcelink(); // internal + +} // namespace zen diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 9974725ff..c9278acb4 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -17,6 +17,16 @@ namespace zen { using namespace std::literals; +namespace { + void ThrowFromJupiterResult(const JupiterResult& Result, std::string_view Prefix) + { + int Error = Result.ErrorCode < (int)HttpResponseCode::Continue ? Result.ErrorCode : 0; + HttpResponseCode Status = + Result.ErrorCode >= int(HttpResponseCode::Continue) ? HttpResponseCode(Result.ErrorCode) : HttpResponseCode::ImATeapot; + throw HttpClientError(fmt::format("{}: {} ({})", Prefix, Result.Reason, Result.ErrorCode), Error, Status); + } +} // namespace + class JupiterBuildStorage : public BuildStorage { public: @@ -46,7 +56,7 @@ public: AddStatistic(ListResult); if (!ListResult.Success) { - throw std::runtime_error(fmt::format("Failed listing namespaces: {} ({})", ListResult.Reason, ListResult.ErrorCode)); + ThrowFromJupiterResult(ListResult, "Failed listing namespaces"); } CbObject NamespaceResponse = PayloadToCbObject("Failed listing namespaces"sv, ListResult.Response); @@ -66,10 +76,10 @@ public: AddStatistic(BucketsResult); if (!BucketsResult.Success) { - throw std::runtime_error( - fmt::format("Failed listing namespaces: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode)); + ThrowFromJupiterResult(BucketsResult, fmt::format("Failed listing buckets in namespace {}", Namespace)); } - CbObject BucketResponse = PayloadToCbObject("Failed listing namespaces"sv, BucketsResult.Response); + CbObject BucketResponse = + PayloadToCbObject(fmt::format("Failed listing buckets in namespace {}", Namespace), BucketsResult.Response); Response.BeginArray("items"); for (CbFieldView BucketField : BucketResponse["buckets"]) @@ -103,7 +113,7 @@ public: AddStatistic(ListResult); if (!ListResult.Success) { - throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode)); + ThrowFromJupiterResult(ListResult, "Failed listing builds"sv); } return PayloadToCbObject("Failed listing builds"sv, ListResult.Response); } @@ -120,7 +130,7 @@ public: AddStatistic(PutResult); if (!PutResult.Success) { - throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode)); + ThrowFromJupiterResult(PutResult, "Failed creating build"sv); } return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); } @@ -135,7 +145,7 @@ public: AddStatistic(GetBuildResult); if (!GetBuildResult.Success) { - throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode)); + ThrowFromJupiterResult(GetBuildResult, "Failed fetching build"sv); } return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); } @@ -150,8 +160,7 @@ public: AddStatistic(FinalizeBuildResult); if (!FinalizeBuildResult.Success) { - throw std::runtime_error( - fmt::format("Failed finalizing build part: {} ({})", FinalizeBuildResult.Reason, FinalizeBuildResult.ErrorCode)); + ThrowFromJupiterResult(FinalizeBuildResult, "Failed finalizing build"sv); } } @@ -170,7 +179,7 @@ public: AddStatistic(PutPartResult); if (!PutPartResult.Success) { - throw std::runtime_error(fmt::format("Failed creating build part: {} ({})", PutPartResult.Reason, PutPartResult.ErrorCode)); + ThrowFromJupiterResult(PutPartResult, "Failed creating build part"sv); } return std::make_pair(PutPartResult.RawHash, std::move(PutPartResult.Needs)); } @@ -185,10 +194,7 @@ public: AddStatistic(GetBuildPartResult); if (!GetBuildPartResult.Success) { - throw std::runtime_error(fmt::format("Failed fetching build part {}: {} ({})", - BuildPartId, - GetBuildPartResult.Reason, - GetBuildPartResult.ErrorCode)); + ThrowFromJupiterResult(GetBuildPartResult, "Failed fetching build part"sv); } return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); } @@ -203,8 +209,7 @@ public: AddStatistic(FinalizePartResult); if (!FinalizePartResult.Success) { - throw std::runtime_error( - fmt::format("Failed finalizing build part: {} ({})", FinalizePartResult.Reason, FinalizePartResult.ErrorCode)); + ThrowFromJupiterResult(FinalizePartResult, "Failed finalizing build part"sv); } return std::move(FinalizePartResult.Needs); } @@ -222,7 +227,7 @@ public: AddStatistic(PutBlobResult); if (!PutBlobResult.Success) { - throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PutBlobResult.Reason, PutBlobResult.ErrorCode)); + ThrowFromJupiterResult(PutBlobResult, "Failed putting build part"sv); } } @@ -249,8 +254,7 @@ public: AddStatistic(PutMultipartBlobResult); if (!PutMultipartBlobResult.Success) { - throw std::runtime_error( - fmt::format("Failed putting build part: {} ({})", PutMultipartBlobResult.Reason, PutMultipartBlobResult.ErrorCode)); + ThrowFromJupiterResult(PutMultipartBlobResult, "Failed putting large build blob"sv); } OnSentBytes(PutMultipartBlobResult.SentBytes, WorkItems.empty()); @@ -265,7 +269,7 @@ public: AddStatistic(PartResult); if (!PartResult.Success) { - throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode)); + ThrowFromJupiterResult(PartResult, "Failed putting large build blob"sv); } OnSentBytes(PartResult.SentBytes, IsComplete); }); @@ -285,8 +289,7 @@ public: AddStatistic(GetBuildBlobResult); if (!GetBuildBlobResult.Success) { - throw std::runtime_error( - fmt::format("Failed fetching build blob {}: {} ({})", RawHash, GetBuildBlobResult.Reason, GetBuildBlobResult.ErrorCode)); + ThrowFromJupiterResult(GetBuildBlobResult, "Failed fetching build blob"sv); } return std::move(GetBuildBlobResult.Response); } @@ -314,8 +317,7 @@ public: AddStatistic(GetMultipartBlobResult); if (!GetMultipartBlobResult.Success) { - throw std::runtime_error( - fmt::format("Failed getting build part: {} ({})", GetMultipartBlobResult.Reason, GetMultipartBlobResult.ErrorCode)); + ThrowFromJupiterResult(GetMultipartBlobResult, "Failed getting large build part"sv); } std::vector<std::function<void()>> WorkList; for (auto& WorkItem : WorkItems) @@ -327,7 +329,7 @@ public: AddStatistic(PartResult); if (!PartResult.Success) { - throw std::runtime_error(fmt::format("Failed getting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode)); + ThrowFromJupiterResult(PartResult, "Failed getting large build part"sv); } }); } @@ -350,8 +352,7 @@ public: { return false; } - throw std::runtime_error( - fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode)); + ThrowFromJupiterResult(PutMetaResult, "Failed putting build block metadata"sv); } return true; } @@ -366,7 +367,7 @@ public: AddStatistic(FindResult); if (!FindResult.Success) { - throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode)); + ThrowFromJupiterResult(FindResult, "Failed fetching known blocks"sv); } return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response); } @@ -392,8 +393,7 @@ public: AddStatistic(GetBlockMetadataResult); if (!GetBlockMetadataResult.Success) { - throw std::runtime_error( - fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode)); + ThrowFromJupiterResult(GetBlockMetadataResult, "Failed fetching block metadatas"sv); } return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response); } @@ -416,14 +416,12 @@ public: AddStatistic(PutBuildPartStatsResult); if (!PutBuildPartStatsResult.Success) { - throw std::runtime_error(fmt::format("Failed posting build part statistics: {} ({})", - PutBuildPartStatsResult.Reason, - PutBuildPartStatsResult.ErrorCode)); + ThrowFromJupiterResult(PutBuildPartStatsResult, "Failed posting build part statistics"sv); } } private: - static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload) + static CbObject PayloadToCbObject(std::string_view ErrorContext, const IoBuffer& Payload) { if (Payload.GetContentType() == ZenContentType::kJSON) { @@ -443,7 +441,7 @@ private: else { throw std::runtime_error( - fmt::format("{}: {} ({})", "Unsupported response format", Context, ToString(Payload.GetContentType()))); + fmt::format("{}: {} ({})", "Unsupported response format", ErrorContext, ToString(Payload.GetContentType()))); } } @@ -453,6 +451,14 @@ private: m_Stats.TotalBytesRead += Result.ReceivedBytes; m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0); m_Stats.TotalRequestCount++; + + SetAtomicMax(m_Stats.PeakSentBytes, Result.SentBytes); + SetAtomicMax(m_Stats.PeakReceivedBytes, Result.ReceivedBytes); + if (Result.ElapsedSeconds > 0.0) + { + uint64_t BytesPerSec = uint64_t((Result.SentBytes + Result.ReceivedBytes) / Result.ElapsedSeconds); + SetAtomicMax(m_Stats.PeakBytesPerSec, BytesPerSec); + } } JupiterSession m_Session; diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp index aa806438b..a571d1d11 100644 --- a/src/zenutil/parallelwork.cpp +++ b/src/zenutil/parallelwork.cpp @@ -33,18 +33,21 @@ ParallelWork::~ParallelWork() "ParallelWork disposed without explicit wait for completion, likely caused by an exception, waiting for dispatched threads " "to complete"); m_PendingWork.CountDown(); + m_DispatchComplete = true; + } + const bool WaitSucceeded = m_PendingWork.Wait(); + const ptrdiff_t RemainingWork = m_PendingWork.Remaining(); + if (!WaitSucceeded) + { + ZEN_ERROR("ParallelWork::~ParallelWork(): waiting for latch failed, pending work count at {}", RemainingWork); } - m_PendingWork.Wait(); - ptrdiff_t RemainingWork = m_PendingWork.Remaining(); if (RemainingWork != 0) { void* Frames[8]; uint32_t FrameCount = GetCallstack(2, 8, Frames); CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames); - ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}", - RemainingWork, - CallstackToString(Callstack, " ")); - FreeCallstack(Callstack); + auto _ = MakeGuard([Callstack]() { FreeCallstack(Callstack); }); + ZEN_WARN("ParallelWork::~ParallelWork(): waited for outstanding work but pending work count is {} instead of 0", RemainingWork); uint32_t WaitedMs = 0; while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000) @@ -52,20 +55,27 @@ ParallelWork::~ParallelWork() Sleep(50); WaitedMs += 50; } - RemainingWork = m_PendingWork.Remaining(); - if (RemainingWork != 0) + ptrdiff_t RemainingWorkAfterSafetyWait = m_PendingWork.Remaining(); + if (RemainingWorkAfterSafetyWait != 0) { - ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork) + ZEN_ERROR("ParallelWork::~ParallelWork(): safety wait for {} tasks failed, pending work count at {} after {}\n{}", + RemainingWork, + RemainingWorkAfterSafetyWait, + NiceLatencyNs(WaitedMs * 1000000u), + CallstackToString(Callstack, " ")) } else { - ZEN_INFO("ParallelWork destructor safety wait succeeded"); + ZEN_ERROR("ParallelWork::~ParallelWork(): safety wait for {} tasks completed after {}\n{}", + RemainingWork, + NiceLatencyNs(WaitedMs * 1000000u), + CallstackToString(Callstack, " ")); } } } catch (const std::exception& Ex) { - ZEN_ERROR("Exception in ~ParallelWork: {}", Ex.what()); + ZEN_ERROR("Exception in ParallelWork::~ParallelWork(): {}", Ex.what()); } } @@ -82,10 +92,9 @@ void ParallelWork::Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback) { ZEN_ASSERT(!m_DispatchComplete); - m_DispatchComplete = true; - ZEN_ASSERT(m_PendingWork.Remaining() > 0); m_PendingWork.CountDown(); + m_DispatchComplete = true; while (!m_PendingWork.Wait(UpdateIntervalMS)) { @@ -99,11 +108,20 @@ void ParallelWork::Wait() { ZEN_ASSERT(!m_DispatchComplete); - m_DispatchComplete = true; - ZEN_ASSERT(m_PendingWork.Remaining() > 0); m_PendingWork.CountDown(); - m_PendingWork.Wait(); + m_DispatchComplete = true; + + const bool WaitSucceeded = m_PendingWork.Wait(); + const ptrdiff_t RemainingWork = m_PendingWork.Remaining(); + if (!WaitSucceeded) + { + ZEN_ERROR("ParallelWork::Wait(): waiting for latch failed, pending work count at {}", RemainingWork); + } + else if (RemainingWork != 0) + { + ZEN_ERROR("ParallelWork::Wait(): pending work count at {} after successful wait for latch", RemainingWork); + } RethrowErrors(); } diff --git a/src/zenutil/wildcard.cpp b/src/zenutil/wildcard.cpp new file mode 100644 index 000000000..df69f6a5e --- /dev/null +++ b/src/zenutil/wildcard.cpp @@ -0,0 +1,112 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/string.h> +#include <zenutil/wildcard.h> + +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +#endif // ZEN_WITH_TESTS + +namespace zen { + +bool +MatchWildcard(std::string_view::const_iterator WildcardIt, + std::string_view::const_iterator WildcardEnd, + std::string_view::const_iterator StringIt, + std::string_view::const_iterator StringEnd) +{ + for (; WildcardIt != WildcardEnd; WildcardIt++) + { + switch (*WildcardIt) + { + case '?': + if (StringIt == StringEnd) + { + return false; + } + StringIt++; + break; + case '*': + { + if ((WildcardIt + 1) == WildcardEnd) + { + return true; + } + size_t Max = std::distance(StringIt, StringEnd); + for (size_t i = 0; i < Max; i++) + { + if (MatchWildcard(WildcardIt + 1, WildcardEnd, StringIt + i, StringEnd)) + { + return true; + } + } + return false; + } + default: + if (*StringIt != *WildcardIt) + { + return false; + } + ++StringIt; + } + } + return StringIt == StringEnd; +} + +bool +MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive) +{ + if (CaseSensitive) + { + return MatchWildcard(begin(Wildcard), end(Wildcard), begin(String), end(String)); + } + else + { + std::string LowercaseWildcard = ToLower(Wildcard); + std::string LowercaseString = ToLower(String); + std::string_view LowercaseWildcardView(LowercaseWildcard); + std::string_view LowercaseStringView(LowercaseString); + return MatchWildcard(begin(LowercaseWildcardView), + end(LowercaseWildcardView), + begin(LowercaseStringView), + end(LowercaseStringView)); + } +} + +#if ZEN_WITH_TESTS + +void +wildcard_forcelink() +{ +} + +TEST_CASE("Wildcard") +{ + CHECK(MatchWildcard("*.*", "normal.txt", true)); + CHECK(MatchWildcard("*.*", "normal.txt", false)); + + CHECK(!MatchWildcard("*.*", "normal", true)); + CHECK(!MatchWildcard("*.*", "normal", false)); + + CHECK(MatchWildcard("*", "hey/normal", true)); + CHECK(MatchWildcard("*", "hey/normal", false)); + + CHECK(MatchWildcard("hey/*.txt", "hey/normal.txt", true)); + CHECK(MatchWildcard("*/?ormal.txt", "hey/normal.txt", true)); + CHECK(!MatchWildcard("*/?rmal.txt", "hey/normal.txt", true)); + CHECK(MatchWildcard("*/?ormal.*", "hey/normal.txt", true)); + CHECK(MatchWildcard("*/?ormal", "hey/normal", true)); + + CHECK(MatchWildcard("hey/*.txt", "hey/normaL.txt", false)); + CHECK(MatchWildcard("*/?ormal.TXT", "hey/normal.txt", false)); + CHECK(MatchWildcard("*/?ORMAL.*", "hey/normal.txt", false)); + CHECK(MatchWildcard("*/?ormal", "HEY/normal", false)); + + CHECK(!MatchWildcard("hey/*.txt", "heY/normal.txt", true)); + CHECK(!MatchWildcard("*/?ormal.TXT", "hey/normal.txt", true)); + CHECK(!MatchWildcard("*/?ORMAL.*", "hey/normal.txt", true)); + CHECK(!MatchWildcard("*/?ormal", "hey/normaL", true)); +} + +#endif +} // namespace zen diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp index fe23b00c1..37b229c49 100644 --- a/src/zenutil/zenutil.cpp +++ b/src/zenutil/zenutil.cpp @@ -9,6 +9,7 @@ # include <zenutil/chunkedfile.h> # include <zenutil/commandlineoptions.h> # include <zenutil/parallelwork.h> +# include <zenutil/wildcard.h> namespace zen { @@ -21,6 +22,7 @@ zenutil_forcelinktests() chunkedfile_forcelink(); commandlineoptions_forcelink(); parallellwork_forcelink(); + wildcard_forcelink(); } } // namespace zen |