aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorLiam Mitchell <[email protected]>2025-08-21 23:58:51 +0000
committerLiam Mitchell <[email protected]>2025-08-21 23:58:51 +0000
commit33209bd6931f49362dfc2d62c6cb6b87a42c99e1 (patch)
treecfc7914634088b3f4feac2d4cec0b5650dfdcc3c /src
parentFix changelog merge issues (diff)
parentavoid new in static IoBuffer (#472) (diff)
downloadzen-33209bd6931f49362dfc2d62c6cb6b87a42c99e1.tar.xz
zen-33209bd6931f49362dfc2d62c6cb6b87a42c99e1.zip
Merge remote-tracking branch 'origin/main' into de/zen-service-command
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/builds_cmd.cpp1236
-rw-r--r--src/zen/cmds/builds_cmd.h8
-rw-r--r--src/zen/cmds/print_cmd.cpp93
-rw-r--r--src/zen/cmds/print_cmd.h10
-rw-r--r--src/zen/cmds/projectstore_cmd.cpp95
-rw-r--r--src/zen/zen.cpp3
-rw-r--r--src/zen/zen.h3
-rw-r--r--src/zencore/callstack.cpp25
-rw-r--r--src/zencore/compactbinaryfile.cpp2
-rw-r--r--src/zencore/compactbinaryjson.cpp128
-rw-r--r--src/zencore/compactbinaryvalidation.cpp9
-rw-r--r--src/zencore/include/zencore/compactbinary.h4
-rw-r--r--src/zencore/include/zencore/jobqueue.h11
-rw-r--r--src/zencore/iobuffer.cpp20
-rw-r--r--src/zencore/jobqueue.cpp22
-rw-r--r--src/zencore/thread.cpp10
-rw-r--r--src/zencore/zencore.cpp9
-rw-r--r--src/zenhttp/httpclient.cpp7
-rw-r--r--src/zenhttp/httpclientauth.cpp23
-rw-r--r--src/zenhttp/include/zenhttp/httpclient.h23
-rw-r--r--src/zenhttp/include/zenhttp/httpclientauth.h3
-rw-r--r--src/zenhttp/packageformat.cpp2
-rw-r--r--src/zenserver-test/zenserver-test.cpp828
-rw-r--r--src/zenserver/admin/admin.cpp29
-rw-r--r--src/zenserver/admin/admin.h12
-rw-r--r--src/zenserver/buildstore/httpbuildstore.cpp22
-rw-r--r--src/zenserver/cache/httpstructuredcache.cpp420
-rw-r--r--src/zenserver/cache/httpstructuredcache.h7
-rw-r--r--src/zenserver/config.cpp13
-rw-r--r--src/zenserver/config.h1
-rw-r--r--src/zenserver/frontend/html.zipbin161002 -> 161185 bytes
-rw-r--r--src/zenserver/frontend/html/indexer/worker.js46
-rw-r--r--src/zenserver/frontend/html/pages/entry.js36
-rw-r--r--src/zenserver/main.cpp17
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp618
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.h5
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp167
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h7
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.cpp4
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.h3
-rw-r--r--src/zenserver/projectstore/projectstore.cpp200
-rw-r--r--src/zenserver/projectstore/projectstore.h13
-rw-r--r--src/zenserver/upstream/upstreamcache.cpp38
-rw-r--r--src/zenserver/upstream/upstreamcache.h3
-rw-r--r--src/zenserver/zenserver.cpp100
-rw-r--r--src/zenserver/zenserver.h28
-rw-r--r--src/zenstore/buildstore/buildstore.cpp782
-rw-r--r--src/zenstore/cache/cachedisklayer.cpp320
-rw-r--r--src/zenstore/cache/cacherpc.cpp288
-rw-r--r--src/zenstore/cache/structuredcachestore.cpp134
-rw-r--r--src/zenstore/cas.cpp20
-rw-r--r--src/zenstore/cas.h5
-rw-r--r--src/zenstore/cidstore.cpp20
-rw-r--r--src/zenstore/compactcas.h2
-rw-r--r--src/zenstore/include/zenstore/buildstore/buildstore.h87
-rw-r--r--src/zenstore/include/zenstore/cache/cachedisklayer.h99
-rw-r--r--src/zenstore/include/zenstore/cache/cacherpc.h18
-rw-r--r--src/zenstore/include/zenstore/cache/cacheshared.h8
-rw-r--r--src/zenstore/include/zenstore/cache/structuredcachestore.h56
-rw-r--r--src/zenstore/include/zenstore/cache/upstreamcacheclient.h2
-rw-r--r--src/zenstore/include/zenstore/cidstore.h5
-rw-r--r--src/zenutil/buildstoragecache.cpp7
-rw-r--r--src/zenutil/filebuildstorage.cpp276
-rw-r--r--src/zenutil/include/zenutil/buildstorage.h3
-rw-r--r--src/zenutil/include/zenutil/buildstoragecache.h3
-rw-r--r--src/zenutil/include/zenutil/wildcard.h13
-rw-r--r--src/zenutil/jupiter/jupiterbuildstorage.cpp76
-rw-r--r--src/zenutil/parallelwork.cpp50
-rw-r--r--src/zenutil/wildcard.cpp112
-rw-r--r--src/zenutil/zenutil.cpp2
70 files changed, 4543 insertions, 2208 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index cd27daa9e..8692e5941 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -33,6 +33,7 @@
#include <zenutil/jupiter/jupiterbuildstorage.h>
#include <zenutil/jupiter/jupitersession.h>
#include <zenutil/parallelwork.h>
+#include <zenutil/wildcard.h>
#include <zenutil/workerpools.h>
#include <zenutil/zenserverprocess.h>
@@ -65,6 +66,7 @@ using namespace std::literals;
namespace {
namespace zenutil {
+
#if ZEN_PLATFORM_WINDOWS
class SecurityAttributes
{
@@ -368,6 +370,7 @@ namespace {
const std::vector<std::string_view> DefaultExcludeExtensions({});
static bool IsVerbose = false;
+ static bool IsQuiet = false;
static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode)
@@ -402,6 +405,20 @@ namespace {
);
+ bool IncludePath(const std::string_view IncludeWildcard, const std::string_view ExcludeWildcard, const std::filesystem::path& Path)
+ {
+ const std::string PathString = Path.generic_string();
+ if (!IncludeWildcard.empty() && !MatchWildcard(IncludeWildcard, PathString, /*CaseSensitive*/ false))
+ {
+ return false;
+ }
+ if (!ExcludeWildcard.empty() && MatchWildcard(ExcludeWildcard, PathString, /*CaseSensitive*/ false))
+ {
+ return false;
+ }
+ return true;
+ }
+
bool IsFileWithRetry(const std::filesystem::path& Path)
{
std::error_code Ec;
@@ -795,7 +812,7 @@ namespace {
}
uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
- if (ElapsedTimeMs >= 200)
+ if (ElapsedTimeMs >= 200 && !IsQuiet)
{
ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}",
Path,
@@ -2134,11 +2151,14 @@ namespace {
Stopwatch Timer;
auto _ = MakeGuard([&]() {
- ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
- BuildId,
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
+ BuildId,
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
});
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount);
@@ -2163,7 +2183,10 @@ namespace {
CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId);
ValidateStats.BuildPartSize = BuildPart.GetSize();
- ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize()));
+ }
std::vector<IoHash> ChunkAttachments;
for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv])
{
@@ -2904,8 +2927,10 @@ namespace {
}
else
{
- throw std::runtime_error(
- fmt::format("Can not upload requested build blob {} as it was not generated by this upload", RawHash));
+ ZEN_CONSOLE(
+ "Warning: Build blob {} was reported as needed for upload but it was reported as existing at the start of the "
+ "operation. Treating it as a transient inconsistent issue and will attempt to retry finalization",
+ RawHash);
}
}
uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize;
@@ -3768,14 +3793,17 @@ namespace {
}
}
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- LocalContent.Paths.size(),
- NiceBytes(TotalRawSize),
- ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(ChunkingStats.UniqueBytesFound.load()),
- Path,
- NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ LocalContent.Paths.size(),
+ NiceBytes(TotalRawSize),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()),
+ Path,
+ NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
}
const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
@@ -3788,15 +3816,18 @@ namespace {
PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get();
- ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}",
- NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
- CreateBuild ? "PutBuild" : "GetBuild",
- NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
- NiceBytes(PrepBuildResult.PayloadSize),
- IgnoreExistingBlocks ? ""
- : fmt::format(". Found {} blocks in {}",
- PrepBuildResult.KnownBlocks.size(),
- NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}",
+ NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
+ CreateBuild ? "PutBuild" : "GetBuild",
+ NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
+ NiceBytes(PrepBuildResult.PayloadSize),
+ IgnoreExistingBlocks ? ""
+ : fmt::format(". Found {} blocks in {}",
+ PrepBuildResult.KnownBlocks.size(),
+ NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ }
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount);
@@ -3827,7 +3858,10 @@ namespace {
if (IgnoreExistingBlocks)
{
- ZEN_CONSOLE("Ignoring any existing blocks in store");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Ignoring any existing blocks in store");
+ }
NewBlockChunkIndexes = std::move(BlockChunkIndexes);
}
else
@@ -3870,40 +3904,43 @@ namespace {
FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) /
(FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount)
: 0.0;
- ZEN_CONSOLE(
- "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
- " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
- " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
- " Rejected {} ({}) chunks in {} blocks\n"
- " Arranged {} ({}) chunks in {} new blocks\n"
- " Keeping {} ({}) chunks as loose chunks\n"
- " Discovery completed in {}",
- FindBlocksStats.FoundBlockChunkCount,
- FindBlocksStats.FoundBlockCount,
- NiceBytes(FindBlocksStats.FoundBlockByteCount),
- NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
+ " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
+ " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
+ " Rejected {} ({}) chunks in {} blocks\n"
+ " Arranged {} ({}) chunks in {} new blocks\n"
+ " Keeping {} ({}) chunks as loose chunks\n"
+ " Discovery completed in {}",
+ FindBlocksStats.FoundBlockChunkCount,
+ FindBlocksStats.FoundBlockCount,
+ NiceBytes(FindBlocksStats.FoundBlockByteCount),
+ NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
- FindBlocksStats.AcceptedChunkCount,
- NiceBytes(FindBlocksStats.AcceptedRawByteCount),
- FindBlocksStats.AcceptedBlockCount,
- AcceptedByteCountPercent,
+ FindBlocksStats.AcceptedChunkCount,
+ NiceBytes(FindBlocksStats.AcceptedRawByteCount),
+ FindBlocksStats.AcceptedBlockCount,
+ AcceptedByteCountPercent,
- FindBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(FindBlocksStats.AcceptedReduntantByteCount),
- AcceptedReduntantByteCountPercent,
+ FindBlocksStats.AcceptedReduntantChunkCount,
+ NiceBytes(FindBlocksStats.AcceptedReduntantByteCount),
+ AcceptedReduntantByteCountPercent,
- FindBlocksStats.RejectedChunkCount,
- NiceBytes(FindBlocksStats.RejectedByteCount),
- FindBlocksStats.RejectedBlockCount,
+ FindBlocksStats.RejectedChunkCount,
+ NiceBytes(FindBlocksStats.RejectedByteCount),
+ FindBlocksStats.RejectedBlockCount,
- FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
- FindBlocksStats.NewBlocksCount,
+ FindBlocksStats.NewBlocksChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ FindBlocksStats.NewBlocksCount,
- LooseChunksStats.ChunkCount,
- NiceBytes(LooseChunksStats.ChunkByteCount),
+ LooseChunksStats.ChunkCount,
+ NiceBytes(LooseChunksStats.ChunkByteCount),
- NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ }
DiskStatistics DiskStats;
UploadStatistics UploadStats;
@@ -3916,15 +3953,18 @@ namespace {
Stopwatch GenerateBuildBlocksTimer;
auto __ = MakeGuard([&]() {
uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs();
- ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
- UploadStats.BlockCount.load(),
- NiceBytes(UploadStats.BlocksBytes.load()),
- NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
- NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+ NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
+ NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
+ GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ }
});
GenerateBuildBlocks(Path,
LocalContent,
@@ -3943,9 +3983,12 @@ namespace {
CbObjectWriter PartManifestWriter;
Stopwatch ManifestGenerationTimer;
auto __ = MakeGuard([&]() {
- ZEN_CONSOLE("Generated build part manifest in {} ({})",
- NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifestWriter.GetSaveSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Generated build part manifest in {} ({})",
+ NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifestWriter.GetSaveSize()));
+ }
});
PartManifestWriter.AddObject("chunker"sv, ChunkerParameters);
@@ -4094,10 +4137,13 @@ namespace {
Stopwatch PutBuildPartResultTimer;
std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest);
- ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.",
- NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifest.GetSize()),
- PutBuildPartResult.second.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.",
+ NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifest.GetSize()),
+ PutBuildPartResult.second.size());
+ }
IoHash PartHash = PutBuildPartResult.first;
auto UploadAttachments = [&Storage,
@@ -4120,24 +4166,27 @@ namespace {
Stopwatch TempUploadTimer;
auto __ = MakeGuard([&]() {
- uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
- ZEN_CONSOLE(
- "Uploaded {} ({}) blocks. "
- "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
- "Transferred {} ({}bits/s) in {}",
- TempUploadStats.BlockCount.load(),
- NiceBytes(TempUploadStats.BlocksBytes),
-
- TempLooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(
- GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)),
- TempUploadStats.ChunkCount.load(),
- NiceBytes(TempUploadStats.ChunksBytes),
-
- NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
- NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ if (!IsQuiet)
+ {
+ uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
+ ZEN_CONSOLE(
+ "Uploaded {} ({}) blocks. "
+ "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
+ "Transferred {} ({}bits/s) in {}",
+ TempUploadStats.BlockCount.load(),
+ NiceBytes(TempUploadStats.BlocksBytes),
+
+ TempLooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
+ TempLooseChunksStats.ChunkByteCount)),
+ TempUploadStats.ChunkCount.load(),
+ NiceBytes(TempUploadStats.ChunksBytes),
+
+ NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
+ NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ }
});
UploadPartBlobs(Storage,
BuildId,
@@ -4191,9 +4240,12 @@ namespace {
{
Stopwatch FinalizeBuildPartTimer;
std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash);
- ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.",
- NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
- Needs.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.",
+ NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
+ Needs.size());
+ }
if (Needs.empty())
{
break;
@@ -4206,7 +4258,10 @@ namespace {
{
Stopwatch FinalizeBuildTimer;
Storage.BuildStorage->FinalizeBuild(BuildId);
- ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
+ }
}
if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag)
@@ -4254,7 +4309,10 @@ namespace {
{
uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs();
UploadStats.ElapsedWallTimeUS += ElapsedUS;
- ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000));
+ }
}
}
@@ -4450,55 +4508,59 @@ namespace {
NiceTimeSpanMs(ValidateStats.ElapsedWallTimeUS / 1000));
}
- ZEN_CONSOLE(
- "Uploaded part {} ('{}') to build {}, {}\n"
- " Scanned files: {:>8} ({}), {}B/sec, {}\n"
- " New data: {:>8} ({}) {:.1f}%\n"
- " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
- " Blocks: {:>8} ({})\n"
- " Chunks: {:>8} ({}){}"
- "{}",
- BuildPartId,
- BuildPartName,
- BuildId,
- NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()),
-
- LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)),
- NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000),
-
- FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount,
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
- DeltaByteCountPercent,
-
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
- NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
-
- LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(LooseChunksStats.ChunkByteCount),
- NiceBytes(LooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)),
- NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
-
- UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(),
- NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
- NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000),
-
- UploadStats.BlockCount.load(),
- NiceBytes(UploadStats.BlocksBytes.load()),
-
- UploadStats.ChunkCount.load(),
- NiceBytes(UploadStats.ChunksBytes.load()),
- MultipartAttachmentStats,
-
- ValidateInfo);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Uploaded part {} ('{}') to build {}, {}\n"
+ " Scanned files: {:>8} ({}), {}B/sec, {}\n"
+ " New data: {:>8} ({}) {:.1f}%\n"
+ " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
+ " Blocks: {:>8} ({})\n"
+ " Chunks: {:>8} ({}){}"
+ "{}",
+ BuildPartId,
+ BuildPartName,
+ BuildId,
+ NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()),
+
+ LocalFolderScanStats.FoundFileCount.load(),
+ NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)),
+ NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000),
+
+ FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
+ DeltaByteCountPercent,
+
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceNum(
+ GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
+
+ LooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(LooseChunksStats.ChunkByteCount),
+ NiceBytes(LooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)),
+ NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
+
+ UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(),
+ NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
+ NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000),
+
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+
+ UploadStats.ChunkCount.load(),
+ NiceBytes(UploadStats.ChunksBytes.load()),
+ MultipartAttachmentStats,
+
+ ValidateInfo);
+ }
Storage.BuildStorage->PutBuildPartStats(
BuildId,
@@ -5726,7 +5788,10 @@ namespace {
const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
- ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ }
const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath);
@@ -6073,7 +6138,7 @@ namespace {
Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
ZEN_UNUSED(PendingWork);
- std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavanging",
+ std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavenging",
PathsScavenged.load(),
ScavengePathCount,
PathsFound.load(),
@@ -6338,37 +6403,39 @@ namespace {
}
CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
}
-
- if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
- CachedSequenceHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
- CachedChunkHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheChunkByteCount),
- CachedBlocksFound.size(),
- NiceBytes(CacheMappingStats.CacheBlocksByteCount),
- NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
- }
+ if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
+ {
+ ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
+ CachedSequenceHashesFound.size(),
+ NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
+ CachedChunkHashesFound.size(),
+ NiceBytes(CacheMappingStats.CacheChunkByteCount),
+ CachedBlocksFound.size(),
+ NiceBytes(CacheMappingStats.CacheBlocksByteCount),
+ NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
+ }
- if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
- {
- ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
- LocalPathIndexesMatchingSequenceIndexes.size(),
- NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
- CacheMappingStats.LocalChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
- }
- if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
- {
- ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
- ScavengedPathsCount,
- CacheMappingStats.ScavengedPathsMatchingSequencesCount,
- NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
- CacheMappingStats.ScavengedChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
+ {
+ ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ LocalPathIndexesMatchingSequenceIndexes.size(),
+ NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
+ CacheMappingStats.LocalChunkMatchingRemoteCount,
+ NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
+ }
+ if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
+ {
+ ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ ScavengedPathsCount,
+ CacheMappingStats.ScavengedPathsMatchingSequencesCount,
+ NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
+ CacheMappingStats.ScavengedChunkMatchingRemoteCount,
+ NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ }
}
uint64_t BytesToWrite = 0;
@@ -6756,7 +6823,7 @@ namespace {
}
}
ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs();
- if (!ExistsResult.ExistingBlobs.empty())
+ if (!ExistsResult.ExistingBlobs.empty() && !IsQuiet)
{
ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}",
ExistsResult.ExistingBlobs.size(),
@@ -7918,10 +7985,13 @@ namespace {
const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex];
ZEN_ASSERT(!IncompletePath.empty());
const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
- ZEN_CONSOLE("{}: Max count {}, Current count {}",
- IncompletePath,
- ExpectedSequenceCount,
- SequenceIndexChunksLeftToWriteCounter.load());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}: Max count {}, Current count {}",
+ IncompletePath,
+ ExpectedSequenceCount,
+ SequenceIndexChunksLeftToWriteCounter.load());
+ }
ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount);
}
}
@@ -7930,14 +8000,17 @@ namespace {
const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() +
+DownloadStats.DownloadedPartialBlockByteCount.load();
- ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
- NiceBytes(DownloadedBytes),
- NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
- NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
- NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
+ NiceBytes(DownloadedBytes),
+ NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
+ NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceBytes(DiskStats.WriteByteCount.load()),
+ NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
+ NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
+ }
WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
@@ -8519,12 +8592,15 @@ namespace {
{
Stopwatch GetBuildTimer;
CbObject BuildObject = Storage.GetBuild(BuildId);
- ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
- NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
- BuildObject["name"sv].AsString(),
- NiceBytes(BuildObject.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
+ NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
+ BuildObject["name"sv].AsString(),
+ NiceBytes(BuildObject.GetSize()));
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
+ ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
+ }
CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
if (!PartsObject)
@@ -8600,6 +8676,8 @@ namespace {
ChunkedFolderContent GetRemoteContent(StorageInstance& Storage,
const Oid& BuildId,
const std::vector<std::pair<Oid, std::string>>& BuildParts,
+ std::string_view IncludeWildcard,
+ std::string_view ExcludeWildcard,
std::unique_ptr<ChunkingController>& OutChunkController,
std::vector<ChunkedFolderContent>& OutPartContents,
std::vector<ChunkBlockDescription>& OutBlockDescriptions,
@@ -8611,12 +8689,15 @@ namespace {
const Oid BuildPartId = BuildParts[0].first;
const std::string_view BuildPartName = BuildParts[0].second;
CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(BuildPartManifest.GetSize()));
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ }
{
CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView();
@@ -8629,6 +8710,8 @@ namespace {
const Oid& BuildId,
const Oid& BuildPartId,
CbObject BuildPartManifest,
+ std::string_view IncludeWildcard,
+ std::string_view ExcludeWildcard,
ChunkedFolderContent& OutRemoteContent,
std::vector<ChunkBlockDescription>& OutBlockDescriptions,
std::vector<IoHash>& OutLooseChunkHashes) {
@@ -8656,7 +8739,7 @@ namespace {
std::vector<ChunkBlockDescription> UnorderedList;
tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup;
- if (Storage.BuildCacheStorage)
+ if (Storage.BuildCacheStorage && !BlockRawHashes.empty())
{
std::vector<CbObject> CacheBlockMetadatas = Storage.BuildCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes);
UnorderedList.reserve(CacheBlockMetadatas.size());
@@ -8743,10 +8826,13 @@ namespace {
}
}
- ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
- BuildPartId,
- NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
- OutBlockDescriptions.size());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
+ BuildPartId,
+ NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
+ OutBlockDescriptions.size());
+ }
}
if (OutBlockDescriptions.size() != BlockRawHashes.size())
@@ -8830,6 +8916,36 @@ namespace {
OutRemoteContent.ChunkedContent.ChunkHashes,
OutRemoteContent.ChunkedContent.ChunkRawSizes,
OutRemoteContent.ChunkedContent.ChunkOrders);
+
+ {
+ std::vector<std::filesystem::path> DeletedPaths;
+ for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
+ {
+ if (!IncludePath(IncludeWildcard, ExcludeWildcard, RemotePath))
+ {
+ DeletedPaths.push_back(RemotePath);
+ }
+ }
+
+ if (!DeletedPaths.empty())
+ {
+ OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths);
+
+ tsl::robin_set<IoHash, IoHash::Hasher> UsedLooseChunkHashes;
+ UsedLooseChunkHashes.insert(OutRemoteContent.RawHashes.begin(), OutRemoteContent.RawHashes.end());
+ for (auto It = OutLooseChunkHashes.begin(); It != OutLooseChunkHashes.end();)
+ {
+ if (!UsedLooseChunkHashes.contains(*It))
+ {
+ It = OutLooseChunkHashes.erase(It);
+ }
+ else
+ {
+ It++;
+ }
+ }
+ }
+ }
};
OutPartContents.resize(1);
@@ -8837,6 +8953,8 @@ namespace {
BuildId,
BuildPartId,
BuildPartManifest,
+ IncludeWildcard,
+ ExcludeWildcard,
OutPartContents[0],
OutBlockDescriptions,
OutLooseChunkHashes);
@@ -8851,11 +8969,14 @@ namespace {
const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
Stopwatch GetOverlayBuildPartTimer;
CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- OverlayBuildPartId,
- OverlayBuildPartName,
- NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(OverlayBuildPartManifest.GetSize()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ OverlayBuildPartId,
+ OverlayBuildPartName,
+ NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(OverlayBuildPartManifest.GetSize()));
+ }
ChunkedFolderContent OverlayPartContent;
std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions;
@@ -8865,6 +8986,8 @@ namespace {
BuildId,
OverlayBuildPartId,
OverlayBuildPartManifest,
+ IncludeWildcard,
+ ExcludeWildcard,
OverlayPartContent,
OverlayPartBlockDescriptions,
OverlayPartLooseChunkHashes);
@@ -8919,6 +9042,8 @@ namespace {
const std::filesystem::path& StateFilePath,
ChunkingController& ChunkController,
std::span<const std::filesystem::path> ReferencePaths,
+ std::string_view IncludeWildcard,
+ std::string_view ExcludeWildcard,
FolderContent& OutLocalFolderContent)
{
FolderContent LocalFolderState;
@@ -8926,13 +9051,13 @@ namespace {
Stopwatch ReadStateTimer;
const bool HasLocalState = IsFile(StateFilePath) && ReadStateFile(StateFilePath, LocalFolderState, LocalContent);
- if (HasLocalState)
+ if (HasLocalState && !IsQuiet)
{
ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
}
{
- const uint32_t LocalPathCount = gsl::narrow<uint32_t>(ReferencePaths.size());
- const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size());
+ const uint32_t LocalPathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size());
+ const uint32_t RemotePathCount = gsl::narrow<uint32_t>(ReferencePaths.size());
std::vector<std::filesystem::path> PathsToCheck;
PathsToCheck.reserve(LocalPathCount + RemotePathCount);
@@ -8942,34 +9067,44 @@ namespace {
for (const std::filesystem::path& LocalPath : LocalFolderState.Paths)
{
- FileSet.insert(LocalPath.generic_string());
- PathsToCheck.push_back(LocalPath);
+ if (IncludePath(IncludeWildcard, ExcludeWildcard, LocalPath))
+ {
+ FileSet.insert(LocalPath.generic_string());
+ PathsToCheck.push_back(LocalPath);
+ }
}
for (const std::filesystem::path& RemotePath : ReferencePaths)
{
- if (FileSet.insert(RemotePath.generic_string()).second)
+ if (IncludePath(IncludeWildcard, ExcludeWildcard, RemotePath))
{
- PathsToCheck.push_back(RemotePath);
+ if (FileSet.insert(RemotePath.generic_string()).second)
+ {
+ PathsToCheck.push_back(RemotePath);
+ }
}
}
- ProgressBar ProgressBar(ProgressMode, "Check Files");
- OutLocalFolderContent = GetValidFolderContent(
- LocalFolderScanStats,
- Path,
- PathsToCheck,
- [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
- std::string Details =
- fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
- ProgressBar.UpdateState({.Task = "Checking files ",
- .Details = Details,
- .TotalCount = PathCount,
- .RemainingCount = PathCount - CompletedPathCount,
- .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
- false);
- });
- ProgressBar.Finish();
+ {
+ ProgressBar ProgressBar(ProgressMode, "Check Files");
+ OutLocalFolderContent =
+ GetValidFolderContent(LocalFolderScanStats,
+ Path,
+ PathsToCheck,
+ [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
+ std::string Details = fmt::format("{}/{} checked, {} found",
+ CompletedPathCount,
+ PathCount,
+ LocalFolderScanStats.FoundFileCount.load());
+ ProgressBar.UpdateState({.Task = "Checking files ",
+ .Details = Details,
+ .TotalCount = PathCount,
+ .RemainingCount = PathCount - CompletedPathCount,
+ .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
+ false);
+ });
+ ProgressBar.Finish();
+ }
if (AbortFlag)
{
return {};
@@ -8982,7 +9117,7 @@ namespace {
{
if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent))
{
- const size_t LocaStatePathCount = LocalFolderState.Paths.size();
+ const size_t LocalStatePathCount = LocalFolderState.Paths.size();
std::vector<std::filesystem::path> DeletedPaths;
FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths);
if (!DeletedPaths.empty())
@@ -8990,10 +9125,13 @@ namespace {
LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
}
- ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
- DeletedPaths.size(),
- UpdatedContent.Paths.size(),
- LocaStatePathCount);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
+ DeletedPaths.size(),
+ UpdatedContent.Paths.size(),
+ LocalStatePathCount);
+ }
if (UpdatedContent.Paths.size() > 0)
{
uint64_t ByteCountToScan = 0;
@@ -9062,6 +9200,85 @@ namespace {
LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
}
}
+
+ // Check files that are present in current folder state that matches the ReferencePaths but not known to the local state
+ {
+ FolderContent UpdatedContent;
+
+ tsl::robin_set<std::string> LocalPathIndexLookup;
+ for (const std::filesystem::path& LocalPath : LocalContent.Paths)
+ {
+ LocalPathIndexLookup.insert(LocalPath.generic_string());
+ }
+
+ tsl::robin_set<std::string> RemotePathIndexLookup;
+ for (const std::filesystem::path& RemotePath : ReferencePaths)
+ {
+ RemotePathIndexLookup.insert(RemotePath.generic_string());
+ }
+
+ for (uint32_t LocalFolderPathIndex = 0; LocalFolderPathIndex < OutLocalFolderContent.Paths.size(); LocalFolderPathIndex++)
+ {
+ const std::filesystem::path& LocalFolderPath = OutLocalFolderContent.Paths[LocalFolderPathIndex];
+ const std::string GenericLocalFolderPath = LocalFolderPath.generic_string();
+ if (RemotePathIndexLookup.contains(GenericLocalFolderPath))
+ {
+ if (!LocalPathIndexLookup.contains(GenericLocalFolderPath))
+ {
+ UpdatedContent.Paths.push_back(LocalFolderPath);
+ UpdatedContent.RawSizes.push_back(OutLocalFolderContent.RawSizes[LocalFolderPathIndex]);
+ UpdatedContent.Attributes.push_back(OutLocalFolderContent.Attributes[LocalFolderPathIndex]);
+ UpdatedContent.ModificationTicks.push_back(OutLocalFolderContent.ModificationTicks[LocalFolderPathIndex]);
+ }
+ }
+ }
+
+ if (UpdatedContent.Paths.size() > 0)
+ {
+ uint64_t ByteCountToScan = 0;
+ for (const uint64_t RawSize : UpdatedContent.RawSizes)
+ {
+ ByteCountToScan += RawSize;
+ }
+ ProgressBar ProgressBar(ProgressMode, "Scan Files");
+ FilteredRate FilteredBytesHashed;
+ FilteredBytesHashed.Start();
+ ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
+ ChunkingStats,
+ GetIOWorkerPool(),
+ Path,
+ UpdatedContent,
+ ChunkController,
+ GetUpdateDelayMS(ProgressMode),
+ [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
+ FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
+ std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
+ ChunkingStats.FilesProcessed.load(),
+ UpdatedContent.Paths.size(),
+ NiceBytes(ChunkingStats.BytesHashed.load()),
+ NiceBytes(ByteCountToScan),
+ NiceNum(FilteredBytesHashed.GetCurrent()),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()));
+ ProgressBar.UpdateState({.Task = "Scanning files ",
+ .Details = Details,
+ .TotalCount = ByteCountToScan,
+ .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load(),
+ .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ },
+ AbortFlag,
+ PauseFlag);
+
+ FilteredBytesHashed.Stop();
+ ProgressBar.Finish();
+ if (!AbortFlag)
+ {
+ LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}});
+ }
+ }
+ }
+
ScanContent = false;
}
@@ -9145,20 +9362,25 @@ namespace {
ZenStateFilePath(Path / ZenFolderName),
ChunkController,
Content.Paths,
+ {},
+ {},
_);
const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
const uint64_t ChunkedRawSize =
std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- Result.Paths.size(),
- NiceBytes(TotalRawSize),
- Result.ChunkedContent.ChunkHashes.size(),
- NiceBytes(ChunkedRawSize),
- Path,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ Result.Paths.size(),
+ NiceBytes(TotalRawSize),
+ Result.ChunkedContent.ChunkHashes.size(),
+ NiceBytes(ChunkedRawSize),
+ Path,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
return Result;
};
@@ -9174,7 +9396,9 @@ namespace {
bool WipeTargetFolder,
bool PostDownloadVerify,
bool PrimeCacheOnly,
- bool EnableScavenging)
+ bool EnableScavenging,
+ std::string_view IncludeWildcard,
+ std::string_view ExcludeWildcard)
{
ZEN_TRACE_CPU("DownloadFolder");
@@ -9220,8 +9444,15 @@ namespace {
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount);
- ChunkedFolderContent RemoteContent =
- GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes);
+ ChunkedFolderContent RemoteContent = GetRemoteContent(Storage,
+ BuildId,
+ AllBuildParts,
+ IncludeWildcard,
+ ExcludeWildcard,
+ ChunkController,
+ PartContents,
+ BlockDescriptions,
+ LooseChunkHashes);
const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
GetFolderContentStatistics LocalFolderScanStats;
@@ -9232,22 +9463,21 @@ namespace {
{
if (IsDir(Path))
{
- if (!WipeTargetFolder)
+ if (!ChunkController && !IsQuiet)
{
- if (!ChunkController)
- {
- ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default");
- ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
- }
-
- LocalContent = GetLocalContent(LocalFolderScanStats,
- ChunkingStats,
- Path,
- ZenStateFilePath(ZenFolderPath),
- *ChunkController,
- RemoteContent.Paths,
- LocalFolderContent);
+ ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default");
+ ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
}
+
+ LocalContent = GetLocalContent(LocalFolderScanStats,
+ ChunkingStats,
+ Path,
+ ZenStateFilePath(ZenFolderPath),
+ *ChunkController,
+ RemoteContent.Paths,
+ IncludeWildcard,
+ ExcludeWildcard,
+ LocalFolderContent);
}
else
{
@@ -9258,7 +9488,6 @@ namespace {
{
return;
}
-
auto CompareContent = [](const ChunkedFolderContent& Lhs, const ChunkedFolderContent& Rhs) {
tsl::robin_map<std::string, size_t> RhsPathToIndex;
const size_t RhsPathCount = Rhs.Paths.size();
@@ -9301,16 +9530,22 @@ namespace {
return true;
};
- if (CompareContent(RemoteContent, LocalContent))
+ if (CompareContent(RemoteContent, LocalContent) && !WipeTargetFolder)
{
- ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
- NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
+ NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+ }
Stopwatch WriteStateTimer;
CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path);
CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
}
@@ -9324,7 +9559,10 @@ namespace {
uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));
- ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
+ }
FolderContent LocalFolderState;
DiskStatistics DiskStats;
@@ -9371,7 +9609,10 @@ namespace {
CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
@@ -9388,30 +9629,33 @@ namespace {
DownloadStats.DownloadedPartialBlockByteCount.load();
const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();
- ZEN_CONSOLE(
- "Downloaded build {}, parts:{} in {}\n"
- " Download: {} ({}) {}bits/s\n"
- " Write: {} ({}) {}B/s\n"
- " Clean: {}\n"
- " Finalize: {}\n"
- " Verify: {}",
- BuildId,
- BuildPartString.ToView(),
- NiceTimeSpanMs(DownloadTimeMs),
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Downloaded build {}, parts:{} in {}\n"
+ " Download: {} ({}) {}bits/s\n"
+ " Write: {} ({}) {}B/s\n"
+ " Clean: {}\n"
+ " Finalize: {}\n"
+ " Verify: {}",
+ BuildId,
+ BuildPartString.ToView(),
+ NiceTimeSpanMs(DownloadTimeMs),
- DownloadCount,
- NiceBytes(DownloadByteCount),
- NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
+ DownloadCount,
+ NiceBytes(DownloadByteCount),
+ NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
- DiskStats.WriteCount.load(),
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())),
+ DiskStats.WriteCount.load(),
+ NiceBytes(DiskStats.WriteByteCount.load()),
+ NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())),
- NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+ NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+ }
}
}
if (PrimeCacheOnly)
@@ -9419,13 +9663,16 @@ namespace {
if (Storage.BuildCacheStorage)
{
Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) {
- if (Remaining == 0)
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Build cache upload complete");
- }
- else
- {
- ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
+ if (Remaining == 0)
+ {
+ ZEN_CONSOLE("Build cache upload complete");
+ }
+ else
+ {
+ ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
+ }
}
return !AbortFlag;
});
@@ -9440,6 +9687,87 @@ namespace {
}
}
+ void ListBuild(StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ std::string_view IncludeWildcard,
+ std::string_view ExcludeWildcard)
+ {
+ std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+
+ Stopwatch GetBuildPartTimer;
+
+ for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
+ {
+ const Oid BuildPartId = AllBuildParts[BuildPartIndex].first;
+ const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second;
+ CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
+
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}Part: {} ('{}'):\n",
+ BuildPartIndex > 0 ? "\n" : "",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ }
+
+ std::vector<std::filesystem::path> Paths;
+ std::vector<IoHash> RawHashes;
+ std::vector<uint64_t> RawSizes;
+ std::vector<uint32_t> Attributes;
+
+ SourcePlatform Platform;
+ std::vector<IoHash> SequenceRawHashes;
+ std::vector<uint32_t> ChunkCounts;
+ std::vector<uint32_t> AbsoluteChunkOrders;
+ std::vector<IoHash> LooseChunkHashes;
+ std::vector<uint64_t> LooseChunkRawSizes;
+ std::vector<IoHash> BlockRawHashes;
+
+ ReadBuildContentFromCompactBinary(BuildPartManifest,
+ Platform,
+ Paths,
+ RawHashes,
+ RawSizes,
+ Attributes,
+ SequenceRawHashes,
+ ChunkCounts,
+ AbsoluteChunkOrders,
+ LooseChunkHashes,
+ LooseChunkRawSizes,
+ BlockRawHashes);
+
+ std::vector<size_t> Order(Paths.size());
+ std::iota(Order.begin(), Order.end(), 0);
+
+ std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
+ const std::filesystem::path& LhsPath = Paths[Lhs];
+ const std::filesystem::path& RhsPath = Paths[Rhs];
+ return LhsPath < RhsPath;
+ });
+
+ for (size_t Index : Order)
+ {
+ const std::filesystem::path& Path = Paths[Index];
+ if (IncludePath(IncludeWildcard, ExcludeWildcard, Path))
+ {
+ const IoHash& RawHash = RawHashes[Index];
+ const uint64_t RawSize = RawSizes[Index];
+ const uint32_t Attribute = Attributes[Index];
+ ZEN_UNUSED(Attribute);
+
+ ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
+ }
+ }
+ }
+ }
+
void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked)
{
ZEN_TRACE_CPU("DiffFolders");
@@ -9772,6 +10100,7 @@ BuildsCommand::BuildsCommand()
cxxopts::value(m_LogProgress),
"<logprogress>");
Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
+ Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
};
auto AddWorkerOptions = [this](cxxopts::Options& Ops) {
@@ -9791,6 +10120,23 @@ BuildsCommand::BuildsCommand()
cxxopts::value(m_ZenFolderPath),
"<boostworkers>");
};
+
+ auto AddWildcardOptions = [this](cxxopts::Options& Ops) {
+ Ops.add_option("",
+ "",
+ "wildcard",
+ "Windows style wildcard (using * and ?) to match file paths to include",
+ cxxopts::value(m_IncludeWildcard),
+ "<wildcard>");
+
+ Ops.add_option("",
+ "",
+ "exclude-wildcard",
+ "Windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter",
+ cxxopts::value(m_ExcludeWildcard),
+ "<excludewildcard>");
+ };
+
m_Options.add_option("",
"v",
"verb",
@@ -9925,6 +10271,8 @@ BuildsCommand::BuildsCommand()
AddCacheOptions(m_DownloadOptions);
AddZenFolderOptions(m_DownloadOptions);
AddWorkerOptions(m_DownloadOptions);
+ AddWildcardOptions(m_DownloadOptions);
+
m_DownloadOptions.add_option("cache",
"",
"cache-prime-only",
@@ -9975,6 +10323,37 @@ BuildsCommand::BuildsCommand()
m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"});
m_DownloadOptions.positional_help("local-path build-id build-part-name");
+ // ls
+ AddSystemOptions(m_LsOptions);
+ AddCloudOptions(m_LsOptions);
+ AddFileOptions(m_LsOptions);
+ AddOutputOptions(m_LsOptions);
+ AddCacheOptions(m_LsOptions);
+ AddZenFolderOptions(m_LsOptions);
+ AddWorkerOptions(m_LsOptions);
+ AddWildcardOptions(m_LsOptions);
+
+ m_LsOptions.add_options()("h,help", "Print help");
+ m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ m_LsOptions.add_option(
+ "",
+ "",
+ "build-part-id",
+ "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
+ cxxopts::value(m_BuildPartIds),
+ "<id>");
+ m_LsOptions.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
+ "all parts will be downloaded",
+ cxxopts::value(m_BuildPartNames),
+ "<name>");
+
+ m_LsOptions.parse_positional({"build-id", "wildcard"});
+ m_LsOptions.positional_help("build-id wildcard");
+
+ // diff
AddOutputOptions(m_DiffOptions);
AddWorkerOptions(m_DiffOptions);
m_DiffOptions.add_options()("h,help", "Print help");
@@ -9989,6 +10368,7 @@ BuildsCommand::BuildsCommand()
m_DiffOptions.parse_positional({"local-path", "compare-path"});
m_DiffOptions.positional_help("local-path compare-path");
+ // test
AddSystemOptions(m_TestOptions);
AddCloudOptions(m_TestOptions);
AddFileOptions(m_TestOptions);
@@ -10018,6 +10398,7 @@ BuildsCommand::BuildsCommand()
m_TestOptions.parse_positional({"local-path"});
m_TestOptions.positional_help("local-path");
+ // fetch-blob
AddSystemOptions(m_FetchBlobOptions);
AddCloudOptions(m_FetchBlobOptions);
AddFileOptions(m_FetchBlobOptions);
@@ -10045,6 +10426,7 @@ BuildsCommand::BuildsCommand()
m_AbortOptions.parse_positional({"process-id"});
m_AbortOptions.positional_help("process-id");
+ // validate-part
AddSystemOptions(m_ValidateBuildPartOptions);
AddCloudOptions(m_ValidateBuildPartOptions);
AddFileOptions(m_ValidateBuildPartOptions);
@@ -10068,6 +10450,7 @@ BuildsCommand::BuildsCommand()
m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"});
m_ValidateBuildPartOptions.positional_help("build-id build-part-id");
+ // multi-test-download
AddSystemOptions(m_MultiTestDownloadOptions);
AddCloudOptions(m_MultiTestDownloadOptions);
AddFileOptions(m_MultiTestDownloadOptions);
@@ -10191,13 +10574,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_EncryptionKey.empty())
{
m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456";
- ZEN_CONSOLE("Warning: Using default encryption key");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Warning: Using default encryption key");
+ }
}
if (m_EncryptionIV.empty())
{
m_EncryptionIV = "0123456789abcdef";
- ZEN_CONSOLE("Warning: Using default encryption initialization vector");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Warning: Using default encryption initialization vector");
+ }
}
AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth",
@@ -10299,7 +10688,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty())
{
const std::string& CloudHost = m_OverrideHost.empty() ? m_Host : m_OverrideHost;
- ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost);
+ ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost, IsQuiet);
}
if (!ClientSettings.AccessTokenProvider)
@@ -10310,15 +10699,36 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
};
auto ParseOutputOptions = [&]() {
+ if (m_Verbose && m_Quiet)
+ {
+ throw std::runtime_error("--verbose option is not compatible with --quiet option");
+ }
IsVerbose = m_Verbose;
+ IsQuiet = m_Quiet;
if (m_LogProgress)
{
+ if (IsQuiet)
+ {
+ throw std::runtime_error("--quiet option is not compatible with --log-progress option");
+ }
ProgressMode = ProgressBar::Mode::Log;
}
- else if (m_Verbose || m_PlainProgress)
+ if (m_PlainProgress)
{
+ if (IsQuiet)
+ {
+ throw std::runtime_error("--quiet option is not compatible with --plain-progress option");
+ }
ProgressMode = ProgressBar::Mode::Plain;
}
+ else if (m_Verbose)
+ {
+ ProgressMode = ProgressBar::Mode::Plain;
+ }
+ else if (IsQuiet)
+ {
+ ProgressMode = ProgressBar::Mode::Quiet;
+ }
else
{
ProgressMode = ProgressBar::Mode::Pretty;
@@ -10375,9 +10785,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON));
if (!ServerInfoResponse.IsSuccess())
{
- throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'",
- m_Host,
- ServerInfoResponse.ErrorMessage("")));
+ ServerInfoResponse.ThrowError(fmt::format("Failed to get list of servers from discovery url '{}'", m_Host));
}
std::string_view JsonResponse = ServerInfoResponse.AsText();
@@ -10487,7 +10895,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
});
}
- if (m_ZenCacheHost.empty())
+ if (m_ZenCacheHost.empty() && !IsQuiet)
{
ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
}
@@ -10576,10 +10984,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName;
}
- ZEN_CONSOLE("Remote: {}", StorageDescription);
- if (!Result.CacheName.empty())
+ if (!IsQuiet)
{
- ZEN_CONSOLE("Cache : {}", CacheDescription);
+ ZEN_CONSOLE("Remote: {}", StorageDescription);
+ if (!Result.CacheName.empty())
+ {
+ ZEN_CONSOLE("Cache : {}", CacheDescription);
+ }
}
return Result;
};
@@ -10592,6 +11003,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
MakeSafeAbsolutePathÍnPlace(m_Path);
};
+ auto ParseFileFilters = [&]() {
+ for (auto It = begin(m_IncludeWildcard); It != end(m_IncludeWildcard); It++)
+ {
+ if (*It == '\\')
+ {
+ *It = '/';
+ }
+ }
+
+ for (auto It = begin(m_ExcludeWildcard); It != end(m_ExcludeWildcard); It++)
+ {
+ if (*It == '\\')
+ {
+ *It = '/';
+ }
+ }
+ };
+
auto ParseDiffPath = [&]() {
if (m_DiffPath.empty())
{
@@ -10740,10 +11169,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!m_ListResultPath.empty())
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
}
BuildStorage::Statistics StorageStats;
@@ -10768,7 +11200,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
/*RequireBucket*/ false);
CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive);
- ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None);
+ ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
if (m_ListResultPath.empty())
{
ExtendableStringBuilder<1024> SB;
@@ -10801,10 +11233,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!m_ListResultPath.empty())
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
}
CbObject QueryObject;
if (m_ListQueryPath.empty())
@@ -10859,7 +11294,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
/*RequireBucket*/ false);
CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject);
- ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None);
+ ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
if (m_ListResultPath.empty())
{
ExtendableStringBuilder<1024> SB;
@@ -10886,7 +11321,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_UploadOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
@@ -10955,34 +11396,44 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (false)
{
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.StorageName,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "{}:\n"
+ "Read: {}\n"
+ "Write: {}\n"
+ "Requests: {}\n"
+ "Avg Request Time: {}\n"
+ "Avg I/O Time: {}",
+ Storage.StorageName,
+ NiceBytes(StorageStats.TotalBytesRead.load()),
+ NiceBytes(StorageStats.TotalBytesWritten.load()),
+ StorageStats.TotalRequestCount.load(),
+ StorageStats.TotalExecutionTimeUs.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0,
+ StorageStats.TotalRequestCount.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0);
+ }
}
return AbortFlag ? 11 : 0;
}
if (SubOption == &m_DownloadOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
ParsePath();
+ ParseFileFilters();
if (m_ZenFolderPath.empty())
{
@@ -11020,6 +11471,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::vector<Oid> BuildPartIds = ParseBuildPartIds();
std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+ // TODO: Add file filters
DownloadFolder(Storage,
BuildId,
BuildPartIds,
@@ -11032,10 +11484,52 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_Clean,
m_PostDownloadVerify,
m_PrimeCacheOnly,
- m_EnableScavenging);
+ m_EnableScavenging,
+ m_IncludeWildcard,
+ m_ExcludeWildcard);
+
+ return AbortFlag ? 11 : 0;
+ }
+
+ if (SubOption == &m_LsOptions)
+ {
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
+
+ ZenState InstanceState;
+
+ ParseFileFilters();
+
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
+
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
+
+ const Oid BuildId = ParseBuildId();
+
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds();
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+
+ ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard);
return AbortFlag ? 11 : 0;
}
+
if (SubOption == &m_DiffOptions)
{
ParsePath();
@@ -11047,7 +11541,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_FetchBlobOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
@@ -11084,16 +11584,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
return 11;
}
- ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
- BlobHash,
- CompressedSize,
- DecompressedSize);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
+ BlobHash,
+ CompressedSize,
+ DecompressedSize);
+ }
return 0;
}
if (SubOption == &m_ValidateBuildPartOptions)
{
- ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
+ }
ZenState InstanceState;
@@ -11181,15 +11690,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildIdString == m_BuildIds.front(),
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Download cancelled");
return 11;
}
- ZEN_CONSOLE("\n");
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("\n");
+ }
+ }
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
- ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
return 0;
}
@@ -11351,7 +11868,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
true,
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Download failed.");
@@ -11375,7 +11894,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
false,
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Re-download failed. (identical target)");
@@ -11493,7 +12014,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
false,
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Re-download failed. (scrambled target)");
@@ -11545,7 +12068,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
false,
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Re-download failed.");
@@ -11565,7 +12090,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
false,
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Re-download failed.");
@@ -11585,7 +12112,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
false,
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Re-download failed.");
@@ -11605,7 +12134,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
false,
true,
false,
- m_EnableScavenging);
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
if (AbortFlag)
{
ZEN_CONSOLE("Re-download failed.");
@@ -11633,6 +12164,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 3;
}
}
+ catch (const HttpClientError& HttpEx)
+ {
+ ZEN_CONSOLE("Operation failed: {}", HttpEx.what());
+ return HttpEx.m_Error != 0 ? HttpEx.m_Error : (int)HttpEx.m_ResponseCode;
+ }
catch (const std::exception& Ex)
{
ZEN_ERROR("{}", Ex.what());
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index b3466c0d3..d057d24ac 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -32,6 +32,7 @@ private:
bool m_Verbose = false;
bool m_BoostWorkerThreads = false;
bool m_UseSparseFiles = true;
+ bool m_Quiet = false;
std::filesystem::path m_ZenFolderPath;
@@ -107,6 +108,10 @@ private:
bool m_PostDownloadVerify = false;
bool m_EnableScavenging = true;
+ cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"};
+ std::string m_IncludeWildcard;
+ std::string m_ExcludeWildcard;
+
cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"};
std::filesystem::path m_DiffPath;
bool m_OnlyChunked = false;
@@ -127,7 +132,7 @@ private:
cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
std::vector<std::string> m_BuildIds;
- cxxopts::Options* m_SubCommands[12] = {&m_ListNamespacesOptions,
+ cxxopts::Options* m_SubCommands[13] = {&m_ListNamespacesOptions,
&m_ListOptions,
&m_UploadOptions,
&m_DownloadOptions,
@@ -135,6 +140,7 @@ private:
&m_ResumeOptions,
&m_AbortOptions,
&m_DiffOptions,
+ &m_LsOptions,
&m_FetchBlobOptions,
&m_ValidateBuildPartOptions,
&m_TestOptions,
diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp
index 469dddf55..5bc8a8ed8 100644
--- a/src/zen/cmds/print_cmd.cpp
+++ b/src/zen/cmds/print_cmd.cpp
@@ -15,25 +15,41 @@ using namespace std::literals;
namespace zen {
static void
-PrintCbObject(CbObject Object)
+PrintCbObject(CbObject Object, bool AddTypeComment)
{
ExtendableStringBuilder<1024> ObjStr;
- CompactBinaryToJson(Object, ObjStr);
+ CompactBinaryToJson(Object, ObjStr, AddTypeComment);
ZEN_CONSOLE("{}", ObjStr);
}
static void
-PrintCompactBinary(IoBuffer Data)
+PrintCompactBinary(IoBuffer Data, bool AddTypeComment)
{
ExtendableStringBuilder<1024> StreamString;
- CompactBinaryToJson(Data.GetView(), StreamString);
+ CompactBinaryToJson(Data.GetView(), StreamString, AddTypeComment);
ZEN_CONSOLE("{}", StreamString);
}
+static CbValidateError
+MakeFilteredResult(CbValidateError Result)
+{
+ CbValidateError FilteredResult = Result;
+ EnumRemoveFlags(FilteredResult,
+ CbValidateError::InvalidString | CbValidateError::InvalidInteger | CbValidateError::InvalidFloat |
+ CbValidateError::NonUniformObject | CbValidateError::NonUniformArray | CbValidateError::Padding);
+ return FilteredResult;
+}
+
PrintCommand::PrintCommand()
{
m_Options.add_options()("h,help", "Print help");
m_Options.add_option("", "s", "source", "Object payload file (use '-' to read from STDIN)", cxxopts::value(m_Filename), "<file name>");
+ m_Options.add_option("",
+ "",
+ "show-type-info",
+ "Add type info annotation to compact binary objects",
+ cxxopts::value(m_ShowCbObjectTypeInfo),
+ "<showtypes>");
m_Options.parse_positional({"source"});
}
@@ -62,6 +78,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
+ MakeSafeAbsolutePathÍnPlace(m_Filename);
Fc = ReadFile(m_Filename);
}
@@ -126,22 +143,59 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("---8<---");
- PrintCbObject(Object);
- }
- else if (CbValidateError Result = ValidateCompactBinary(Data,
- CbValidateMode::Default | CbValidateMode::Names | CbValidateMode::Format |
- CbValidateMode::Package | CbValidateMode::PackageHash);
- Result == CbValidateError::None)
- {
- PrintCompactBinary(Data);
+ CbValidateError Result = ValidateCompactBinary(Object.GetView(), CbValidateMode::All);
+
+ CbValidateError FilteredResult = MakeFilteredResult(Result);
+
+ if (FilteredResult == CbValidateError::None && FilteredResult != Result)
+ {
+ ZEN_WARN(
+ "Object in package message file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')",
+ m_Filename,
+ uint32_t(Result),
+ ToString(Result));
+ }
+
+ if (FilteredResult != CbValidateError::None)
+ {
+ ZEN_ERROR("Object in package message file '{}' does not appear to be compact binary (validation error {:#x}: '{}')",
+ m_Filename,
+ uint32_t(FilteredResult),
+ ToString(FilteredResult));
+ return 1;
+ }
+ else
+ {
+ PrintCbObject(Object, m_ShowCbObjectTypeInfo);
+ }
}
else
{
- ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x})", m_Filename, uint32_t(Result));
+ CbValidateError Result = ValidateCompactBinary(Data, CbValidateMode::All);
- return 1;
- }
+ CbValidateError FilteredResult = MakeFilteredResult(Result);
+
+ if (FilteredResult == CbValidateError::None && FilteredResult != Result)
+ {
+ ZEN_WARN("Data in file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')",
+ m_Filename,
+ uint32_t(Result),
+ ToString(Result));
+ }
+ if (FilteredResult != CbValidateError::None)
+ {
+ ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x}: '{}')",
+ m_Filename,
+ uint32_t(FilteredResult),
+ ToString(FilteredResult));
+ return 1;
+ }
+ else
+ {
+ PrintCompactBinary(Data, m_ShowCbObjectTypeInfo);
+ }
+ }
return 0;
}
@@ -151,6 +205,12 @@ PrintPackageCommand::PrintPackageCommand()
{
m_Options.add_options()("h,help", "Print help");
m_Options.add_option("", "s", "source", "Package payload file", cxxopts::value(m_Filename), "<file name>");
+ m_Options.add_option("",
+ "",
+ "show-type-info",
+ "Add type info annotation to compact binary objects",
+ cxxopts::value(m_ShowCbObjectTypeInfo),
+ "<showtypes>");
m_Options.parse_positional({"source"});
}
@@ -173,6 +233,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
if (m_Filename.empty())
throw std::runtime_error("No file specified");
+ MakeSafeAbsolutePathÍnPlace(m_Filename);
FileContents Fc = ReadFile(m_Filename);
IoBuffer Data = Fc.Flatten();
CbPackage Package;
@@ -182,7 +243,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
if (Ok)
{
ExtendableStringBuilder<1024> ObjStr;
- CompactBinaryToJson(Package.GetObject(), ObjStr);
+ CompactBinaryToJson(Package.GetObject(), ObjStr, m_ShowCbObjectTypeInfo);
ZEN_CONSOLE("{}", ObjStr);
}
else
diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h
index 4d6a492b7..80729901e 100644
--- a/src/zen/cmds/print_cmd.h
+++ b/src/zen/cmds/print_cmd.h
@@ -19,8 +19,9 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"print", "Print compact binary object"};
- std::string m_Filename;
+ cxxopts::Options m_Options{"print", "Print compact binary object"};
+ std::filesystem::path m_Filename;
+ bool m_ShowCbObjectTypeInfo = false;
};
/** Print Compact Binary Package
@@ -36,8 +37,9 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"printpkg", "Print compact binary package"};
- std::string m_Filename;
+ cxxopts::Options m_Options{"printpkg", "Print compact binary package"};
+ std::filesystem::path m_Filename;
+ bool m_ShowCbObjectTypeInfo = false;
};
} // namespace zen
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 58af0577e..f919edc87 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -149,6 +149,16 @@ namespace {
}
}
+ class AsyncJobError : public std::runtime_error
+ {
+ public:
+ using _Mybase = runtime_error;
+
+ AsyncJobError(const std::string& Message, int ReturnCode) : _Mybase(Message), m_ReturnCode(ReturnCode) {}
+
+ const int m_ReturnCode = 0;
+ };
+
void ExecuteAsyncOperation(HttpClient& Http, std::string_view Url, IoBuffer&& Payload, bool PlainProgress)
{
signal(SIGINT, SignalCallbackHandler);
@@ -252,13 +262,14 @@ namespace {
if (Status == "Aborted")
{
std::string_view AbortReason = StatusObject["AbortReason"].AsString();
+ int ReturnCode = StatusObject["ReturnCode"].AsInt32(-1);
if (!AbortReason.empty())
{
- throw std::runtime_error(std::string(AbortReason));
+ throw AsyncJobError(std::string(AbortReason), ReturnCode);
}
else
{
- throw std::runtime_error("Aborted");
+ throw AsyncJobError("Aborted", ReturnCode);
}
break;
}
@@ -1335,24 +1346,44 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
ZEN_CONSOLE("Saving oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, m_HostName, TargetDescription);
- if (m_Async)
+ try
{
- if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName),
- std::move(Payload),
- HttpClient::Accept(ZenContentType::kJSON));
- Result)
+ if (m_Async)
{
- ZEN_CONSOLE("{}", Result.ToText());
+ if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName),
+ std::move(Payload),
+ HttpClient::Accept(ZenContentType::kJSON));
+ Result)
+ {
+ ZEN_CONSOLE("{}", Result.ToText());
+ }
+ else
+ {
+ Result.ThrowError("failed requesting loading oplog export"sv);
+ }
}
else
{
- Result.ThrowError("failed requesting loading oplog export"sv);
- return 1;
+ ExecuteAsyncOperation(Http,
+ fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName),
+ std::move(Payload),
+ m_PlainProgress);
}
}
- else
+ catch (const HttpClientError& Ex)
{
- ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload), m_PlainProgress);
+ ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
+ return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode;
+ }
+ catch (const AsyncJobError& Ex)
+ {
+ ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
+ return Ex.m_ReturnCode;
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
+ return 1;
}
return 0;
}
@@ -1668,24 +1699,44 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
ZEN_CONSOLE("Loading oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, SourceDescription, m_HostName);
- if (m_Async)
+ try
{
- if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName),
- std::move(Payload),
- HttpClient::Accept(ZenContentType::kJSON));
- Result)
+ if (m_Async)
{
- ZEN_CONSOLE("{}", Result.ToText());
+ if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName),
+ std::move(Payload),
+ HttpClient::Accept(ZenContentType::kJSON));
+ Result)
+ {
+ ZEN_CONSOLE("{}", Result.ToText());
+ }
+ else
+ {
+ Result.ThrowError("failed requesting loading oplog import"sv);
+ }
}
else
{
- Result.ThrowError("failed requesting loading oplog import"sv);
- return 1;
+ ExecuteAsyncOperation(Http,
+ fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName),
+ std::move(Payload),
+ m_PlainProgress);
}
}
- else
+ catch (const HttpClientError& Ex)
+ {
+ ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what());
+ return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode;
+ }
+ catch (const AsyncJobError& Ex)
{
- ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload), m_PlainProgress);
+ ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
+ return Ex.m_ReturnCode;
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what());
+ return 1;
}
return 0;
}
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 15988080a..df1d8b5c1 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -787,7 +787,8 @@ main(int argc, char** argv)
std::string_view ThisArg(argv[j]);
PassthroughArgV.push_back(std::string(ThisArg));
- const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos);
+ const bool NeedsQuotes =
+ (ThisArg.find(' ') != std::string_view::npos) && !(ThisArg.starts_with("\"") && ThisArg.ends_with("\""));
if (NeedsQuotes)
{
diff --git a/src/zen/zen.h b/src/zen/zen.h
index 995bd13b6..40c745bc7 100644
--- a/src/zen/zen.h
+++ b/src/zen/zen.h
@@ -105,7 +105,8 @@ public:
{
Plain,
Pretty,
- Log
+ Log,
+ Quiet
};
static void SetLogOperationName(Mode InMode, std::string_view Name);
diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp
index 9b06d4575..b22f2ec1f 100644
--- a/src/zencore/callstack.cpp
+++ b/src/zencore/callstack.cpp
@@ -179,19 +179,26 @@ FormatCallstack(const CallstackFrames* Callstack, StringBuilderBase& SB, std::st
bool First = true;
for (const std::string& Symbol : GetFrameSymbols(Callstack))
{
- if (!First)
+ try
{
- SB.Append("\n");
- }
- else
- {
- First = false;
+ if (!First)
+ {
+ SB.Append("\n");
+ }
+ else
+ {
+ First = false;
+ }
+ if (!Prefix.empty())
+ {
+ SB.Append(Prefix);
+ }
+ SB.Append(Symbol);
}
- if (!Prefix.empty())
+ catch (const std::exception&)
{
- SB.Append(Prefix);
+ break;
}
- SB.Append(Symbol);
}
}
diff --git a/src/zencore/compactbinaryfile.cpp b/src/zencore/compactbinaryfile.cpp
index f2121a0bd..1526c21d5 100644
--- a/src/zencore/compactbinaryfile.cpp
+++ b/src/zencore/compactbinaryfile.cpp
@@ -19,7 +19,7 @@ LoadCompactBinaryObject(const std::filesystem::path& FilePath)
IoBuffer ObjectBuffer = ObjectFile.Flatten();
- if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::All); Result == CbValidateError::None)
+ if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::Default); Result == CbValidateError::None)
{
CbObject Object = LoadCompactBinaryObject(ObjectBuffer);
const IoHash WorkerId = IoHash::HashBuffer(ObjectBuffer);
diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp
index 68ed09549..02f22ba4d 100644
--- a/src/zencore/compactbinaryjson.cpp
+++ b/src/zencore/compactbinaryjson.cpp
@@ -22,7 +22,10 @@ namespace zen {
class CbJsonWriter
{
public:
- explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; }
+ explicit CbJsonWriter(StringBuilderBase& InBuilder, bool AddTypeComment) : Builder(InBuilder), m_AddTypeComment(AddTypeComment)
+ {
+ NewLineAndIndent << LINE_TERMINATOR_ANSI;
+ }
void BeginObject()
{
@@ -74,11 +77,32 @@ public:
switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
{
case CbFieldType::Null:
+ if (m_AddTypeComment)
+ {
+ Builder << "[Null] ";
+ }
Builder << "null"sv;
break;
case CbFieldType::Object:
+ {
+ if (m_AddTypeComment)
+ {
+ Builder << "[Object] ";
+ }
+ BeginObject();
+ for (CbFieldView It : Field)
+ {
+ WriteField(It);
+ }
+ EndObject();
+ }
+ break;
case CbFieldType::UniformObject:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[UniformObject] ";
+ }
BeginObject();
for (CbFieldView It : Field)
{
@@ -88,8 +112,25 @@ public:
}
break;
case CbFieldType::Array:
+ {
+ if (m_AddTypeComment)
+ {
+ Builder << "[Array] ";
+ }
+ BeginArray();
+ for (CbFieldView It : Field)
+ {
+ WriteField(It);
+ }
+ EndArray();
+ }
+ break;
case CbFieldType::UniformArray:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[UniformArray] ";
+ }
BeginArray();
for (CbFieldView It : Field)
{
@@ -99,19 +140,39 @@ public:
}
break;
case CbFieldType::Binary:
+ if (m_AddTypeComment)
+ {
+ Builder << "[Binary] ";
+ }
AppendBase64String(Accessor.AsBinary());
break;
case CbFieldType::String:
+ if (m_AddTypeComment)
+ {
+ Builder << "[String] ";
+ }
AppendQuotedString(Accessor.AsU8String());
break;
case CbFieldType::IntegerPositive:
+ if (m_AddTypeComment)
+ {
+ Builder << "[IntegerPositive] ";
+ }
Builder << Accessor.AsIntegerPositive();
break;
case CbFieldType::IntegerNegative:
+ if (m_AddTypeComment)
+ {
+ Builder << "[IntegerNegative] ";
+ }
Builder << Accessor.AsIntegerNegative();
break;
case CbFieldType::Float32:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[Float32] ";
+ }
const float Value = Accessor.AsFloat32();
if (std::isfinite(Value))
{
@@ -125,6 +186,10 @@ public:
break;
case CbFieldType::Float64:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[Float64] ";
+ }
const double Value = Accessor.AsFloat64();
if (std::isfinite(Value))
{
@@ -137,14 +202,36 @@ public:
}
break;
case CbFieldType::BoolFalse:
+ if (m_AddTypeComment)
+ {
+ Builder << "[BoolFalse] ";
+ }
Builder << "false"sv;
break;
case CbFieldType::BoolTrue:
+ if (m_AddTypeComment)
+ {
+ Builder << "[BoolTrue] ";
+ }
Builder << "true"sv;
break;
case CbFieldType::ObjectAttachment:
+ {
+ if (m_AddTypeComment)
+ {
+ Builder << "[ObjectAttachment] ";
+ }
+ Builder << '"';
+ Accessor.AsAttachment().ToHexString(Builder);
+ Builder << '"';
+ }
+ break;
case CbFieldType::BinaryAttachment:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[BinaryAttachment] ";
+ }
Builder << '"';
Accessor.AsAttachment().ToHexString(Builder);
Builder << '"';
@@ -152,6 +239,10 @@ public:
break;
case CbFieldType::Hash:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[Hash] ";
+ }
Builder << '"';
Accessor.AsHash().ToHexString(Builder);
Builder << '"';
@@ -159,16 +250,28 @@ public:
break;
case CbFieldType::Uuid:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[Uuid] ";
+ }
Builder << '"';
Accessor.AsUuid().ToString(Builder);
Builder << '"';
}
break;
case CbFieldType::DateTime:
+ if (m_AddTypeComment)
+ {
+ Builder << "[DateTime] ";
+ }
Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"';
break;
case CbFieldType::TimeSpan:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[TimeSpan] ";
+ }
const TimeSpan Span(Accessor.AsTimeSpanTicks());
if (Span.GetDays() == 0)
{
@@ -181,12 +284,20 @@ public:
break;
}
case CbFieldType::ObjectId:
+ if (m_AddTypeComment)
+ {
+ Builder << "[ObjectId] ";
+ }
Builder << '"';
Accessor.AsObjectId().ToString(Builder);
Builder << '"';
break;
case CbFieldType::CustomById:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[CustomById] ";
+ }
CbCustomById Custom = Accessor.AsCustomById();
Builder << "{ \"Id\": ";
Builder << Custom.Id;
@@ -197,6 +308,10 @@ public:
}
case CbFieldType::CustomByName:
{
+ if (m_AddTypeComment)
+ {
+ Builder << "[CustomByName] ";
+ }
CbCustomByName Custom = Accessor.AsCustomByName();
Builder << "{ \"Name\": ";
AppendQuotedString(Custom.Name);
@@ -299,29 +414,30 @@ private:
private:
StringBuilderBase& Builder;
ExtendableStringBuilder<32> NewLineAndIndent;
+ const bool m_AddTypeComment;
bool NeedsComma{false};
bool NeedsNewLine{false};
};
void
-CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder)
+CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment)
{
- CbJsonWriter Writer(Builder);
+ CbJsonWriter Writer(Builder, AddTypeComment);
Writer.WriteField(Object.AsFieldView());
}
void
CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder)
{
- CbJsonWriter Writer(Builder);
+ CbJsonWriter Writer(Builder, /*AddTypeComment*/ false);
Writer.WriteField(Array.AsFieldView());
}
void
-CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder)
+CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment)
{
std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data);
- CbJsonWriter Writer(InBuilder);
+ CbJsonWriter Writer(InBuilder, AddTypeComment);
if (!Fields.empty())
{
if (Fields.size() == 1)
diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp
index 833649b88..d7292f405 100644
--- a/src/zencore/compactbinaryvalidation.cpp
+++ b/src/zencore/compactbinaryvalidation.cpp
@@ -705,10 +705,11 @@ ToString(const CbValidateError Error)
ExtendableStringBuilder<128> Out;
- auto AppendFlag = [&, IsFirst = false](std::string_view FlagString) {
+ auto AppendFlag = [&, IsFirst = true](std::string_view FlagString) mutable {
if (!IsFirst)
Out.Append('|');
Out.Append(FlagString);
+ IsFirst = false;
};
#define _ENUM_CASE(V) \
@@ -737,7 +738,11 @@ ToString(const CbValidateError Error)
#undef _ENUM_CASE
- return "Error";
+ if (Out.Size() == 0)
+ {
+ return "Error";
+ }
+ return Out.ToString();
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h
index 0fdb56d67..82ca055ab 100644
--- a/src/zencore/include/zencore/compactbinary.h
+++ b/src/zencore/include/zencore/compactbinary.h
@@ -996,7 +996,7 @@ private:
/**
* Serialize a compact binary object to JSON.
*/
-ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder);
+ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment = false);
/**
* Serialize a compact binary object to YAML.
*/
@@ -1520,7 +1520,7 @@ end(CbFieldView&)
/**
* Serialize serialized compact binary blob to JSON. It must be 0 to n fields with including type for each field
*/
-ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder);
+ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment = false);
/**
* Serialize serialized compact binary blob to YAML. It must be 0 to n fields with including type for each field
diff --git a/src/zencore/include/zencore/jobqueue.h b/src/zencore/include/zencore/jobqueue.h
index d5ec6255a..470ed3fc6 100644
--- a/src/zencore/include/zencore/jobqueue.h
+++ b/src/zencore/include/zencore/jobqueue.h
@@ -28,6 +28,16 @@ public:
virtual void ReportProgress(std::string_view CurrentOp, std::string_view Details, ptrdiff_t TotalCount, ptrdiff_t RemainingCount) = 0;
};
+class JobError : public std::runtime_error
+{
+public:
+ using _Mybase = runtime_error;
+
+ JobError(const std::string& Message, int ReturnCode) : _Mybase(Message), m_ReturnCode(ReturnCode) {}
+
+ const int m_ReturnCode = 0;
+};
+
class JobQueue
{
public:
@@ -73,6 +83,7 @@ public:
std::chrono::system_clock::time_point StartTime;
std::chrono::system_clock::time_point EndTime;
int WorkerThreadId;
+ int ReturnCode;
};
// Will only respond once when status is Complete or Aborted
diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index 8e9a37a27..be9b39e7a 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -429,7 +429,25 @@ IoBufferExtendedCore::SetDeleteOnClose(bool DeleteOnClose)
//////////////////////////////////////////////////////////////////////////
-RefPtr<IoBufferCore> IoBuffer::NullBufferCore(new IoBufferCore);
+static IoBufferCore*
+GetNullBufferCore()
+{
+ // This is safe from a threading standpoint since the first call is non-threaded (during static init) and for the following
+ // calls Core is never nullptr
+ // We do this workaround since we don't want to call new (IoBufferCore) at static initializers
+ // Calling new during static initialize causes problem with memtracing since the flags are not set up correctly yet
+
+ static IoBufferCore NullBufferCore;
+ static IoBufferCore* Core = nullptr;
+ if (Core == nullptr)
+ {
+ Core = &NullBufferCore;
+ Core->AddRef(); // Make sure we never deallocate it as it is a static instance
+ }
+ return Core;
+}
+
+RefPtr<IoBufferCore> IoBuffer::NullBufferCore(GetNullBufferCore());
IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize))
{
diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp
index b97484458..5d727b69c 100644
--- a/src/zencore/jobqueue.cpp
+++ b/src/zencore/jobqueue.cpp
@@ -50,6 +50,7 @@ public:
JobClock::Tick StartTick;
JobClock::Tick EndTick;
int WorkerThreadId;
+ int ReturnCode;
virtual bool IsCancelled() const override { return CancelFlag.load(); }
virtual void ReportMessage(std::string_view Message) override { Queue->ReportMessage(Id, Message); }
@@ -101,6 +102,7 @@ public:
NewJob->StartTick = JobClock::Never();
NewJob->EndTick = JobClock::Never();
NewJob->WorkerThreadId = 0;
+ NewJob->ReturnCode = -1;
ZEN_DEBUG("Scheduling background job {}:'{}'", NewJob->Id.Id, NewJob->Name);
QueueLock.WithExclusiveLock([&]() { QueuedJobs.emplace_back(std::move(NewJob)); });
@@ -274,7 +276,8 @@ public:
.CreateTime = JobClock::TimePointFromTick(Job.CreateTick),
.StartTime = JobClock::TimePointFromTick(Job.StartTick),
.EndTime = JobClock::TimePointFromTick(Job.EndTick),
- .WorkerThreadId = Job.WorkerThreadId};
+ .WorkerThreadId = Job.WorkerThreadId,
+ .ReturnCode = Job.ReturnCode};
};
std::optional<JobDetails> Result;
@@ -365,6 +368,7 @@ public:
ZEN_DEBUG("Executing background job {}:'{}'", CurrentJob->Id.Id, CurrentJob->Name);
CurrentJob->Callback(*CurrentJob);
ZEN_DEBUG("Completed background job {}:'{}'", CurrentJob->Id.Id, CurrentJob->Name);
+ CurrentJob->ReturnCode = 0;
QueueLock.WithExclusiveLock([&]() {
CurrentJob->EndTick = JobClock::Now();
CurrentJob->WorkerThreadId = 0;
@@ -383,6 +387,22 @@ public:
AbortedJobs.insert_or_assign(CurrentJob->Id.Id, std::move(CurrentJob));
});
}
+ catch (const JobError& Ex)
+ {
+ ZEN_DEBUG("Background job {}:'{}' failed. Reason: '{}'. Return code {}",
+ CurrentJob->Id.Id,
+ CurrentJob->Name,
+ Ex.what(),
+ Ex.m_ReturnCode);
+ QueueLock.WithExclusiveLock([&]() {
+ CurrentJob->State.AbortReason = Ex.what();
+ CurrentJob->EndTick = JobClock::Now();
+ CurrentJob->WorkerThreadId = 0;
+ CurrentJob->ReturnCode = Ex.m_ReturnCode;
+ RunningJobs.erase(CurrentJob->Id.Id);
+ AbortedJobs.insert_or_assign(CurrentJob->Id.Id, std::move(CurrentJob));
+ });
+ }
catch (const std::exception& Ex)
{
ZEN_DEBUG("Background job {}:'{}' aborted. Reason: '{}'", CurrentJob->Id.Id, CurrentJob->Name, Ex.what());
diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp
index fef5c28a4..b8ec85a4a 100644
--- a/src/zencore/thread.cpp
+++ b/src/zencore/thread.cpp
@@ -80,8 +80,10 @@ SetNameInternal(DWORD thread_id, const char* name)
void
SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName)
{
- StringBuilder<256> ThreadNameZ;
- ThreadNameZ << ThreadName;
+ constexpr std::string_view::size_type MaxThreadNameLength = 255;
+ std::string_view LimitedThreadName = ThreadName.substr(0, MaxThreadNameLength);
+ StringBuilder<MaxThreadNameLength + 1> ThreadNameZ;
+ ThreadNameZ << LimitedThreadName;
const int ThreadId = GetCurrentThreadId();
#if ZEN_WITH_TRACE
@@ -95,8 +97,8 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName)
if (SetThreadDescriptionFunc)
{
- WideStringBuilder<256> ThreadNameW;
- Utf8ToWide(ThreadName, ThreadNameW);
+ WideStringBuilder<MaxThreadNameLength + 1> ThreadNameW;
+ Utf8ToWide(LimitedThreadName, ThreadNameW);
SetThreadDescriptionFunc(::GetCurrentThread(), ThreadNameW.c_str());
}
diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp
index 82d28c0e3..51e06ae14 100644
--- a/src/zencore/zencore.cpp
+++ b/src/zencore/zencore.cpp
@@ -124,7 +124,14 @@ AssertImpl::ExecAssert(const char* Filename, int LineNumber, const char* Functio
AssertImpl* AssertImpl = CurrentAssertImpl;
while (AssertImpl)
{
- AssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg, Callstack);
+ try
+ {
+ AssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg, Callstack);
+ }
+ catch (const std::exception&)
+ {
+ // Just keep exception silent - we don't want exception thrown from assert callbacks
+ }
AssertImpl = AssertImpl->NextAssertImpl;
}
ThrowAssertException(Filename, LineNumber, FunctionName, Msg, Callstack);
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index a2d323b5e..30a2bfc65 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -1628,12 +1628,13 @@ HttpClient::Response::ErrorMessage(std::string_view Prefix) const
}
else if (StatusCode != HttpResponseCode::ImATeapot && (int)StatusCode)
{
- return fmt::format("{}{}HTTP error {} {} ({})",
+ std::string TextResponse = ToText();
+ return fmt::format("{}{}HTTP error {} {}{}",
Prefix,
Prefix.empty() ? ""sv : ": "sv,
(int)StatusCode,
zen::ToString(StatusCode),
- ToText());
+ TextResponse.empty() ? ""sv : fmt::format(" ({})", TextResponse));
}
else
{
@@ -1646,7 +1647,7 @@ HttpClient::Response::ThrowError(std::string_view ErrorPrefix)
{
if (!IsSuccess())
{
- throw std::runtime_error(ErrorMessage(ErrorPrefix));
+ throw HttpClientError(ErrorMessage(ErrorPrefix), Error.has_value() ? Error.value().ErrorCode : 0, StatusCode);
}
}
diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp
index 39efe1d0c..62e1b77bc 100644
--- a/src/zenhttp/httpclientauth.cpp
+++ b/src/zenhttp/httpclientauth.cpp
@@ -89,14 +89,25 @@ namespace zen { namespace httpclientauth {
static HttpClientAccessToken GetOidcTokenFromExe(const std::filesystem::path& OidcExecutablePath,
std::string_view CloudHost,
- bool Unattended)
+ bool Unattended,
+ bool Quiet)
{
Stopwatch Timer;
CreateProcOptions ProcOptions;
+ if (Quiet)
+ {
+ ProcOptions.StdoutFile = std::filesystem::temp_directory_path() / fmt::format(".zen-auth-output-{}", Oid::NewOid());
+ }
const std::filesystem::path AuthTokenPath(std::filesystem::temp_directory_path() / fmt::format(".zen-auth-{}", Oid::NewOid()));
- auto _ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); });
+ auto _ = MakeGuard([AuthTokenPath, &ProcOptions]() {
+ RemoveFile(AuthTokenPath);
+ if (!ProcOptions.StdoutFile.empty())
+ {
+ RemoveFile(ProcOptions.StdoutFile);
+ }
+ });
const std::string ProcArgs = fmt::format("{} --AuthConfigUrl {} --OutFile {} --Unattended={}",
OidcExecutablePath,
@@ -164,13 +175,15 @@ namespace zen { namespace httpclientauth {
}
std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath,
- std::string_view CloudHost)
+ std::string_view CloudHost,
+ bool Quiet)
{
- HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, false);
+ HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ false, Quiet);
if (InitialToken.IsValid())
{
return [OidcExecutablePath = std::filesystem::path(OidcExecutablePath),
CloudHost = std::string(CloudHost),
+ Quiet,
InitialToken]() mutable {
if (InitialToken.IsValid())
{
@@ -178,7 +191,7 @@ namespace zen { namespace httpclientauth {
InitialToken = {};
return Result;
}
- return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, true);
+ return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, /* Unattended */ true, Quiet);
};
}
return {};
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index c991a71ea..50bd5b53a 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -57,6 +57,29 @@ struct HttpClientSettings
uint8_t RetryCount = 0;
};
+class HttpClientError : public std::runtime_error
+{
+public:
+ using _Mybase = runtime_error;
+
+ HttpClientError(const std::string& Message, int Error, HttpResponseCode ResponseCode)
+ : _Mybase(Message)
+ , m_Error(Error)
+ , m_ResponseCode(ResponseCode)
+ {
+ }
+
+ HttpClientError(const char* Message, int Error, HttpResponseCode ResponseCode)
+ : _Mybase(Message)
+ , m_Error(Error)
+ , m_ResponseCode(ResponseCode)
+ {
+ }
+
+ const int m_Error = 0;
+ const HttpResponseCode m_ResponseCode = HttpResponseCode::ImATeapot;
+};
+
class HttpClient
{
public:
diff --git a/src/zenhttp/include/zenhttp/httpclientauth.h b/src/zenhttp/include/zenhttp/httpclientauth.h
index 5b9b9d305..32d00f87f 100644
--- a/src/zenhttp/include/zenhttp/httpclientauth.h
+++ b/src/zenhttp/include/zenhttp/httpclientauth.h
@@ -27,7 +27,8 @@ namespace httpclientauth {
std::function<HttpClientAccessToken()> CreateFromDefaultOpenIdProvider(AuthMgr& AuthManager);
std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath,
- std::string_view CloudHost);
+ std::string_view CloudHost,
+ bool Quiet);
} // namespace httpclientauth
} // namespace zen
diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp
index 9d423ecbc..0b7848f79 100644
--- a/src/zenhttp/packageformat.cpp
+++ b/src/zenhttp/packageformat.cpp
@@ -576,7 +576,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
if (!MalformedAttachments.empty())
{
- StringBuilder<1024> SB;
+ ExtendableStringBuilder<1024> SB;
SB << (uint64_t)MalformedAttachments.size() << " malformed attachments in package message:\n";
for (const auto& It : MalformedAttachments)
{
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 78d8ab68a..77ed87cb1 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -13,6 +13,7 @@
#include <zencore/iohash.h>
#include <zencore/logging.h>
#include <zencore/memoryview.h>
+#include <zencore/scopeguard.h>
#include <zencore/stream.h>
#include <zencore/string.h>
#include <zencore/testutils.h>
@@ -22,6 +23,7 @@
#include <zenhttp/httpclient.h>
#include <zenhttp/packageformat.h>
#include <zenhttp/zenhttp.h>
+#include <zenutil/buildstoragecache.h>
#include <zenutil/cache/cache.h>
#include <zenutil/cache/cacherequests.h>
#include <zenutil/chunkrequests.h>
@@ -37,6 +39,7 @@
ZEN_THIRD_PARTY_INCLUDES_START
#include <cpr/cpr.h>
+#include <tsl/robin_set.h>
#undef GetObject
ZEN_THIRD_PARTY_INCLUDES_END
@@ -554,9 +557,13 @@ namespace utils {
return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(), .Port = Port, .Args = std::move(Args)};
}
- static ZenConfig NewWithUpstream(uint16_t Port, uint16_t UpstreamPort)
+ static ZenConfig NewWithUpstream(uint16_t Port, uint16_t UpstreamPort, std::string Args = "")
{
- return New(Port, fmt::format("--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", UpstreamPort));
+ return New(Port,
+ fmt::format("{}{}--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}",
+ Args,
+ Args.length() > 0 ? " " : "",
+ UpstreamPort));
}
static ZenConfig NewWithThreadedUpstreams(uint16_t NewPort, std::span<uint16_t> UpstreamPorts, bool Debug)
@@ -1302,8 +1309,10 @@ TEST_CASE("zcache.rpc")
std::string_view Namespace,
std::string_view Bucket,
size_t Num,
- size_t PayloadSize = 1024,
- size_t KeyOffset = 1) -> std::vector<CacheKey> {
+ size_t PayloadSize = 1024,
+ size_t KeyOffset = 1,
+ CachePolicy PutPolicy = CachePolicy::Default,
+ std::vector<CbPackage>* OutPackages = nullptr) -> std::vector<CacheKey> {
std::vector<zen::CacheKey> OutKeys;
for (uint32_t Key = 1; Key <= Num; ++Key)
@@ -1313,7 +1322,7 @@ TEST_CASE("zcache.rpc")
const zen::CacheKey CacheKey = zen::CacheKey::Create(Bucket, KeyHash);
cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)};
- AppendCacheRecord(Request, CacheKey, PayloadSize, CachePolicy::Default);
+ AppendCacheRecord(Request, CacheKey, PayloadSize, PutPolicy);
OutKeys.push_back(CacheKey);
CbPackage Package;
@@ -1325,6 +1334,10 @@ TEST_CASE("zcache.rpc")
cpr::Body{(const char*)Body.GetData(), Body.GetSize()});
CHECK(Result.status_code == 200);
+ if (OutPackages)
+ {
+ OutPackages->emplace_back(std::move(Package));
+ }
}
return OutKeys;
@@ -1503,6 +1516,339 @@ TEST_CASE("zcache.rpc")
}
}
+ SUBCASE("policy - 'QueryLocal' on put allows overwrite with differing value when not limiting overwrites")
+ {
+ using namespace utils;
+
+ ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
+ ZenServerInstance UpstreamServer(TestEnv);
+ SpawnServer(UpstreamServer, UpstreamCfg);
+
+ ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+ ZenServerInstance LocalServer(TestEnv);
+ SpawnServer(LocalServer, LocalCfg);
+
+ size_t PayloadSize = 1024;
+ std::string_view Namespace("ue4.ddc"sv);
+ std::string_view Bucket("mastodon"sv);
+ const size_t NumRecords = 4;
+ std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize);
+
+ for (const zen::CacheKey& CacheKey : Keys)
+ {
+ cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)};
+ AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Default);
+
+ CbPackage Package;
+ CHECK(Request.Format(Package));
+
+ IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer();
+ cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)},
+ cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}},
+ cpr::Body{(const char*)Body.GetData(), Body.GetSize()});
+
+ CHECK(Result.status_code == 200);
+ cacherequests::PutCacheRecordsResult ParsedResult;
+ CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()));
+ CHECK(!Response.IsNull());
+ CHECK(ParsedResult.Parse(Response));
+ for (bool ResponseSuccess : ParsedResult.Success)
+ {
+ CHECK(ResponseSuccess);
+ }
+ }
+
+ auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) {
+ CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote);
+ GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy);
+
+ CHECK(Result.Result.Results.size() == Keys.size());
+
+ for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results)
+ {
+ CHECK(Record);
+ const CacheKey& ExpectedKey = Keys[Index++];
+ CHECK(Record->Key == ExpectedKey);
+ for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values)
+ {
+ CHECK(Value.RawSize == PayloadSize * 2);
+ }
+ }
+ };
+
+ // Check that the records are present and overwritten in the local server
+ CheckRecordCorrectness(LocalCfg);
+
+ // Check that the records are present and overwritten in the upstream server
+ CheckRecordCorrectness(UpstreamCfg);
+ }
+
+ SUBCASE("policy - 'QueryLocal' on put denies overwrite with differing value when limiting overwrites")
+ {
+ using namespace utils;
+
+ ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
+ ZenServerInstance UpstreamServer(TestEnv);
+ SpawnServer(UpstreamServer, UpstreamCfg);
+
+ ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port, "--cache-bucket-limit-overwrites");
+ ZenServerInstance LocalServer(TestEnv);
+ SpawnServer(LocalServer, LocalCfg);
+
+ size_t PayloadSize = 1024;
+ std::string_view Namespace("ue4.ddc"sv);
+ std::string_view Bucket("mastodon"sv);
+ const size_t NumRecords = 4;
+ std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize);
+
+ for (const zen::CacheKey& CacheKey : Keys)
+ {
+ cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)};
+ AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Default);
+
+ CbPackage Package;
+ CHECK(Request.Format(Package));
+
+ IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer();
+ cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)},
+ cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}},
+ cpr::Body{(const char*)Body.GetData(), Body.GetSize()});
+
+ CHECK(Result.status_code == 200);
+ cacherequests::PutCacheRecordsResult ParsedResult;
+ CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()));
+ CHECK(!Response.IsNull());
+ CHECK(ParsedResult.Parse(Response));
+ for (bool ResponseSuccess : ParsedResult.Success)
+ {
+ CHECK(!ResponseSuccess);
+ }
+ }
+
+ auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) {
+ CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote);
+ GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy);
+
+ CHECK(Result.Result.Results.size() == Keys.size());
+
+ for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results)
+ {
+ CHECK(Record);
+ const CacheKey& ExpectedKey = Keys[Index++];
+ CHECK(Record->Key == ExpectedKey);
+ for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values)
+ {
+ CHECK(Value.RawSize == PayloadSize);
+ }
+ }
+ };
+
+ // Check that the records are present and not overwritten in the local server
+ CheckRecordCorrectness(LocalCfg);
+
+ // Check that the records are present and not overwritten in the upstream server
+ CheckRecordCorrectness(UpstreamCfg);
+ }
+
+ SUBCASE("policy - no 'QueryLocal' on put allows overwrite with differing value when limiting overwrites")
+ {
+ using namespace utils;
+
+ ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
+ ZenServerInstance UpstreamServer(TestEnv);
+ SpawnServer(UpstreamServer, UpstreamCfg);
+
+ ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port, "--cache-bucket-limit-overwrites");
+ ZenServerInstance LocalServer(TestEnv);
+ SpawnServer(LocalServer, LocalCfg);
+
+ size_t PayloadSize = 1024;
+ std::string_view Namespace("ue4.ddc"sv);
+ std::string_view Bucket("mastodon"sv);
+ const size_t NumRecords = 4;
+ std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize);
+
+ for (const zen::CacheKey& CacheKey : Keys)
+ {
+ cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)};
+ AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Store);
+
+ CbPackage Package;
+ CHECK(Request.Format(Package));
+
+ IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer();
+ cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)},
+ cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}},
+ cpr::Body{(const char*)Body.GetData(), Body.GetSize()});
+
+ CHECK(Result.status_code == 200);
+ cacherequests::PutCacheRecordsResult ParsedResult;
+ CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()));
+ CHECK(!Response.IsNull());
+ CHECK(ParsedResult.Parse(Response));
+ for (bool ResponseSuccess : ParsedResult.Success)
+ {
+ CHECK(ResponseSuccess);
+ }
+ }
+
+ auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) {
+ CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote);
+ GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy);
+
+ CHECK(Result.Result.Results.size() == Keys.size());
+
+ for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results)
+ {
+ CHECK(Record);
+ const CacheKey& ExpectedKey = Keys[Index++];
+ CHECK(Record->Key == ExpectedKey);
+ for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values)
+ {
+ CHECK(Value.RawSize == PayloadSize * 2);
+ }
+ }
+ };
+
+ // Check that the records are present and overwritten in the local server
+ CheckRecordCorrectness(LocalCfg);
+
+ // Check that the records are present and overwritten in the upstream server
+ CheckRecordCorrectness(UpstreamCfg);
+ }
+
+ SUBCASE("policy - 'QueryLocal' on put allows overwrite with equivalent value when limiting overwrites")
+ {
+ using namespace utils;
+
+ ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
+ ZenServerInstance UpstreamServer(TestEnv);
+ SpawnServer(UpstreamServer, UpstreamCfg);
+
+ ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port, "--cache-bucket-limit-overwrites");
+ ZenServerInstance LocalServer(TestEnv);
+ SpawnServer(LocalServer, LocalCfg);
+
+ size_t PayloadSize = 1024;
+ std::string_view Namespace("ue4.ddc"sv);
+ std::string_view Bucket("mastodon"sv);
+ const size_t NumRecords = 4;
+ std::vector<CbPackage> Packages;
+ std::vector<zen::CacheKey> Keys =
+ PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize, 1, CachePolicy::Default, &Packages);
+
+ for (const CbPackage& Package : Packages)
+ {
+ IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer();
+ cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)},
+ cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}},
+ cpr::Body{(const char*)Body.GetData(), Body.GetSize()});
+
+ CHECK(Result.status_code == 200);
+ cacherequests::PutCacheRecordsResult ParsedResult;
+ CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()));
+ CHECK(!Response.IsNull());
+ CHECK(ParsedResult.Parse(Response));
+ for (bool ResponseSuccess : ParsedResult.Success)
+ {
+ CHECK(ResponseSuccess);
+ }
+ }
+
+ auto CheckRecordCorrectness = [&](const ZenConfig& Cfg) {
+ CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote);
+ GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy);
+
+ CHECK(Result.Result.Results.size() == Keys.size());
+
+ for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results)
+ {
+ CHECK(Record);
+ const CacheKey& ExpectedKey = Keys[Index++];
+ CHECK(Record->Key == ExpectedKey);
+ for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values)
+ {
+ CHECK(Value.RawSize == PayloadSize);
+ }
+ }
+ };
+
+ // Check that the records are present and unchanged in the local server
+ CheckRecordCorrectness(LocalCfg);
+
+ // Check that the records are present and unchanged in the upstream server
+ CheckRecordCorrectness(UpstreamCfg);
+ }
+
+ // TODO: Propagation for rejected PUTs
+ // SUBCASE("policy - 'QueryLocal' on put denies overwrite with differing value when limiting overwrites but allows propagation to
+ // upstream")
+ // {
+ // using namespace utils;
+
+ // ZenConfig UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
+ // ZenServerInstance UpstreamServer(TestEnv);
+ // SpawnServer(UpstreamServer, UpstreamCfg);
+
+ // ZenConfig LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port,
+ // "--cache-bucket-limit-overwrites"); ZenServerInstance LocalServer(TestEnv); SpawnServer(LocalServer, LocalCfg);
+
+ // size_t PayloadSize = 1024;
+ // std::string_view Namespace("ue4.ddc"sv);
+ // std::string_view Bucket("mastodon"sv);
+ // const size_t NumRecords = 4;
+ // std::vector<zen::CacheKey> Keys = PutCacheRecords(LocalCfg.BaseUri, Namespace, Bucket, NumRecords, PayloadSize, 1,
+ // CachePolicy::Local);
+
+ // for (const zen::CacheKey& CacheKey : Keys)
+ // {
+ // cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)};
+ // AppendCacheRecord(Request, CacheKey, PayloadSize * 2, CachePolicy::Default);
+
+ // CbPackage Package;
+ // CHECK(Request.Format(Package));
+
+ // IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer();
+ // cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", LocalCfg.BaseUri)},
+ // cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}},
+ // cpr::Body{(const char*)Body.GetData(), Body.GetSize()});
+
+ // CHECK(Result.status_code == 200);
+ // cacherequests::PutCacheRecordsResult ParsedResult;
+ // CbPackage Response = ParsePackageMessage(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()));
+ // CHECK(!Response.IsNull());
+ // CHECK(ParsedResult.Parse(Response));
+ // for (bool ResponseSuccess : ParsedResult.Success)
+ // {
+ // CHECK(!ResponseSuccess);
+ // }
+ // }
+
+ // auto CheckRecordCorrectness = [&](const ZenConfig& Cfg, size_t ExpectedPayloadSize) {
+ // CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote);
+ // GetCacheRecordResult Result = GetCacheRecords(Cfg.BaseUri, "ue4.ddc"sv, Keys, Policy);
+
+ // CHECK(Result.Result.Results.size() == Keys.size());
+
+ // for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results)
+ // {
+ // CHECK(Record);
+ // const CacheKey& ExpectedKey = Keys[Index++];
+ // CHECK(Record->Key == ExpectedKey);
+ // for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values)
+ // {
+ // CHECK(Value.RawSize == ExpectedPayloadSize);
+ // }
+ // }
+ // };
+
+ // // Check that the records are present and not overwritten in the local server
+ // CheckRecordCorrectness(LocalCfg, PayloadSize);
+
+ // // Check that the records are present and are the newer size in the upstream server
+ // CheckRecordCorrectness(UpstreamCfg, PayloadSize*2);
+ // }
+
SUBCASE("RpcAcceptOptions")
{
using namespace utils;
@@ -3817,6 +4163,478 @@ TEST_CASE("workspaces.share")
CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)).StatusCode == HttpResponseCode::NotFound);
}
+TEST_CASE("buildstore.blobs")
+{
+ std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir();
+ auto _ = MakeGuard([&SystemRootPath]() { DeleteDirectories(SystemRootPath); });
+
+ std::string_view Namespace = "ns"sv;
+ std::string_view Bucket = "bkt"sv;
+ Oid BuildId = Oid::NewOid();
+
+ std::vector<IoHash> CompressedBlobsHashes;
+ {
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ for (size_t I = 0; I < 5; I++)
+ {
+ IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7);
+ CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+ CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+ IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+ Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+ HttpClient::Response Result =
+ Client.Put(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, CompressedBlobsHashes.back()), Payload);
+ CHECK(Result);
+ }
+
+ for (const IoHash& RawHash : CompressedBlobsHashes)
+ {
+ HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCompressedBinary));
+ CHECK(Result);
+ IoBuffer Payload = Result.ResponsePayload;
+ CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary);
+ IoHash VerifyRawHash;
+ uint64_t VerifyRawSize;
+ CompressedBuffer CompressedBlob =
+ CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize);
+ CHECK(CompressedBlob);
+ CHECK(VerifyRawHash == RawHash);
+ IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer();
+ CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
+ }
+ }
+ {
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ for (const IoHash& RawHash : CompressedBlobsHashes)
+ {
+ HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCompressedBinary));
+ CHECK(Result);
+ IoBuffer Payload = Result.ResponsePayload;
+ CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary);
+ IoHash VerifyRawHash;
+ uint64_t VerifyRawSize;
+ CompressedBuffer CompressedBlob =
+ CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize);
+ CHECK(CompressedBlob);
+ CHECK(VerifyRawHash == RawHash);
+ IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer();
+ CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
+ }
+
+ for (size_t I = 0; I < 5; I++)
+ {
+ IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7);
+ CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+ CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+ IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+ Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+ HttpClient::Response Result =
+ Client.Put(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, CompressedBlobsHashes.back()), Payload);
+ CHECK(Result);
+ }
+ }
+ {
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ for (const IoHash& RawHash : CompressedBlobsHashes)
+ {
+ HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCompressedBinary));
+ CHECK(Result);
+ IoBuffer Payload = Result.ResponsePayload;
+ CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary);
+ IoHash VerifyRawHash;
+ uint64_t VerifyRawSize;
+ CompressedBuffer CompressedBlob =
+ CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize);
+ CHECK(CompressedBlob);
+ CHECK(VerifyRawHash == RawHash);
+ IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer();
+ CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
+ }
+ }
+}
+
+namespace {
+ CbObject MakeMetadata(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues)
+ {
+ CbObjectWriter Writer;
+ Writer.AddHash("rawHash"sv, BlobHash);
+ Writer.BeginObject("values");
+ {
+ for (const auto& V : KeyValues)
+ {
+ Writer.AddString(V.first, V.second);
+ }
+ }
+ Writer.EndObject(); // values
+ return Writer.Save();
+ };
+
+} // namespace
+
+TEST_CASE("buildstore.metadata")
+{
+ std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir();
+ auto _ = MakeGuard([&SystemRootPath]() { DeleteDirectories(SystemRootPath); });
+
+ std::string_view Namespace = "ns"sv;
+ std::string_view Bucket = "bkt"sv;
+ Oid BuildId = Oid::NewOid();
+
+ std::vector<IoHash> BlobHashes;
+ std::vector<CbObject> Metadatas;
+ std::vector<IoHash> MetadataHashes;
+
+ auto GetMetadatas =
+ [](HttpClient& Client, std::string_view Namespace, std::string_view Bucket, const Oid& BuildId, std::vector<IoHash> BlobHashes) {
+ CbObjectWriter Request;
+
+ Request.BeginArray("blobHashes"sv);
+ for (const IoHash& BlobHash : BlobHashes)
+ {
+ Request.AddHash(BlobHash);
+ }
+ Request.EndArray();
+
+ IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+ Payload.SetContentType(ZenContentType::kCbObject);
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/getBlobMetadata", Namespace, Bucket, BuildId),
+ Payload,
+ HttpClient::Accept(ZenContentType::kCbObject));
+ CHECK(Result);
+
+ std::vector<CbObject> ResultMetadatas;
+
+ CbPackage ResponsePackage = ParsePackageMessage(Result.ResponsePayload);
+ CbObject ResponseObject = ResponsePackage.GetObject();
+
+ CbArrayView BlobHashArray = ResponseObject["blobHashes"sv].AsArrayView();
+ CbArrayView MetadatasArray = ResponseObject["metadatas"sv].AsArrayView();
+ ResultMetadatas.reserve(MetadatasArray.Num());
+ auto BlobHashesIt = BlobHashes.begin();
+ auto BlobHashArrayIt = begin(BlobHashArray);
+ auto MetadataArrayIt = begin(MetadatasArray);
+ while (MetadataArrayIt != end(MetadatasArray))
+ {
+ const IoHash BlobHash = (*BlobHashArrayIt).AsHash();
+ while (BlobHash != *BlobHashesIt)
+ {
+ ZEN_ASSERT(BlobHashesIt != BlobHashes.end());
+ BlobHashesIt++;
+ }
+
+ ZEN_ASSERT(BlobHash == *BlobHashesIt);
+
+ const IoHash MetaHash = (*MetadataArrayIt).AsAttachment();
+ const CbAttachment* MetaAttachment = ResponsePackage.FindAttachment(MetaHash);
+ ZEN_ASSERT(MetaAttachment);
+
+ CbObject Metadata = MetaAttachment->AsObject();
+ ResultMetadatas.emplace_back(std::move(Metadata));
+
+ BlobHashArrayIt++;
+ MetadataArrayIt++;
+ BlobHashesIt++;
+ }
+ return ResultMetadatas;
+ };
+
+ {
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ const size_t BlobCount = 5;
+
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I)));
+ Metadatas.push_back(MakeMetadata(BlobHashes.back(), {{"index", fmt::format("{}", I)}}));
+ MetadataHashes.push_back(IoHash::HashBuffer(Metadatas.back().GetBuffer().AsIoBuffer()));
+ }
+
+ {
+ CbPackage RequestPackage;
+ std::vector<CbAttachment> Attachments;
+ tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes;
+ Attachments.reserve(BlobCount);
+ AttachmentHashes.reserve(BlobCount);
+ {
+ CbObjectWriter RequestWriter;
+ RequestWriter.BeginArray("blobHashes");
+ for (size_t BlockHashIndex = 0; BlockHashIndex < BlobHashes.size(); BlockHashIndex++)
+ {
+ RequestWriter.AddHash(BlobHashes[BlockHashIndex]);
+ }
+ RequestWriter.EndArray(); // blobHashes
+
+ RequestWriter.BeginArray("metadatas");
+ for (size_t BlockHashIndex = 0; BlockHashIndex < BlobHashes.size(); BlockHashIndex++)
+ {
+ const IoHash ObjectHash = Metadatas[BlockHashIndex].GetHash();
+ RequestWriter.AddBinaryAttachment(ObjectHash);
+ if (!AttachmentHashes.contains(ObjectHash))
+ {
+ Attachments.push_back(CbAttachment(Metadatas[BlockHashIndex], ObjectHash));
+ AttachmentHashes.insert(ObjectHash);
+ }
+ }
+
+ RequestWriter.EndArray(); // metadatas
+
+ RequestPackage.SetObject(RequestWriter.Save());
+ }
+ RequestPackage.AddAttachments(Attachments);
+
+ CompositeBuffer RpcRequestBuffer = FormatPackageMessageBuffer(RequestPackage);
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/putBlobMetadata", Namespace, Bucket, BuildId),
+ RpcRequestBuffer,
+ ZenContentType::kCbPackage);
+ CHECK(Result);
+ }
+
+ {
+ std::vector<CbObject> ResultMetadatas = GetMetadatas(Client, Namespace, Bucket, BuildId, BlobHashes);
+
+ for (size_t Index = 0; Index < MetadataHashes.size(); Index++)
+ {
+ const IoHash& ExpectedHash = MetadataHashes[Index];
+ IoHash Hash = IoHash::HashBuffer(ResultMetadatas[Index].GetBuffer().AsIoBuffer());
+ CHECK_EQ(ExpectedHash, Hash);
+ }
+ }
+ }
+ {
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ std::vector<CbObject> ResultMetadatas = GetMetadatas(Client, Namespace, Bucket, BuildId, BlobHashes);
+
+ for (size_t Index = 0; Index < MetadataHashes.size(); Index++)
+ {
+ const IoHash& ExpectedHash = MetadataHashes[Index];
+ IoHash Hash = IoHash::HashBuffer(ResultMetadatas[Index].GetBuffer().AsIoBuffer());
+ CHECK_EQ(ExpectedHash, Hash);
+ }
+ }
+}
+
+TEST_CASE("buildstore.cache")
+{
+ std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir();
+ std::filesystem::path TempDir = TestEnv.CreateNewTestDir();
+ auto _ = MakeGuard([&SystemRootPath, &TempDir]() {
+ DeleteDirectories(SystemRootPath);
+ DeleteDirectories(TempDir);
+ });
+
+ std::string_view Namespace = "ns"sv;
+ std::string_view Bucket = "bkt"sv;
+ Oid BuildId = Oid::NewOid();
+
+ std::vector<IoHash> BlobHashes;
+ std::vector<CbObject> Metadatas;
+ std::vector<IoHash> MetadataHashes;
+
+ const size_t BlobCount = 5;
+ {
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri());
+
+ BuildStorageCache::Statistics Stats;
+ std::unique_ptr<BuildStorageCache> Cache(CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false));
+
+ {
+ IoHash NoneBlob = IoHash::HashBuffer("data", 4);
+ std::vector<BuildStorageCache::BlobExistsResult> NoneExists = Cache->BlobsExists(BuildId, std::vector<IoHash>{NoneBlob});
+ CHECK(NoneExists.size() == 1);
+ CHECK(!NoneExists[0].HasBody);
+ CHECK(!NoneExists[0].HasMetadata);
+ }
+
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7);
+ CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+ BlobHashes.push_back(CompressedBlob.DecodeRawHash());
+ Cache->PutBuildBlob(BuildId, BlobHashes.back(), ZenContentType::kCompressedBinary, CompressedBlob.GetCompressed());
+ }
+
+ Cache->Flush(500);
+ Cache = CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false);
+
+ {
+ std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
+ CHECK(Exists.size() == BlobHashes.size());
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ CHECK(Exists[I].HasBody);
+ CHECK(!Exists[I].HasMetadata);
+ }
+
+ std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
+ CHECK_EQ(0, FetchedMetadatas.size());
+ }
+
+ {
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ IoBuffer BuildBlob = Cache->GetBuildBlob(BuildId, BlobHashes[I]);
+ CHECK(BuildBlob);
+ CHECK_EQ(BlobHashes[I],
+ IoHash::HashBuffer(CompressedBuffer::FromCompressedNoValidate(std::move(BuildBlob)).Decompress().AsIoBuffer()));
+ }
+ }
+
+ {
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ CbObject Metadata = MakeMetadata(BlobHashes[I],
+ {{"key", fmt::format("{}", I)},
+ {"key_plus_one", fmt::format("{}", I + 1)},
+ {"block_hash", fmt::format("{}", BlobHashes[I])}});
+ Metadatas.push_back(Metadata);
+ MetadataHashes.push_back(IoHash::HashBuffer(Metadata.GetBuffer().AsIoBuffer()));
+ }
+ Cache->PutBlobMetadatas(BuildId, BlobHashes, Metadatas);
+ }
+
+ Cache->Flush(500);
+ Cache = CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false);
+
+ {
+ std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
+ CHECK(Exists.size() == BlobHashes.size());
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ CHECK(Exists[I].HasBody);
+ CHECK(Exists[I].HasMetadata);
+ }
+
+ std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
+ CHECK_EQ(BlobCount, FetchedMetadatas.size());
+
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ CHECK_EQ(MetadataHashes[I], IoHash::HashBuffer(FetchedMetadatas[I].GetBuffer().AsIoBuffer()));
+ }
+ }
+
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7);
+ CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+ BlobHashes.push_back(CompressedBlob.DecodeRawHash());
+ Cache->PutBuildBlob(BuildId, BlobHashes.back(), ZenContentType::kCompressedBinary, CompressedBlob.GetCompressed());
+ }
+
+ Cache->Flush(500);
+ Cache = CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false);
+
+ {
+ std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
+ CHECK(Exists.size() == BlobHashes.size());
+ for (size_t I = 0; I < BlobCount * 2; I++)
+ {
+ CHECK(Exists[I].HasBody);
+ CHECK_EQ(I < BlobCount, Exists[I].HasMetadata);
+ }
+
+ std::vector<CbObject> MetaDatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
+ CHECK_EQ(BlobCount, MetaDatas.size());
+
+ std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
+ CHECK_EQ(BlobCount, FetchedMetadatas.size());
+
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ CHECK_EQ(MetadataHashes[I], IoHash::HashBuffer(FetchedMetadatas[I].GetBuffer().AsIoBuffer()));
+ }
+ }
+ }
+
+ {
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri());
+
+ BuildStorageCache::Statistics Stats;
+ std::unique_ptr<BuildStorageCache> Cache(CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, false));
+
+ std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
+ CHECK(Exists.size() == BlobHashes.size());
+ for (size_t I = 0; I < BlobCount * 2; I++)
+ {
+ CHECK(Exists[I].HasBody);
+ CHECK_EQ(I < BlobCount, Exists[I].HasMetadata);
+ }
+
+ for (size_t I = 0; I < BlobCount * 2; I++)
+ {
+ IoBuffer BuildBlob = Cache->GetBuildBlob(BuildId, BlobHashes[I]);
+ CHECK(BuildBlob);
+ CHECK_EQ(BlobHashes[I],
+ IoHash::HashBuffer(CompressedBuffer::FromCompressedNoValidate(std::move(BuildBlob)).Decompress().AsIoBuffer()));
+ }
+
+ std::vector<CbObject> MetaDatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
+ CHECK_EQ(BlobCount, MetaDatas.size());
+
+ std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
+ CHECK_EQ(BlobCount, FetchedMetadatas.size());
+
+ for (size_t I = 0; I < BlobCount; I++)
+ {
+ CHECK_EQ(MetadataHashes[I], IoHash::HashBuffer(FetchedMetadatas[I].GetBuffer().AsIoBuffer()));
+ }
+ }
+}
+
# if 0
TEST_CASE("lifetime.owner")
{
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 73166e608..8c2e6d771 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -17,14 +17,11 @@
# include <mimalloc.h>
#endif
-#include <zenstore/cidstore.h>
#include <zenstore/gc.h>
-#include <zenstore/buildstore/buildstore.h>
#include <zenstore/cache/structuredcachestore.h>
#include <zenutil/workerpools.h>
#include "config.h"
-#include "projectstore/projectstore.h"
#include <chrono>
@@ -104,17 +101,13 @@ GetStatsForStateDirectory(std::filesystem::path StateDir)
HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
JobQueue& BackgroundJobQueue,
ZenCacheStore* CacheStore,
- CidStore* CidStore,
- ProjectStore* ProjectStore,
- BuildStore* BuildStore,
+ std::function<void()>&& FlushFunction,
const LogPaths& LogPaths,
const ZenServerOptions& ServerOptions)
: m_GcScheduler(Scheduler)
, m_BackgroundJobQueue(BackgroundJobQueue)
, m_CacheStore(CacheStore)
-, m_CidStore(CidStore)
-, m_ProjectStore(ProjectStore)
-, m_BuildStore(BuildStore)
+, m_FlushFunction(std::move(FlushFunction))
, m_LogPaths(LogPaths)
, m_ServerOptions(ServerOptions)
{
@@ -247,6 +240,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime));
Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime));
Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now));
+ Obj.AddInteger("ReturnCode", CurrentState->ReturnCode);
Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
}
break;
@@ -782,22 +776,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
"flush",
[this](HttpRouterRequest& Req) {
HttpServerRequest& HttpReq = Req.ServerRequest();
- if (m_CidStore)
- {
- m_CidStore->Flush();
- }
- if (m_CacheStore)
- {
- m_CacheStore->Flush();
- }
- if (m_ProjectStore)
- {
- m_ProjectStore->Flush();
- }
- if (m_BuildStore)
- {
- m_BuildStore->Flush();
- }
+ m_FlushFunction();
HttpReq.WriteResponse(HttpResponseCode::OK);
},
HttpVerb::kPost);
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index e7821dead..9a49f5120 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -4,15 +4,13 @@
#include <zencore/compactbinary.h>
#include <zenhttp/httpserver.h>
+#include <functional>
namespace zen {
class GcScheduler;
class JobQueue;
class ZenCacheStore;
-class CidStore;
-class ProjectStore;
-class BuildStore;
struct ZenServerOptions;
class HttpAdminService : public zen::HttpService
@@ -27,9 +25,7 @@ public:
HttpAdminService(GcScheduler& Scheduler,
JobQueue& BackgroundJobQueue,
ZenCacheStore* CacheStore,
- CidStore* CidStore,
- ProjectStore* ProjectStore,
- BuildStore* BuildStore,
+ std::function<void()>&& FlushFunction,
const LogPaths& LogPaths,
const ZenServerOptions& ServerOptions);
~HttpAdminService();
@@ -42,9 +38,7 @@ private:
GcScheduler& m_GcScheduler;
JobQueue& m_BackgroundJobQueue;
ZenCacheStore* m_CacheStore;
- CidStore* m_CidStore;
- ProjectStore* m_ProjectStore;
- BuildStore* m_BuildStore;
+ std::function<void()> m_FlushFunction;
LogPaths m_LogPaths;
const ZenServerOptions& m_ServerOptions;
};
diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp
index bcec74ce6..2a3ce41b7 100644
--- a/src/zenserver/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/buildstore/httpbuildstore.cpp
@@ -266,7 +266,7 @@ HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req)
BlobsArrayIt++;
MetadataArrayIt++;
}
- m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads);
+ m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads, &GetSmallWorkerPool(EWorkloadType::Burst));
return ServerRequest.WriteResponse(HttpResponseCode::OK);
}
@@ -484,7 +484,7 @@ HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req)
ResponseWriter.BeginArray("metadataExists"sv);
for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists)
{
- ResponseWriter.AddBool(BlobExists.HasBody);
+ ResponseWriter.AddBool(BlobExists.HasMetadata);
if (BlobExists.HasMetadata)
{
m_BuildStoreStats.BlobExistsMetaHitCount++;
@@ -529,23 +529,11 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats();
Cbo << "count" << StorageStats.EntryCount;
- Cbo << "bytes" << StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes + StorageStats.MetadataByteCount;
+ Cbo << "bytes" << StorageStats.BlobBytes + StorageStats.MetadataByteCount;
Cbo.BeginObject("blobs");
{
- Cbo << "count" << (StorageStats.LargeBlobCount + StorageStats.SmallBlobCount);
- Cbo << "bytes" << (StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes);
- Cbo.BeginObject("large");
- {
- Cbo << "count" << StorageStats.LargeBlobCount;
- Cbo << "bytes" << StorageStats.LargeBlobBytes;
- }
- Cbo.EndObject(); // large
- Cbo.BeginObject("small");
- {
- Cbo << "count" << StorageStats.SmallBlobCount;
- Cbo << "bytes" << StorageStats.SmallBlobBytes;
- }
- Cbo.EndObject(); // small
+ Cbo << "count" << StorageStats.BlobCount;
+ Cbo << "bytes" << StorageStats.BlobBytes;
}
Cbo.EndObject(); // blobs
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index bb0c55618..19ac3a216 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -85,7 +85,7 @@ namespace {
//////////////////////////////////////////////////////////////////////////
HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CidStore& InCidStore,
+ GetCidStoreFunc&& GetCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
UpstreamCache& UpstreamCache,
@@ -95,11 +95,10 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach
, m_CacheStore(InCacheStore)
, m_StatsService(StatsService)
, m_StatusService(StatusService)
-, m_CidStore(InCidStore)
, m_UpstreamCache(UpstreamCache)
, m_DiskWriteBlocker(InDiskWriteBlocker)
, m_OpenProcessCache(InOpenProcessCache)
-, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker)
+, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, std::move(GetCidStore), InDiskWriteBlocker)
{
m_StatsService.RegisterHandler("z$", *this);
m_StatusService.RegisterHandler("z$", *this);
@@ -131,24 +130,6 @@ HttpStructuredCacheService::Flush()
}
void
-HttpStructuredCacheService::ScrubStorage(ScrubContext& Ctx)
-{
- if (m_LastScrubTime == Ctx.ScrubTimestamp())
- {
- return;
- }
-
- ZenCacheStore::Info Info = m_CacheStore.GetInfo();
-
- ZEN_INFO("scrubbing '{}'", Info.BasePath);
-
- m_LastScrubTime = Ctx.ScrubTimestamp();
-
- m_CidStore.ScrubStorage(Ctx);
- m_CacheStore.ScrubStorage(Ctx);
-}
-
-void
HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
{
std::string_view Key = Request.RelativeUri();
@@ -243,6 +224,9 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
for (const auto& NamespaceIt : ValueDetails.Namespaces)
{
const std::string& Namespace = NamespaceIt.first;
+
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace);
+
for (const auto& BucketIt : NamespaceIt.second.Buckets)
{
const std::string& Bucket = BucketIt.first;
@@ -252,7 +236,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
{
for (const IoHash& Hash : ValueIt.second.Attachments)
{
- IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
CSVWriter << "\r\n"
<< Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString()
<< ", " << gsl::narrow<uint64_t>(Payload.GetSize());
@@ -270,7 +254,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
size_t AttachmentsSize = 0;
for (const IoHash& Hash : ValueIt.second.Attachments)
{
- IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
AttachmentsSize += Payload.GetSize();
}
CSVWriter << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
@@ -292,6 +276,9 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
for (const auto& NamespaceIt : ValueDetails.Namespaces)
{
const std::string& Namespace = NamespaceIt.first;
+
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace);
+
Cbo.BeginObject();
{
Cbo.AddString("name", Namespace);
@@ -334,7 +321,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
{
Cbo.BeginObject();
Cbo.AddHash("cid", Hash);
- IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
Cbo.AddInteger("size", gsl::narrow<uint64_t>(Payload.GetSize()));
Cbo.EndObject();
}
@@ -348,7 +335,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
size_t AttachmentsSize = 0;
for (const IoHash& Hash : ValueIt.second.Attachments)
{
- IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
AttachmentsSize += Payload.GetSize();
}
Cbo.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
@@ -623,6 +610,8 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount);
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName);
+
if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty())
{
ResponseWriter.BeginObject("BucketSizes");
@@ -681,7 +670,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
uint64_t AttachmentsSize = 0;
- m_CidStore.IterateChunks(
+ ChunkStore.IterateChunks(
AllAttachments,
[&](size_t Index, const IoBuffer& Payload) {
ZEN_UNUSED(Index);
@@ -749,6 +738,8 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount);
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName);
+
if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true")
{
CacheContentStats ContentStats;
@@ -775,7 +766,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background);
- m_CidStore.IterateChunks(
+ ChunkStore.IterateChunks(
ContentStats.Attachments,
[&](size_t Index, const IoBuffer& Payload) {
ZEN_UNUSED(Index);
@@ -850,6 +841,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()};
Stopwatch Timer;
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
+
if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) &&
m_CacheStore.Get(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue))
{
@@ -864,17 +857,17 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
uint32_t MissingCount = 0;
CbObjectView CacheRecord(ClientResultValue.Value.Data());
- CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) {
+ CacheRecord.IterateAttachments([this, &ChunkStore, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) {
if (SkipData)
{
- if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash()))
+ if (!ChunkStore.ContainsChunk(AttachmentHash.AsHash()))
{
MissingCount++;
}
}
else
{
- if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
+ if (IoBuffer Chunk = ChunkStore.FindChunkByCid(AttachmentHash.AsHash()))
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
if (Compressed)
@@ -974,6 +967,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
{
Success = true;
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
+
ClientResultValue.Value = UpstreamResult.Value;
ClientResultValue.Value.SetContentType(AcceptType);
@@ -997,8 +992,14 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
if (Success && StoreLocal)
{
- m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, nullptr);
- m_CacheStats.WriteCount++;
+ const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal);
+ ZenCacheStore::PutResult PutResult =
+ m_CacheStore
+ .Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, Overwrite, nullptr);
+ if (PutResult.Status == zen::PutStatus::Success)
+ {
+ m_CacheStats.WriteCount++;
+ }
}
}
else if (AcceptType == ZenContentType::kCbPackage)
@@ -1018,6 +1019,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
CacheRecord.IterateAttachments([this,
&Package,
&Ref,
+ &ChunkStore,
&WriteAttachmentBuffers,
&WriteRawHashes,
&ReferencedAttachments,
@@ -1052,12 +1054,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
{
if (SkipData)
{
- if (m_CidStore.ContainsChunk(Hash))
+ if (ChunkStore.ContainsChunk(Hash))
{
Count.Valid++;
}
}
- else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash))
+ else if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Hash))
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
if (Compressed)
@@ -1083,30 +1085,35 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
if (StoreLocal)
{
- m_CacheStore.Put(RequestContext,
- Ref.Namespace,
- Ref.BucketSegment,
- Ref.HashKey,
- CacheValue,
- ReferencedAttachments,
- nullptr);
- m_CacheStats.WriteCount++;
-
- if (!WriteAttachmentBuffers.empty())
+ const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal);
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext,
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ CacheValue,
+ ReferencedAttachments,
+ Overwrite,
+ nullptr);
+ if (PutResult.Status == zen::PutStatus::Success)
{
- std::vector<CidStore::InsertResult> InsertResults =
- m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
- for (const CidStore::InsertResult& Result : InsertResults)
+ m_CacheStats.WriteCount++;
+
+ if (!WriteAttachmentBuffers.empty())
{
- if (Result.New)
+ std::vector<CidStore::InsertResult> InsertResults =
+ ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
+ for (const CidStore::InsertResult& Result : InsertResults)
{
- Count.New++;
+ if (Result.New)
+ {
+ Count.New++;
+ }
}
}
- }
- WriteAttachmentBuffers = {};
- WriteRawHashes = {};
+ WriteAttachmentBuffers = {};
+ WriteRawHashes = {};
+ }
}
BinaryWriter MemStream;
@@ -1197,6 +1204,24 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
return Request.WriteResponse(HttpResponseCode::InsufficientStorage);
}
+ auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) {
+ ZEN_UNUSED(PutResult);
+
+ HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError;
+ switch (PutResult.Status)
+ {
+ case zen::PutStatus::Conflict:
+ ResponseCode = HttpResponseCode::Conflict;
+ break;
+ case zen::PutStatus::Invalid:
+ ResponseCode = HttpResponseCode::BadRequest;
+ break;
+ }
+
+ return PutResult.Message.empty() ? Request.WriteResponse(ResponseCode)
+ : Request.WriteResponse(ResponseCode, zen::HttpContentType::kText, PutResult.Message);
+ };
+
const HttpContentType ContentType = Request.RequestContentType();
Body.SetContentType(ContentType);
@@ -1225,18 +1250,28 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
{
RawHash = IoHash::HashBuffer(SharedBuffer(Body));
}
- m_CacheStore.Put(RequestContext,
- Ref.Namespace,
- Ref.BucketSegment,
- Ref.HashKey,
- {.Value = Body, .RawSize = RawSize, .RawHash = RawHash},
- {},
- nullptr);
+ const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal);
+ // TODO: Propagation for rejected PUTs
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext,
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ {.Value = Body, .RawSize = RawSize, .RawHash = RawHash},
+ {},
+ Overwrite,
+ nullptr);
+ if (PutResult.Status != zen::PutStatus::Success)
+ {
+ return WriteFailureResponse(PutResult);
+ }
m_CacheStats.WriteCount++;
if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote))
{
- m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}});
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
+ m_UpstreamCache.EnqueueUpstream(
+ {.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}},
+ [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
}
ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' in {}",
@@ -1270,17 +1305,34 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
std::vector<IoHash> ReferencedAttachments;
int32_t TotalCount = 0;
- CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) {
- const IoHash Hash = AttachmentHash.AsHash();
- ReferencedAttachments.push_back(Hash);
- if (m_CidStore.ContainsChunk(Hash))
- {
- ValidAttachments.emplace_back(Hash);
- }
- TotalCount++;
- });
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
- m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}, ReferencedAttachments, nullptr);
+ CacheRecord.IterateAttachments(
+ [this, &ChunkStore, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) {
+ const IoHash Hash = AttachmentHash.AsHash();
+ ReferencedAttachments.push_back(Hash);
+ if (ChunkStore.ContainsChunk(Hash))
+ {
+ ValidAttachments.emplace_back(Hash);
+ }
+ TotalCount++;
+ });
+
+ const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal);
+
+ // TODO: Propagation for rejected PUTs
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext,
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ {.Value = Body},
+ ReferencedAttachments,
+ Overwrite,
+ nullptr);
+ if (PutResult.Status != zen::PutStatus::Success)
+ {
+ return WriteFailureResponse(PutResult);
+ }
m_CacheStats.WriteCount++;
ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}",
@@ -1298,10 +1350,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
CachePolicy Policy = PolicyFromUrl;
if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
- m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject,
- .Namespace = Ref.Namespace,
- .Key = {Ref.BucketSegment, Ref.HashKey},
- .ValueContentIds = std::move(ValidAttachments)});
+ m_UpstreamCache.EnqueueUpstream(
+ {.Type = ZenContentType::kCbObject,
+ .Namespace = Ref.Namespace,
+ .Key = {Ref.BucketSegment, Ref.HashKey},
+ .ValueContentIds = std::move(ValidAttachments)},
+ [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -1334,38 +1388,46 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
WriteAttachmentBuffers.reserve(NumAttachments);
WriteRawHashes.reserve(NumAttachments);
- CacheRecord.IterateAttachments(
- [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count](
- CbFieldView HashView) {
- const IoHash Hash = HashView.AsHash();
- ReferencedAttachments.push_back(Hash);
- if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
- {
- if (Attachment->IsCompressedBinary())
- {
- WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer());
- WriteRawHashes.push_back(Hash);
- ValidAttachments.emplace_back(Hash);
- Count.Valid++;
- }
- else
- {
- ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
- Ref.Namespace,
- Ref.BucketSegment,
- Ref.HashKey,
- ToString(HttpContentType::kCbPackage),
- Hash);
- Count.Invalid++;
- }
- }
- else if (m_CidStore.ContainsChunk(Hash))
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
+
+ CacheRecord.IterateAttachments([this,
+ &Ref,
+ &Package,
+ &ChunkStore,
+ &WriteAttachmentBuffers,
+ &WriteRawHashes,
+ &ValidAttachments,
+ &ReferencedAttachments,
+ &Count](CbFieldView HashView) {
+ const IoHash Hash = HashView.AsHash();
+ ReferencedAttachments.push_back(Hash);
+ if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
+ {
+ if (Attachment->IsCompressedBinary())
{
+ WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer());
+ WriteRawHashes.push_back(Hash);
ValidAttachments.emplace_back(Hash);
Count.Valid++;
}
- Count.Total++;
- });
+ else
+ {
+ ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ ToString(HttpContentType::kCbPackage),
+ Hash);
+ Count.Invalid++;
+ }
+ }
+ else if (ChunkStore.ContainsChunk(Hash))
+ {
+ ValidAttachments.emplace_back(Hash);
+ Count.Valid++;
+ }
+ Count.Total++;
+ });
if (Count.Invalid > 0)
{
@@ -1373,15 +1435,23 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv);
}
+ const bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal);
+
ZenCacheValue CacheValue;
CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
CacheValue.Value.SetContentType(ZenContentType::kCbObject);
- m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments);
+ // TODO: Propagation for rejected PUTs
+ ZenCacheStore::PutResult PutResult =
+ m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments, Overwrite);
+ if (PutResult.Status != zen::PutStatus::Success)
+ {
+ return WriteFailureResponse(PutResult);
+ }
m_CacheStats.WriteCount++;
if (!WriteAttachmentBuffers.empty())
{
- std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
+ std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
for (const CidStore::InsertResult& InsertResult : InsertResults)
{
if (InsertResult.New)
@@ -1408,10 +1478,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
- m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage,
- .Namespace = Ref.Namespace,
- .Key = {Ref.BucketSegment, Ref.HashKey},
- .ValueContentIds = std::move(ValidAttachments)});
+ m_UpstreamCache.EnqueueUpstream(
+ {.Type = ZenContentType::kCbPackage,
+ .Namespace = Ref.Namespace,
+ .Key = {Ref.BucketSegment, Ref.HashKey},
+ .ValueContentIds = std::move(ValidAttachments)},
+ [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -1445,7 +1517,9 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons
{
Stopwatch Timer;
- IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId);
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
+
+ IoBuffer Value = ChunkStore.FindChunkByCid(Ref.ValueContentId);
const UpstreamEndpointInfo* Source = nullptr;
CachePolicy Policy = PolicyFromUrl;
@@ -1467,7 +1541,7 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons
{
if (AreDiskWritesAllowed())
{
- m_CidStore.AddChunk(UpstreamResult.Value, RawHash);
+ ChunkStore.AddChunk(UpstreamResult.Value, RawHash);
}
Source = UpstreamResult.Source;
}
@@ -1561,7 +1635,9 @@ HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, cons
"ValueContentId does not match attachment hash"sv);
}
- CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash);
+ CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
+
+ CidStore::InsertResult Result = ChunkStore.AddChunk(Body, RawHash);
ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}",
Ref.Namespace,
@@ -1776,16 +1852,51 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
EmitSnapshot("requests", m_HttpRequests, Cbo);
- const uint64_t HitCount = m_CacheStats.HitCount;
- const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
- const uint64_t MissCount = m_CacheStats.MissCount;
- const uint64_t WriteCount = m_CacheStats.WriteCount;
- const uint64_t BadRequestCount = m_CacheStats.BadRequestCount;
- struct CidStoreStats StoreStats = m_CidStore.Stats();
- const uint64_t ChunkHitCount = StoreStats.HitCount;
- const uint64_t ChunkMissCount = StoreStats.MissCount;
- const uint64_t ChunkWriteCount = StoreStats.WriteCount;
- const uint64_t TotalCount = HitCount + MissCount;
+ const uint64_t HitCount = m_CacheStats.HitCount;
+ const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+ const uint64_t MissCount = m_CacheStats.MissCount;
+ const uint64_t WriteCount = m_CacheStats.WriteCount;
+ const uint64_t BadRequestCount = m_CacheStats.BadRequestCount;
+
+ uint64_t TotalChunkHitCount = 0;
+ uint64_t TotalChunkMissCount = 0;
+ uint64_t TotalChunkWriteCount = 0;
+ CidStoreSize TotalCidSize;
+
+ tsl::robin_map<CidStore*, std::string> UniqueStores;
+ {
+ std::vector<std::string> NamespaceNames = m_CacheStore.GetNamespaces();
+
+ for (const std::string& NamespaceName : NamespaceNames)
+ {
+ CidStore* Store = &m_RpcHandler.GetCidStore(NamespaceName);
+ if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
+ {
+ UniqueStores.insert_or_assign(Store, NamespaceName);
+ }
+ else
+ {
+ UniqueStores.insert_or_assign(Store, std::string{});
+ }
+ }
+
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+
+ CidStoreStats StoreStats = ChunkStore->Stats();
+ CidStoreSize StoreSize = ChunkStore->TotalSize();
+
+ TotalChunkHitCount += StoreStats.HitCount;
+ TotalChunkMissCount += StoreStats.MissCount;
+ TotalChunkWriteCount += StoreStats.WriteCount;
+
+ TotalCidSize.TinySize += StoreSize.TinySize;
+ TotalCidSize.SmallSize += StoreSize.SmallSize;
+ TotalCidSize.LargeSize += StoreSize.LargeSize;
+ TotalCidSize.TotalSize += StoreSize.TotalSize;
+ }
+ }
const uint64_t RpcRequests = m_CacheStats.RpcRequests;
const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests;
@@ -1795,17 +1906,11 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests;
const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests;
- const CidStoreSize CidSize = m_CidStore.TotalSize();
- const GcStorageSize CacheSize = m_CacheStore.StorageSize();
+ const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true";
bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
- CidStoreStats CidStoreStats = {};
- if (ShowCidStoreStats)
- {
- CidStoreStats = m_CidStore.Stats();
- }
ZenCacheStore::CacheStoreStats CacheStoreStats = {};
if (ShowCacheStoreStats)
{
@@ -1840,6 +1945,7 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
Cbo.EndObject();
Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
+ const uint64_t TotalCount = HitCount + MissCount;
Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
if (m_UpstreamCache.IsActive())
@@ -1850,7 +1956,9 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
}
- Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+ Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
+ const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
+ Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
if (ShowCacheStoreStats)
{
@@ -1959,20 +2067,58 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
{
Cbo.BeginObject("size");
{
- Cbo << "tiny" << CidSize.TinySize;
- Cbo << "small" << CidSize.SmallSize;
- Cbo << "large" << CidSize.LargeSize;
- Cbo << "total" << CidSize.TotalSize;
+ Cbo << "tiny" << TotalCidSize.TinySize;
+ Cbo << "small" << TotalCidSize.SmallSize;
+ Cbo << "large" << TotalCidSize.LargeSize;
+ Cbo << "total" << TotalCidSize.TotalSize;
}
Cbo.EndObject();
if (ShowCidStoreStats)
{
Cbo.BeginObject("store");
- Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount;
- EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo);
- EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo);
- // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo);
+
+ auto OutputStats = [&](CidStore& ChunkStore) {
+ CidStoreStats StoreStats = ChunkStore.Stats();
+ Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
+ const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
+ Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
+ EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
+ EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
+ };
+
+ if (UniqueStores.size() > 1)
+ {
+ Cbo.BeginArray("namespaces");
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+ const std::string& Namespace = It.second;
+ CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
+ Cbo.BeginObject();
+ {
+ Cbo << "namespace" << Namespace;
+ Cbo.BeginObject("stats");
+ OutputStats(*ChunkStore);
+ Cbo.EndObject();
+
+ Cbo.BeginObject("size");
+ {
+ Cbo << "tiny" << ChunkStoreSize.TinySize;
+ Cbo << "small" << ChunkStoreSize.SmallSize;
+ Cbo << "large" << ChunkStoreSize.LargeSize;
+ Cbo << "total" << ChunkStoreSize.TotalSize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndArray(); // namespaces
+ }
+ else if (UniqueStores.size() != 0)
+ {
+ OutputStats(*UniqueStores.begin()->first);
+ }
Cbo.EndObject();
}
}
diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h
index 13c1d6475..d46ca145d 100644
--- a/src/zenserver/cache/httpstructuredcache.h
+++ b/src/zenserver/cache/httpstructuredcache.h
@@ -70,8 +70,10 @@ namespace cache {
class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider
{
public:
+ typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
+
HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CidStore& InCidStore,
+ GetCidStoreFunc&& GetCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
UpstreamCache& UpstreamCache,
@@ -83,7 +85,6 @@ public:
virtual void HandleRequest(HttpServerRequest& Request) override;
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
private:
struct CacheRef
@@ -116,9 +117,7 @@ private:
ZenCacheStore& m_CacheStore;
HttpStatsService& m_StatsService;
HttpStatusService& m_StatusService;
- CidStore& m_CidStore;
UpstreamCache& m_UpstreamCache;
- uint64_t m_LastScrubTime = 0;
metrics::OperationTiming m_HttpRequests;
metrics::OperationTiming m_UpstreamGetRequestTiming;
CacheStats m_CacheStats;
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 1f9ae5fb6..23bb3ad78 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -342,6 +342,7 @@ public:
Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment));
Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment));
+ Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites));
}
Writer.EndContainer();
}
@@ -397,6 +398,8 @@ public:
}
BucketConfig.LargeObjectThreshold = LargeObjectThreshold;
+ BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites);
+
Value.push_back(std::make_pair(std::move(Name), BucketConfig));
}
}
@@ -542,6 +545,9 @@ ParseConfigFile(const std::filesystem::path& Path,
LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv,
ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold,
"cache-bucket-largeobjectthreshold"sv);
+ LuaOptions.AddOption("cache.bucket.limitoverwrites"sv,
+ ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites,
+ "cache-bucket-limit-overwrites"sv);
////// cache.upstream
LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv);
@@ -1120,6 +1126,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"),
"");
+ options.add_option("cache",
+ "",
+ "cache-bucket-limit-overwrites",
+ "Whether to require policy flag pattern before allowing overwrites in cache bucket",
+ cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"),
+ "");
+
options.add_option("gc",
"",
"gc-cache-attachment-store",
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 9753e3ae2..4a022a807 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -125,6 +125,7 @@ struct ZenStructuredCacheBucketConfig
uint32_t PayloadAlignment = 1u << 4;
uint64_t MemCacheSizeThreshold = 1 * 1024;
uint64_t LargeObjectThreshold = 128 * 1024;
+ bool LimitOverwrites = false;
};
struct ZenStructuredCacheConfig
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index 5778fa3d2..1de4c3d74 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
Binary files differ
diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js
index 69ee234fa..c0cbb7e11 100644
--- a/src/zenserver/frontend/html/indexer/worker.js
+++ b/src/zenserver/frontend/html/indexer/worker.js
@@ -73,33 +73,38 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride)
else if (field.is_named("packagedata")) pkg_data = field;
else if (field.is_named("bulkdata")) bulk_data = field;
}
- if (key == undefined || pkg_data == undefined)
+
+ if (key == undefined)
continue;
var id = 0n;
var size = 0n;
var raw_size = 0n;
- for (const item of pkg_data.as_array())
- {
- var found = 0, pkg_id = undefined;
- for (const field of item.as_object())
+
+ if (pkg_data)
+ {
+ for (const item of pkg_data.as_array())
{
- if (!id && field.is_named("id")) pkg_id = field.as_value();
- else if (field.is_named("size")) size += field.as_value();
- else if (field.is_named("rawsize")) raw_size += field.as_value();
- else continue;
- if (found++ >= 3)
- break;
- }
+ var found = 0, pkg_id = undefined;
+ for (const field of item.as_object())
+ {
+ if (!id && field.is_named("id")) pkg_id = field.as_value();
+ else if (field.is_named("size")) size += field.as_value();
+ else if (field.is_named("rawsize")) raw_size += field.as_value();
+ else continue;
+ if (found++ >= 3)
+ break;
+ }
- if (pkg_id === undefined)
- continue;
+ if (pkg_id === undefined)
+ continue;
- pkg_id = pkg_id.subarray(0, 8);
- for (var i = 7; i >= 0; --i)
- {
- id <<= 8n;
- id |= BigInt(pkg_id[i]);
+ pkg_id = pkg_id.subarray(0, 8);
+ for (var i = 7; i >= 0; --i)
+ {
+ id <<= 8n;
+ id |= BigInt(pkg_id[i]);
+ }
}
}
@@ -119,9 +124,6 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride)
}
}
- if (id == 0)
- continue;
-
result[count] = [id, key.as_value(), size, raw_size];
count++;
}
diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js
index 54fb11c18..08589b090 100644
--- a/src/zenserver/frontend/html/pages/entry.js
+++ b/src/zenserver/frontend/html/pages/entry.js
@@ -239,30 +239,34 @@ export class Page extends ZenPage
_convert_legacy_to_tree(entry)
{
- const pkg_data = entry.find("packagedata");
- if (pkg_data == undefined)
- return
+ const raw_pkgst_entry = entry.find("packagestoreentry");
+ if (raw_pkgst_entry == undefined) //if there is no packagestorentry then don't show the fancy webpage, just show the raw json
+ return;
const tree = {};
- var id = 0n;
- for (var item of pkg_data.as_array())
+ const pkg_data = entry.find("packagedata");
+ if (pkg_data)
{
- var pkg_id = item.as_object().find("id");
- if (pkg_id == undefined)
- continue;
-
- pkg_id = pkg_id.as_value().subarray(0, 8);
- for (var i = 7; i >= 0; --i)
+ var id = 0n;
+ for (var item of pkg_data.as_array())
{
- id <<= 8n;
- id |= BigInt(pkg_id[i]);
+ var pkg_id = item.as_object().find("id");
+ if (pkg_id == undefined)
+ continue;
+
+ pkg_id = pkg_id.as_value().subarray(0, 8);
+ for (var i = 7; i >= 0; --i)
+ {
+ id <<= 8n;
+ id |= BigInt(pkg_id[i]);
+ }
+ break;
}
- break;
+ tree["$id"] = id;
}
- tree["$id"] = id;
- const pkgst_entry = entry.find("packagestoreentry").as_object();
+ const pkgst_entry = raw_pkgst_entry.as_object();
for (const field of pkgst_entry)
{
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 76e022017..4ea4ee87e 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -167,6 +167,14 @@ ZenEntryPoint::Run()
ZEN_INFO("ServerState.Sweep()");
ServerState.Sweep();
+ auto NotifyReady = [&] {
+ if (!m_ServerOptions.ChildId.empty())
+ {
+ NamedEvent ParentEvent{m_ServerOptions.ChildId};
+ ParentEvent.Set();
+ }
+ };
+
uint32_t AttachSponsorProcessRetriesLeft = 3;
ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(m_ServerOptions.BasePort);
while (Entry)
@@ -205,6 +213,7 @@ ZenEntryPoint::Run()
// Sponsor processes are checked every second, so 2 second wait time should be enough
if (Entry->AddSponsorProcess(m_ServerOptions.OwnerPid, 2000))
{
+ NotifyReady();
std::exit(0);
}
if (AttachSponsorProcessRetriesLeft-- > 0)
@@ -350,14 +359,8 @@ ZenEntryPoint::Run()
Server.SetIsReadyFunc([&] {
m_LockFile.Update(MakeLockData(true), Ec);
-
- if (!m_ServerOptions.ChildId.empty())
- {
- NamedEvent ParentEvent{m_ServerOptions.ChildId};
- ParentEvent.Set();
- }
-
ReportServiceStatus(ServiceStatus::Running);
+ NotifyReady();
});
Server.Run();
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index ab96ae92d..52cdc5983 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -3,14 +3,12 @@
#include "buildsremoteprojectstore.h"
#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryvalidation.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
#include <zenhttp/httpclientauth.h>
-
-#include <zenutil/jupiter/jupiterclient.h>
-#include <zenutil/jupiter/jupitersession.h>
+#include <zenutil/jupiter/jupiterbuildstorage.h>
namespace zen {
@@ -21,20 +19,24 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv;
class BuildsRemoteStore : public RemoteProjectStore
{
public:
- BuildsRemoteStore(Ref<JupiterClient>&& InJupiterClient,
- std::string_view Namespace,
- std::string_view Bucket,
- const Oid& BuildId,
- const IoBuffer& MetaData,
- bool ForceDisableBlocks,
- bool ForceDisableTempBlocks,
- const std::filesystem::path& TempFilePath)
- : m_JupiterClient(std::move(InJupiterClient))
+ BuildsRemoteStore(std::unique_ptr<BuildStorage::Statistics>&& BuildStorageStats,
+ std::unique_ptr<HttpClient>&& BuildStorageHttp,
+ std::unique_ptr<BuildStorage>&& BuildStorage,
+ std::string_view Url,
+ std::string_view Namespace,
+ std::string_view Bucket,
+ const Oid& BuildId,
+ const IoBuffer& MetaData,
+ bool ForceDisableBlocks,
+ bool ForceDisableTempBlocks)
+ : m_BuildStorageStats(std::move(BuildStorageStats))
+ , m_BuildStorageHttp(std::move(BuildStorageHttp))
+ , m_BuildStorage(std::move(BuildStorage))
+ , m_Url(Url)
, m_Namespace(Namespace)
, m_Bucket(Bucket)
, m_BuildId(BuildId)
, m_MetaData(MetaData)
- , m_TempFilePath(TempFilePath)
, m_EnableBlocks(!ForceDisableBlocks)
, m_UseTempBlocks(!ForceDisableTempBlocks)
{
@@ -47,63 +49,91 @@ public:
.UseTempBlockFiles = m_UseTempBlocks,
.AllowChunking = true,
.ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId),
- .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)};
+ .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_Url, m_Namespace, m_Bucket, m_BuildId)};
}
virtual Stats GetStats() const override
{
- return {.m_SentBytes = m_SentBytes.load(),
- .m_ReceivedBytes = m_ReceivedBytes.load(),
- .m_RequestTimeNS = m_RequestTimeNS.load(),
- .m_RequestCount = m_RequestCount.load(),
- .m_PeakSentBytes = m_PeakSentBytes.load(),
- .m_PeakReceivedBytes = m_PeakReceivedBytes.load(),
- .m_PeakBytesPerSec = m_PeakBytesPerSec.load()};
+ return {
+ .m_SentBytes = m_BuildStorageStats->TotalBytesWritten.load(),
+ .m_ReceivedBytes = m_BuildStorageStats->TotalBytesRead.load(),
+ .m_RequestTimeNS = m_BuildStorageStats->TotalRequestTimeUs.load() * 1000,
+ .m_RequestCount = m_BuildStorageStats->TotalRequestCount.load(),
+ .m_PeakSentBytes = m_BuildStorageStats->PeakSentBytes.load(),
+ .m_PeakReceivedBytes = m_BuildStorageStats->PeakReceivedBytes.load(),
+ .m_PeakBytesPerSec = m_BuildStorageStats->PeakBytesPerSec.load(),
+ };
}
virtual CreateContainerResult CreateContainer() override
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
-
- IoBuffer Payload = m_MetaData;
- Payload.SetContentType(ZenContentType::kCbObject);
- JupiterResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload);
- AddStats(PutResult);
+ CreateContainerResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
- CreateContainerResult Result{ConvertResult(PutResult)};
- if (Result.ErrorCode)
+ CbObject Payload = LoadCompactBinaryObject(m_MetaData);
+ try
{
- Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- Result.Reason);
+ CbObject PutBuildResult = m_BuildStorage->PutBuild(m_BuildId, Payload);
+ ZEN_UNUSED(PutBuildResult);
+ m_OplogBuildPartId = Oid::NewOid();
}
- m_OplogBuildPartId = Oid::NewOid();
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason =
+ fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason =
+ fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+ }
+
return Result;
}
virtual SaveResult SaveContainer(const IoBuffer& Payload) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- PutBuildPartResult PutResult =
- Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload);
- AddStats(PutResult);
- SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash};
- if (Result.ErrorCode)
+ SaveResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
{
- Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- m_OplogBuildPartId,
- Result.Reason);
+ CbObject ObjectPayload = LoadCompactBinaryObject(Payload);
+
+ std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
+ m_BuildStorage->PutBuildPart(m_BuildId, m_OplogBuildPartId, OplogContainerPartName, ObjectPayload);
+ Result.RawHash = PutBuildPartResult.first;
+ Result.Needs = std::unordered_set<IoHash, IoHash::Hasher>(PutBuildPartResult.second.begin(), PutBuildPartResult.second.end());
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
}
return Result;
@@ -114,52 +144,84 @@ public:
ChunkBlockDescription&& Block) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult PutResult =
- Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
- AddStats(PutResult);
+ SaveAttachmentResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
- SaveAttachmentResult Result{ConvertResult(PutResult)};
- if (Result.ErrorCode)
+ try
{
- Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- RawHash,
- Result.Reason);
- return Result;
- }
+ m_BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
- if (Block.BlockHash == RawHash)
- {
- CbObjectWriter BlockMetaData;
- BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string());
-
- IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer();
- MetaPayload.SetContentType(ZenContentType::kCbObject);
- JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, RawHash, MetaPayload);
- AddStats(PutMetaResult);
- RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult);
- if (MetaDataResult.ErrorCode)
+ if (Block.BlockHash == RawHash)
{
- ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- RawHash,
- MetaDataResult.Reason);
+ try
+ {
+ CbObjectWriter BlockMetaData;
+ BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string());
+ CbObject MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save());
+ if (!m_BuildStorage->PutBlockMetadata(m_BuildId, RawHash, MetaPayload))
+ {
+ ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ RawHash,
+ "not found");
+ }
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
}
}
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+
return Result;
}
virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override
{
SaveAttachmentsResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
for (const SharedBuffer& Chunk : Chunks)
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer());
@@ -177,38 +239,68 @@ public:
ZEN_UNUSED(RawHash);
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- FinalizeBuildPartResult FinalizeRefResult =
- Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash);
- AddStats(FinalizeRefResult);
+ FinalizeResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
- FinalizeResult Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}};
- if (Result.ErrorCode)
+ try
+ {
+ std::vector<IoHash> Needs = m_BuildStorage->FinalizeBuildPart(m_BuildId, m_OplogBuildPartId, RawHash);
+ Result.Needs = std::unordered_set<IoHash, IoHash::Hasher>(Needs.begin(), Needs.end());
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0;
+ Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
{
- Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- m_OplogBuildPartId,
- Result.Reason);
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
}
- else if (Result.Needs.empty())
+
+ if (!Result.ErrorCode && Result.Needs.empty())
{
- JupiterResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId);
- AddStats(FinalizeBuildResult);
- FinalizeBuildResult.ElapsedSeconds += FinalizeRefResult.ElapsedSeconds;
- Result = {ConvertResult(FinalizeBuildResult)};
- if (Result.ErrorCode)
+ try
{
- Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- FinalizeBuildResult.Reason);
+ m_BuildStorage->FinalizeBuild(m_BuildId);
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error
+ : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode
+ : 0;
+ Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
}
}
+
return Result;
}
@@ -216,161 +308,128 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
- AddStats(GetBuildResult);
- LoadContainerResult Result{ConvertResult(GetBuildResult)};
- if (Result.ErrorCode)
- {
- Result.Reason = fmt::format("Failed fetching oplog container build from {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- Result.Reason);
- return Result;
- }
- CbObject BuildObject = LoadCompactBinaryObject(GetBuildResult.Response);
- if (!BuildObject)
- {
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build {}/{}/{}/{} payload is not formatted as a compact binary object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
- }
- CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
- if (!PartsObject)
+ LoadContainerResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
+ CbObject BuildObject = m_BuildStorage->GetBuild(m_BuildId);
+
+ CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
+ if (!PartsObject)
+ {
+ throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId));
+ }
+ m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId();
+ if (m_OplogBuildPartId == Oid::Zero)
+ {
+ throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ OplogContainerPartName));
+ }
+
+ Result.ContainerObject = m_BuildStorage->GetBuildPart(m_BuildId, m_OplogBuildPartId);
}
- m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId();
- if (m_OplogBuildPartId == Oid::Zero)
+ catch (const HttpClientError& Ex)
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
- m_JupiterClient->ServiceUrl(),
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
m_Namespace,
m_Bucket,
m_BuildId,
- OplogContainerPartName);
- return Result;
+ Ex.what());
}
-
- JupiterResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
- AddStats(GetBuildPartResult);
- Result = {ConvertResult(GetBuildResult)};
- Result.ElapsedSeconds += GetBuildResult.ElapsedSeconds;
- if (Result.ErrorCode)
+ catch (const std::exception& Ex)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("Failed fetching oplog build part from {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
+ Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
- Result.Reason);
- return Result;
+ Ex.what());
}
- CbObject ContainerObject = LoadCompactBinaryObject(GetBuildPartResult.Response);
- if (!ContainerObject)
- {
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build part for oplog container {}/{}/{}/{}/{} is not formatted as a compact binary object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- m_OplogBuildPartId);
- return Result;
- }
- Result.ContainerObject = std::move(ContainerObject);
return Result;
}
virtual GetKnownBlocksResult GetKnownBlocks() override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1);
- AddStats(FindResult);
- GetKnownBlocksResult Result{ConvertResult(FindResult)};
- if (Result.ErrorCode)
+
+ GetKnownBlocksResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- Result.Reason);
- return Result;
+ CbObject KnownBlocks = m_BuildStorage->FindBlocks(m_BuildId, 10000u);
+ std::optional<std::vector<ChunkBlockDescription>> Blocks = ParseChunkBlockDescriptionList(KnownBlocks);
+ Result.Blocks.reserve(Blocks.value().size());
+ for (ChunkBlockDescription& BlockDescription : Blocks.value())
+ {
+ Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash,
+ .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)});
+ }
}
- if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None)
+ catch (const HttpClientError& Ex)
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a compact binary object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
}
- std::optional<std::vector<ChunkBlockDescription>> Blocks =
- ParseChunkBlockDescriptionList(LoadCompactBinaryObject(FindResult.Response));
- if (!Blocks)
+ catch (const std::exception& Ex)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a list of blocks"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
- }
- Result.Blocks.reserve(Blocks.value().size());
- for (ChunkBlockDescription& BlockDescription : Blocks.value())
- {
- Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash,
- .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)});
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
}
+
return Result;
}
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath);
- AddStats(GetResult);
- LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
- if (GetResult.ErrorCode)
+ LoadAttachmentResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
+ {
+ Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash);
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+ }
+ catch (const std::exception& Ex)
{
- Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- RawHash,
- Result.Reason);
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
}
+
return Result;
}
virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override
{
LoadAttachmentsResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
for (const IoHash& Hash : RawHashes)
{
LoadAttachmentResult ChunkResult = LoadAttachment(Hash);
@@ -386,81 +445,27 @@ public:
}
private:
- void AddStats(const JupiterResult& Result)
- {
- m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes));
- m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes));
- m_RequestTimeNS.fetch_add(static_cast<uint64_t>(Result.ElapsedSeconds * 1000000000));
- SetAtomicMax(m_PeakSentBytes, Result.SentBytes);
- SetAtomicMax(m_PeakReceivedBytes, Result.ReceivedBytes);
- if (Result.ElapsedSeconds > 0.0)
- {
- uint64_t BytesPerSec = static_cast<uint64_t>((Result.SentBytes + Result.ReceivedBytes) / Result.ElapsedSeconds);
- SetAtomicMax(m_PeakBytesPerSec, BytesPerSec);
- }
-
- m_RequestCount.fetch_add(1);
- }
-
- static Result ConvertResult(const JupiterResult& Response)
+ static int MakeErrorCode(const HttpClientError& Ex)
{
- std::string Text;
- int32_t ErrorCode = 0;
- if (Response.ErrorCode != 0 || !Response.Success)
- {
- if (Response.Response)
- {
- HttpContentType ContentType = Response.Response.GetContentType();
- if (ContentType == ZenContentType::kText || ContentType == ZenContentType::kJSON)
- {
- ExtendableStringBuilder<256> SB;
- SB.Append("\n");
- SB.Append(std::string_view(reinterpret_cast<const std::string::value_type*>(Response.Response.GetData()),
- Response.Response.GetSize()));
- Text = SB.ToString();
- }
- else if (ContentType == ZenContentType::kCbObject)
- {
- ExtendableStringBuilder<256> SB;
- SB.Append("\n");
- CompactBinaryToJson(Response.Response.GetView(), SB);
- Text = SB.ToString();
- }
- }
- }
- if (Response.ErrorCode != 0)
- {
- ErrorCode = Response.ErrorCode;
- }
- else if (!Response.Success)
- {
- ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- }
- return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text};
+ return Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0;
}
- Ref<JupiterClient> m_JupiterClient;
- const std::string m_Namespace;
- const std::string m_Bucket;
- const Oid m_BuildId;
- IoBuffer m_MetaData;
- Oid m_OplogBuildPartId = Oid::Zero;
- std::filesystem::path m_TempFilePath;
- const bool m_EnableBlocks = true;
- const bool m_UseTempBlocks = true;
- const bool m_AllowRedirect = false;
-
- std::atomic_uint64_t m_SentBytes = {};
- std::atomic_uint64_t m_ReceivedBytes = {};
- std::atomic_uint64_t m_RequestTimeNS = {};
- std::atomic_uint64_t m_RequestCount = {};
- std::atomic_uint64_t m_PeakSentBytes = {};
- std::atomic_uint64_t m_PeakReceivedBytes = {};
- std::atomic_uint64_t m_PeakBytesPerSec = {};
+ std::unique_ptr<BuildStorage::Statistics> m_BuildStorageStats;
+ std::unique_ptr<HttpClient> m_BuildStorageHttp;
+ std::unique_ptr<BuildStorage> m_BuildStorage;
+ const std::string m_Url;
+ const std::string m_Namespace;
+ const std::string m_Bucket;
+ const Oid m_BuildId;
+ IoBuffer m_MetaData;
+ Oid m_OplogBuildPartId = Oid::Zero;
+ const bool m_EnableBlocks = true;
+ const bool m_UseTempBlocks = true;
+ const bool m_AllowRedirect = false;
};
std::shared_ptr<RemoteProjectStore>
-CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
+CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet)
{
std::string Url = Options.Url;
if (Url.find("://"sv) == std::string::npos)
@@ -468,13 +473,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
// Assume https URL
Url = fmt::format("https://{}"sv, Url);
}
- JupiterClientOptions ClientOptions{.Name = "Remote store"sv,
- .ServiceUrl = Url,
- .ConnectTimeout = std::chrono::milliseconds(2000),
- .Timeout = std::chrono::milliseconds(1800000),
- .AssumeHttp2 = Options.AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 4};
+
// 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
// 2) Access token as parameter in request
// 3) Environment variable (different win vs linux/mac)
@@ -491,7 +490,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
}
else if (!Options.OidcExePath.empty())
{
- if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe)
{
TokenProvider = TokenProviderMaybe.value();
}
@@ -502,16 +501,31 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
}
- Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider)));
+ HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
+ .ConnectTimeout = std::chrono::milliseconds(2000),
+ .Timeout = std::chrono::milliseconds(1800000),
+ .AccessTokenProvider = std::move(TokenProvider),
+ .AssumeHttp2 = Options.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 4};
+
+ std::unique_ptr<BuildStorage::Statistics> BuildStorageStats(std::make_unique<BuildStorage::Statistics>());
+
+ std::unique_ptr<HttpClient> BuildStorageHttp = std::make_unique<HttpClient>(Url, ClientSettings);
+
+ std::unique_ptr<BuildStorage> BuildStorage =
+ CreateJupiterBuildStorage(Log(), *BuildStorageHttp, *BuildStorageStats, Options.Namespace, Options.Bucket, false, TempFilePath);
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(Client),
+ std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(BuildStorageStats),
+ std::move(BuildStorageHttp),
+ std::move(BuildStorage),
+ Url,
Options.Namespace,
Options.Bucket,
Options.BuildId,
Options.MetaData,
Options.ForceDisableBlocks,
- Options.ForceDisableTempBlocks,
- TempFilePath);
+ Options.ForceDisableTempBlocks);
return RemoteStore;
}
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h
index c52b13886..60b6caef7 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.h
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.h
@@ -24,7 +24,8 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions
IoBuffer MetaData;
};
-std::shared_ptr<RemoteProjectStore> CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
- const std::filesystem::path& TempFilePath);
+std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
+ const std::filesystem::path& TempFilePath,
+ bool Quiet);
} // namespace zen
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 317a419eb..9600133f3 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -235,13 +235,11 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store,
- ProjectStore* Projects,
+HttpProjectService::HttpProjectService(ProjectStore* Projects,
HttpStatusService& StatusService,
HttpStatsService& StatsService,
AuthMgr& AuthMgr)
: m_Log(logging::Get("project"))
-, m_CidStore(Store)
, m_ProjectStore(Projects)
, m_StatusService(StatusService)
, m_StatsService(StatsService)
@@ -407,8 +405,45 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
ZEN_TRACE_CPU("ProjectService::Stats");
- const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
- const CidStoreSize CidSize = m_CidStore.TotalSize();
+ bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true";
+
+ const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+ uint64_t TotalChunkHitCount = 0;
+ uint64_t TotalChunkMissCount = 0;
+ uint64_t TotalChunkWriteCount = 0;
+ CidStoreSize TotalCidSize;
+
+ tsl::robin_map<CidStore*, std::string> UniqueStores;
+ {
+ m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) {
+ CidStore* Store = &Project.GetCidStore();
+ if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
+ {
+ UniqueStores.insert_or_assign(Store, Project.Identifier);
+ }
+ else
+ {
+ UniqueStores.insert_or_assign(Store, std::string{});
+ }
+ });
+
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+
+ CidStoreStats ChunkStoreStats = ChunkStore->Stats();
+ CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
+
+ TotalChunkHitCount += ChunkStoreStats.HitCount;
+ TotalChunkMissCount += ChunkStoreStats.MissCount;
+ TotalChunkWriteCount += ChunkStoreStats.WriteCount;
+
+ TotalCidSize.TinySize += ChunkStoreSize.TinySize;
+ TotalCidSize.SmallSize += ChunkStoreSize.SmallSize;
+ TotalCidSize.LargeSize += ChunkStoreSize.LargeSize;
+ TotalCidSize.TotalSize += ChunkStoreSize.TotalSize;
+ }
+ }
CbObjectWriter Cbo;
@@ -460,12 +495,66 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
Cbo.BeginObject("size");
{
- Cbo << "tiny" << CidSize.TinySize;
- Cbo << "small" << CidSize.SmallSize;
- Cbo << "large" << CidSize.LargeSize;
- Cbo << "total" << CidSize.TotalSize;
+ Cbo << "tiny" << TotalCidSize.TinySize;
+ Cbo << "small" << TotalCidSize.SmallSize;
+ Cbo << "large" << TotalCidSize.LargeSize;
+ Cbo << "total" << TotalCidSize.TotalSize;
}
Cbo.EndObject();
+
+ if (ShowCidStoreStats)
+ {
+ Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
+ const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
+ Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
+
+ Cbo.BeginObject("store");
+
+ auto OutputStats = [&](CidStore& ChunkStore) {
+ CidStoreStats StoreStats = ChunkStore.Stats();
+ Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
+ const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
+ Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
+ EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
+ EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
+ };
+
+ if (UniqueStores.size() > 1)
+ {
+ Cbo.BeginArray("projects");
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+ const std::string& ProjectId = It.second;
+ CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
+
+ Cbo.BeginObject();
+ {
+ Cbo << "project" << ProjectId;
+ Cbo.BeginObject("stats");
+ OutputStats(*ChunkStore);
+ Cbo.EndObject();
+
+ Cbo.BeginObject("size");
+ {
+ Cbo << "tiny" << ChunkStoreSize.TinySize;
+ Cbo << "small" << ChunkStoreSize.SmallSize;
+ Cbo << "large" << ChunkStoreSize.LargeSize;
+ Cbo << "total" << ChunkStoreSize.TotalSize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndArray(); // projects
+ }
+ else if (UniqueStores.size() != 0)
+ {
+ CidStore& ChunkStore = *UniqueStores.begin()->first;
+ OutputStats(ChunkStore);
+ }
+ Cbo.EndObject();
+ }
}
Cbo.EndObject();
@@ -1125,6 +1214,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
ProjectStore::Oplog& Oplog = *FoundLog;
IoBuffer Payload = HttpReq.ReadPayload();
@@ -1137,7 +1228,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
std::vector<IoHash> MissingChunks;
CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
- if (m_CidStore.ContainsChunk(Hash))
+ if (ChunkStore.ContainsChunk(Hash))
{
// Return null attachment as we already have it, no point in reading it and storing it again
return {};
@@ -1393,6 +1484,8 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
ProjectStore::Oplog& Oplog = *FoundLog;
if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
@@ -1407,7 +1500,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
Op.IterateAttachments([&](CbFieldView FieldView) {
const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(AttachmentHash);
if (Payload)
{
switch (Payload.GetContentType())
@@ -2036,11 +2129,14 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+ CidStore& ChunkStore = Project.GetCidStore();
+
Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
+ const Oid& Key,
+ CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
});
@@ -2054,8 +2150,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
m_ProjectStore->DiscoverProjects();
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- std::vector<std::string> OpLogs = Project.ScanForOplogs();
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ CidStore& ChunkStore = Project.GetCidStore();
+ std::vector<std::string> OpLogs = Project.ScanForOplogs();
+ CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
});
}
Cbo.EndArray();
@@ -2084,7 +2181,8 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
{
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
if (CSV)
{
@@ -2092,10 +2190,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
+ const Oid& Key,
+ CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2105,7 +2204,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
Cbo.BeginArray("projects");
{
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2141,16 +2240,17 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
+ ProjectStore::Oplog& Oplog = *FoundLog;
if (CSV)
{
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ [this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2159,7 +2259,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("oplogs");
{
- CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2204,9 +2304,10 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId));
}
- const Oid ObjId = Oid::FromHexString(OpId);
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ const Oid ObjId = Oid::FromHexString(OpId);
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
+ ProjectStore::Oplog& Oplog = *FoundLog;
std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId);
if (!Op.has_value())
@@ -2224,7 +2325,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
else
@@ -2232,7 +2333,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("ops");
{
- CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
+ CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 295defa5c..5782188e6 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -35,11 +35,7 @@ class ProjectStore;
class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(CidStore& Store,
- ProjectStore* InProjectStore,
- HttpStatusService& StatusService,
- HttpStatsService& StatsService,
- AuthMgr& AuthMgr);
+ HttpProjectService(ProjectStore* InProjectStore, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr);
~HttpProjectService();
virtual const char* BaseUri() const override;
@@ -92,7 +88,6 @@ private:
inline LoggerRef Log() { return m_Log; }
LoggerRef m_Log;
- CidStore& m_CidStore;
HttpRequestRouter m_Router;
Ref<ProjectStore> m_ProjectStore;
HttpStatusService& m_StatusService;
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 3728babb5..dba5cd4a7 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -337,7 +337,7 @@ private:
};
std::shared_ptr<RemoteProjectStore>
-CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
+CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet)
{
std::string Url = Options.Url;
if (Url.find("://"sv) == std::string::npos)
@@ -368,7 +368,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
}
else if (!Options.OidcExePath.empty())
{
- if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe)
{
TokenProvider = TokenProviderMaybe.value();
}
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h
index 8bf79d563..ac2d25b47 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.h
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h
@@ -25,6 +25,7 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions
};
std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
- const std::filesystem::path& TempFilePath);
+ const std::filesystem::path& TempFilePath,
+ bool Quiet);
} // namespace zen
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 53e687983..d6dd6ef9b 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -267,7 +267,7 @@ namespace {
ForceDisableBlocks,
ForceDisableTempBlocks,
AssumeHttp2};
- RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath);
+ RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false);
}
if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -364,7 +364,7 @@ namespace {
ForceDisableTempBlocks,
AssumeHttp2,
MetaData};
- RemoteStore = CreateBuildsRemoteStore(Options, TempFilePath);
+ RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false);
}
if (!RemoteStore)
@@ -1176,7 +1176,7 @@ ProjectStore::Oplog::Flush()
}
void
-ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
+ProjectStore::Oplog::Scrub(ScrubContext& Ctx)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
@@ -1492,7 +1492,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f
if (ValidationError != CbValidateError::None)
{
- ZEN_ERROR("validation error {} hit for oplog config at '{}'", int(ValidationError), StateFilePath);
+ ZEN_ERROR("validation error {} hit for oplog config at '{}'", ToString(ValidationError), StateFilePath);
return CbObject();
}
@@ -3168,7 +3168,7 @@ ProjectStore::Project::Read()
}
else
{
- ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath);
+ ZEN_ERROR("validation error {} hit for '{}'", ToString(ValidationError), ProjectStateFilePath);
}
ReadAccessTimes();
@@ -3260,7 +3260,7 @@ ProjectStore::Project::ReadAccessTimes()
}
else
{
- ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, int(ValidationError), ProjectAccessTimesFilePath);
+ ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, ToString(ValidationError), ProjectAccessTimesFilePath);
}
}
@@ -3575,7 +3575,7 @@ ProjectStore::Project::Flush()
}
void
-ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
+ProjectStore::Project::Scrub(ScrubContext& Ctx)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
// Scrubbing needs to check all existing oplogs
@@ -3587,7 +3587,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) {
if (!IsExpired(GcClock::TimePoint::min(), Ops))
{
- Ops.ScrubStorage(Ctx);
+ Ops.Scrub(Ctx);
}
});
}
@@ -3832,7 +3832,7 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const
//////////////////////////////////////////////////////////////////////////
-ProjectStore::ProjectStore(CidStore& Store,
+ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -3840,7 +3840,7 @@ ProjectStore::ProjectStore(CidStore& Store,
const Configuration& Config)
: m_Log(logging::Get("project"))
, m_Gc(Gc)
-, m_CidStore(Store)
+, m_GetCidStore(std::move(GetCidStore))
, m_JobQueue(JobQueue)
, m_OpenProcessCache(InOpenProcessCache)
, m_ProjectBasePath(BasePath)
@@ -3973,7 +3973,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx)
}
for (const Ref<Project>& Project : Projects)
{
- Project->ScrubStorage(Ctx);
+ Project->Scrub(Ctx);
}
}
@@ -4025,6 +4025,8 @@ ProjectStore::OpenProject(std::string_view ProjectId)
}
}
+ CidStore& ChunkStore = m_GetCidStore(ProjectId);
+
RwLock::ExclusiveLockScope _(m_ProjectsLock);
if (auto ProjIt = m_Projects.find(std::string{ProjectId}); ProjIt != m_Projects.end())
{
@@ -4041,7 +4043,7 @@ ProjectStore::OpenProject(std::string_view ProjectId)
Ref<Project>& Prj =
m_Projects
- .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+ .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->Read();
@@ -4068,12 +4070,14 @@ ProjectStore::NewProject(const std::filesystem::path& BasePath,
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::NewProject");
+ CidStore& ChunkStore = m_GetCidStore(ProjectId);
+
RwLock::ExclusiveLockScope _(m_ProjectsLock);
ZEN_INFO("project '{}': creating project at '{}'", ProjectId, BasePath);
Ref<Project>& Prj =
- m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+ m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->RootDir = RootDir;
@@ -4802,7 +4806,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
}
const IoHash Hash = IoHash::FromHexString(Cid);
- OutChunk = m_CidStore.FindChunkByCid(Hash);
+ OutChunk = Project->GetCidStore().FindChunkByCid(Hash);
if (!OutChunk)
{
@@ -4865,7 +4869,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
}
FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash});
- CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash);
+ CidStore::InsertResult Result = Project->GetCidStore().AddChunk(Chunk, Hash);
return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}};
}
@@ -4894,18 +4898,19 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
if (RequestObject["chunks"sv].IsArray())
{
// Legacy full chunks only by rawhash
- CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
-
+ CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
CbObjectWriter ResponseWriter;
ResponseWriter.BeginArray("chunks"sv);
for (CbFieldView FieldView : ChunksArray)
{
IoHash RawHash = FieldView.AsHash();
- IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash);
+ IoBuffer ChunkBuffer = ChunkStore.FindChunkByCid(RawHash);
if (ChunkBuffer)
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer));
@@ -5057,7 +5062,7 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
if (ChunkRequest.Input.Id.index() == 0)
{
const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id);
- IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(ChunkHash);
if (Payload)
{
ChunkRequest.Output.Exists = true;
@@ -5244,7 +5249,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
return {HttpResponseCode::BadRequest, "Invalid payload format"};
}
- CidStore& ChunkStore = m_CidStore;
+ CidStore& ChunkStore = Project->GetCidStore();
RwLock AttachmentsLock;
tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
@@ -5350,7 +5355,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId,
}
}
- CidStore& ChunkStore = m_CidStore;
+ CidStore& ChunkStore = Project->GetCidStore();
RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer(
ChunkStore,
@@ -5462,6 +5467,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
if (Method == "import"sv)
{
if (!AreDiskWritesAllowed())
@@ -5543,7 +5550,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Oplog->CaptureAddedAttachments(WriteRawHashes);
- m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
+ ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
}
HttpReq.WriteResponse(HttpResponseCode::OK);
return true;
@@ -5716,14 +5723,14 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
ResponseObj.EndArray();
}
- // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new
- // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have
+ // Ops that have moved chunks to a compressed buffer for storage in ChunkStore have been rewritten with references to the new
+ // chunk(s). Make sure we add the chunks to ChunkStore, and do it after we update the oplog so GC doesn't think we have
// unreferenced chunks.
for (auto It : AddedChunks)
{
const IoHash& RawHash = It.first;
AddedChunk& Chunk = It.second;
- CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash);
+ CidStore::InsertResult Result = ChunkStore.AddChunk(Chunk.Buffer, RawHash);
if (Result.New)
{
InlinedBytes += Chunk.RawSize;
@@ -5786,7 +5793,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
EmbedLooseFile,
Force,
IgnoreMissingAttachments](JobContext& Context) {
- RemoteProjectStore::Result Result = SaveOplog(m_CidStore,
+ RemoteProjectStore::Result Result = SaveOplog(Project->GetCidStore(),
*ActualRemoteStore,
*Project.Get(),
*OplogPtr,
@@ -5801,7 +5808,8 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
{
- throw std::runtime_error(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second);
+ throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second,
+ (int)Response.first);
}
});
@@ -5830,19 +5838,26 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
}
std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
+ CidStore& ChunkStore = Project.GetCidStore();
ZEN_INFO("Loading oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description);
JobId JobId = m_JobQueue.QueueJob(
fmt::format("Import oplog '{}/{}'", Project.Identifier, Oplog.OplogId()),
- [this, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, Force, IgnoreMissingAttachments, CleanOplog](
- JobContext& Context) {
+ [this,
+ ChunkStore = &ChunkStore,
+ ActualRemoteStore = std::move(RemoteStore),
+ OplogPtr = &Oplog,
+ Force,
+ IgnoreMissingAttachments,
+ CleanOplog](JobContext& Context) {
RemoteProjectStore::Result Result =
- LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
+ LoadOplog(*ChunkStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
auto Response = ConvertResult(Result);
ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
{
- throw std::runtime_error(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second);
+ throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second,
+ (int)Response.first);
}
});
@@ -6875,6 +6890,11 @@ namespace testutils {
return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags);
}
+ ProjectStore::GetCidStoreFunc SingleChunkStore(CidStore& ChunkStore)
+ {
+ return [ChunkStore = &ChunkStore](std::string_view) -> CidStore& { return *ChunkStore; };
+ }
+
} // namespace testutils
TEST_CASE("project.opkeys")
@@ -6937,13 +6957,18 @@ TEST_CASE("project.store.create")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::string_view ProjectName("proj1"sv);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -6968,12 +6993,17 @@ TEST_CASE("project.store.lifetimes")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7031,12 +7061,17 @@ TEST_CASE_TEMPLATE("project.store.export",
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7070,7 +7105,7 @@ TEST_CASE_TEMPLATE("project.store.export",
std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
- RemoteProjectStore::Result ExportResult = SaveOplog(CidStore,
+ RemoteProjectStore::Result ExportResult = SaveOplog(ChunkStore,
*RemoteStore,
*Project.Get(),
*Oplog,
@@ -7087,7 +7122,7 @@ TEST_CASE_TEMPLATE("project.store.export",
ProjectStore::Oplog* OplogImport = Project->NewOplog("oplog2", {});
CHECK(OplogImport != nullptr);
- RemoteProjectStore::Result ImportResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7096,7 +7131,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportForceResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7105,7 +7140,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportForceResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportCleanResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7114,7 +7149,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportCleanResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7134,12 +7169,17 @@ TEST_CASE("project.store.gc")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7335,12 +7375,17 @@ TEST_CASE("project.store.gc.prep")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7383,7 +7428,7 @@ TEST_CASE("project.store.gc.prep")
// Equivalent of a `prep` existance check call
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7397,7 +7442,7 @@ TEST_CASE("project.store.gc.prep")
// If a gc comes in between our prep and op write the chunks will be removed
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7417,7 +7462,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7431,7 +7476,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7445,7 +7490,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained across multiple GCs if retain time is still valud
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7457,7 +7502,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
@@ -7474,7 +7519,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7507,7 +7552,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
Sleep(200);
@@ -7522,7 +7567,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
// This pass the retention time has expired and the last GC pass cleared the entries
@@ -7536,7 +7581,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
}
@@ -7550,12 +7595,17 @@ TEST_CASE("project.store.rpc.getchunks")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8472,12 +8522,17 @@ TEST_CASE("project.store.partial.read")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8650,12 +8705,17 @@ TEST_CASE("project.store.iterateoplog")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv";
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 368da5ea4..eb27665f9 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -68,7 +68,9 @@ public:
{
};
- ProjectStore(CidStore& Store,
+ typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
+
+ ProjectStore(GetCidStoreFunc&& GetCidStore,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -156,7 +158,7 @@ public:
LoggerRef Log() { return m_OuterProject->Log(); }
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
+ void Scrub(ScrubContext& Ctx);
static uint64_t TotalSize(const std::filesystem::path& BasePath);
uint64_t TotalSize() const;
@@ -326,11 +328,13 @@ public:
Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath);
virtual ~Project();
+ CidStore& GetCidStore() { return m_CidStore; };
+
void Read();
void Write();
[[nodiscard]] static bool Exists(const std::filesystem::path& BasePath);
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
+ void Scrub(ScrubContext& Ctx);
LoggerRef Log() const;
static uint64_t TotalSize(const std::filesystem::path& BasePath);
uint64_t TotalSize() const;
@@ -405,6 +409,7 @@ public:
LoggerRef Log() { return m_Log; }
const std::filesystem::path& BasePath() const { return m_ProjectBasePath; }
+ // GcStorage
virtual void ScrubStorage(ScrubContext& Ctx) override;
virtual GcStorageSize StorageSize() const override;
@@ -498,7 +503,7 @@ public:
private:
LoggerRef m_Log;
GcManager& m_Gc;
- CidStore& m_CidStore;
+ GetCidStoreFunc m_GetCidStore;
JobQueue& m_JobQueue;
OpenProcessCache& m_OpenProcessCache;
std::filesystem::path m_ProjectBasePath;
diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp
index 744b861dd..a1c460bc0 100644
--- a/src/zenserver/upstream/upstreamcache.cpp
+++ b/src/zenserver/upstream/upstreamcache.cpp
@@ -1475,12 +1475,17 @@ namespace detail {
class UpstreamCacheImpl final : public UpstreamCache
{
+ struct EnqueuedRequest
+ {
+ UpstreamCacheRecord Record;
+ std::function<IoBuffer(const IoHash& ChunkHash)> GetValueFunc;
+ };
+
public:
- UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
+ UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore)
: m_Log(logging::Get("upstream"))
, m_Options(Options)
, m_CacheStore(CacheStore)
- , m_CidStore(CidStore)
{
}
@@ -1836,17 +1841,17 @@ public:
}
}
- virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) override
+ virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) override
{
if (m_RunState.IsRunning && m_Options.WriteUpstream && m_Endpoints.size() > 0)
{
if (!m_UpstreamThreads.empty())
{
- m_UpstreamQueue.Enqueue(std::move(CacheRecord));
+ m_UpstreamQueue.Enqueue(EnqueuedRequest{.Record = std::move(CacheRecord), .GetValueFunc = GetValueFunc});
}
else
{
- ProcessCacheRecord(std::move(CacheRecord));
+ ProcessCacheRecord(std::move(CacheRecord), std::move(GetValueFunc));
}
}
}
@@ -1900,7 +1905,7 @@ public:
}
private:
- void ProcessCacheRecord(UpstreamCacheRecord CacheRecord)
+ void ProcessCacheRecord(const UpstreamCacheRecord& CacheRecord, std::function<IoBuffer(const IoHash& ChunkHash)>&& GetValueFunc)
{
ZEN_TRACE_CPU("Upstream::ProcessCacheRecord");
@@ -1918,7 +1923,7 @@ private:
for (const IoHash& ValueContentId : CacheRecord.ValueContentIds)
{
- if (IoBuffer Payload = m_CidStore.FindChunkByCid(ValueContentId))
+ if (IoBuffer Payload = GetValueFunc(ValueContentId))
{
Payloads.push_back(Payload);
}
@@ -1970,19 +1975,19 @@ private:
for (;;)
{
- UpstreamCacheRecord CacheRecord;
- if (m_UpstreamQueue.WaitAndDequeue(CacheRecord))
+ EnqueuedRequest Request;
+ if (m_UpstreamQueue.WaitAndDequeue(Request))
{
try
{
- ProcessCacheRecord(std::move(CacheRecord));
+ ProcessCacheRecord(Request.Record, std::move(Request.GetValueFunc));
}
catch (const std::exception& Err)
{
ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'",
- CacheRecord.Namespace,
- CacheRecord.Key.Bucket,
- CacheRecord.Key.Hash,
+ Request.Record.Namespace,
+ Request.Record.Key.Bucket,
+ Request.Record.Key.Hash,
Err.what());
}
}
@@ -2076,7 +2081,7 @@ private:
LoggerRef Log() { return m_Log; }
- using UpstreamQueue = BlockingQueue<UpstreamCacheRecord>;
+ using UpstreamQueue = BlockingQueue<EnqueuedRequest>;
struct RunState
{
@@ -2102,7 +2107,6 @@ private:
LoggerRef m_Log;
UpstreamCacheOptions m_Options;
ZenCacheStore& m_CacheStore;
- CidStore& m_CidStore;
UpstreamQueue m_UpstreamQueue;
std::shared_mutex m_EndpointsMutex;
std::vector<std::unique_ptr<UpstreamEndpoint>> m_Endpoints;
@@ -2126,9 +2130,9 @@ UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, con
}
std::unique_ptr<UpstreamCache>
-CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
+CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore)
{
- return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore);
+ return std::make_unique<UpstreamCacheImpl>(Options, CacheStore);
}
} // namespace zen
diff --git a/src/zenserver/upstream/upstreamcache.h b/src/zenserver/upstream/upstreamcache.h
index 26e5decac..e4b9a73ad 100644
--- a/src/zenserver/upstream/upstreamcache.h
+++ b/src/zenserver/upstream/upstreamcache.h
@@ -24,7 +24,6 @@ class AuthMgr;
class CbObjectView;
class CbPackage;
class CbObjectWriter;
-class CidStore;
class ZenCacheStore;
struct JupiterClientOptions;
class JupiterAccessTokenProvider;
@@ -162,6 +161,6 @@ struct UpstreamCacheOptions
bool WriteUpstream = true;
};
-std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore);
+std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore);
} // namespace zen
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 27ec4c690..5cab54acc 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -239,18 +239,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
CidStoreConfiguration Config;
Config.RootDirectory = m_DataRoot / "cas";
- m_CidStore = std::make_unique<CidStore>(m_GcManager);
- m_CidStore->Initialize(Config);
+ m_CidStores.insert_or_assign({}, std::make_unique<CidStore>(m_GcManager));
+ m_CidStores.at({})->Initialize(Config);
ZEN_INFO("instantiating project service");
- m_ProjectStore = new ProjectStore(*m_CidStore,
+ m_ProjectStore = new ProjectStore([this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); },
m_DataRoot / "projects",
m_GcManager,
*m_JobQueue,
*m_OpenProcessCache,
ProjectStore::Configuration{});
- m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr});
+ m_HttpProjectService.reset(new HttpProjectService(m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr));
if (ServerOptions.WorksSpacesConfig.Enabled)
{
@@ -265,10 +265,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
if (ServerOptions.BuildStoreConfig.Enabled)
{
+ CidStoreConfiguration BuildCidConfig;
+ BuildCidConfig.RootDirectory = m_DataRoot / "builds_cas";
+ m_BuildCidStore = std::make_unique<CidStore>(m_GcManager);
+ m_BuildCidStore->Initialize(BuildCidConfig);
+
BuildStoreConfig BuildsCfg;
BuildsCfg.RootDirectory = m_DataRoot / "builds";
BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit;
- m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager);
+ m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore);
}
if (ServerOptions.StructuredCacheConfig.Enabled)
@@ -357,17 +362,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
m_GcScheduler.Initialize(GcConfig);
// Create and register admin interface last to make sure all is properly initialized
- m_AdminService =
- std::make_unique<HttpAdminService>(m_GcScheduler,
- *m_JobQueue,
- m_CacheStore.Get(),
- m_CidStore.get(),
- m_ProjectStore,
- m_BuildStore.get(),
- HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile,
- .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log",
- .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
- ServerOptions);
+ m_AdminService = std::make_unique<HttpAdminService>(
+ m_GcScheduler,
+ *m_JobQueue,
+ m_CacheStore.Get(),
+ [this]() { Flush(); },
+ HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile,
+ .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log",
+ .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
+ ServerOptions);
m_Http->RegisterService(*m_AdminService);
return EffectiveBasePort;
@@ -419,10 +422,10 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All);
ValidationResult != CbValidateError::None)
{
- ZEN_WARN("Manifest validation failed: {}, state will be wiped", uint32_t(ValidationResult));
+ ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult));
WipeState = true;
- WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, uint32_t(ValidationResult));
+ WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult));
}
else
{
@@ -547,6 +550,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
Config.AllowAutomaticCreationOfNamespaces = true;
Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled,
.EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled};
+
for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs)
{
const std::string& BucketName = It.first;
@@ -554,7 +558,8 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize,
.PayloadAlignment = ZenBucketConfig.PayloadAlignment,
.MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold,
- .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold};
+ .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold,
+ .LimitOverwrites = ZenBucketConfig.LimitOverwrites};
Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig);
}
Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize,
@@ -564,6 +569,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold,
Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold =
ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold,
+ Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites;
Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes;
Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds;
Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds;
@@ -587,7 +593,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount);
}
- m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore);
+ m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore);
m_UpstreamService = std::make_unique<HttpUpstreamService>(*m_UpstreamCache, *m_AuthMgr);
m_UpstreamCache->Initialize();
@@ -651,19 +657,24 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
}
}
- m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(*m_CacheStore,
- *m_CidStore,
- m_StatsService,
- m_StatusService,
- *m_UpstreamCache,
- m_GcManager.GetDiskWriteBlocker(),
- *m_OpenProcessCache);
+ m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(
+ *m_CacheStore,
+ [this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); },
+ m_StatsService,
+ m_StatusService,
+ *m_UpstreamCache,
+ m_GcManager.GetDiskWriteBlocker(),
+ *m_OpenProcessCache);
m_Http->RegisterService(*m_StructuredCacheService);
m_Http->RegisterService(*m_UpstreamService);
m_StatsReporter.AddProvider(m_CacheStore.Get());
- m_StatsReporter.AddProvider(m_CidStore.get());
+ for (const auto& It : m_CidStores)
+ {
+ m_StatsReporter.AddProvider(It.second.get());
+ }
+ m_StatsReporter.AddProvider(m_BuildCidStore.get());
}
void
@@ -836,6 +847,7 @@ ZenServer::Cleanup()
m_BuildStoreService.reset();
m_BuildStore = {};
+ m_BuildCidStore.reset();
m_StructuredCacheService.reset();
m_UpstreamService.reset();
@@ -847,7 +859,7 @@ ZenServer::Cleanup()
m_Workspaces.reset();
m_HttpProjectService.reset();
m_ProjectStore = {};
- m_CidStore.reset();
+ m_CidStores.clear();
m_AuthService.reset();
m_AuthMgr.reset();
m_Http = {};
@@ -1028,37 +1040,21 @@ ZenServer::CheckOwnerPid()
}
void
-ZenServer::ScrubStorage()
-{
- Stopwatch Timer;
- ZEN_INFO("Storage validation STARTING");
-
- WorkerThreadPool ThreadPool{1, "Scrub"};
- ScrubContext Ctx{ThreadPool};
- m_CidStore->ScrubStorage(Ctx);
- m_ProjectStore->ScrubStorage(Ctx);
- m_StructuredCacheService->ScrubStorage(Ctx);
-
- const uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
-
- ZEN_INFO("Storage validation DONE in {}, ({} in {} chunks - {})",
- NiceTimeSpanMs(ElapsedTimeMs),
- NiceBytes(Ctx.ScrubbedBytes()),
- Ctx.ScrubbedChunks(),
- NiceByteRate(Ctx.ScrubbedBytes(), ElapsedTimeMs));
-}
-
-void
ZenServer::Flush()
{
- if (m_CidStore)
- m_CidStore->Flush();
+ for (auto& It : m_CidStores)
+ {
+ It.second->Flush();
+ }
if (m_StructuredCacheService)
m_StructuredCacheService->Flush();
if (m_ProjectStore)
m_ProjectStore->Flush();
+
+ if (m_BuildCidStore)
+ m_BuildCidStore->Flush();
}
void
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 5cfa04ba1..0a17446ae 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -82,7 +82,6 @@ public:
void CheckStateExitFlag();
void CheckOwnerPid();
bool UpdateProcessMonitor();
- void ScrubStorage();
void Flush();
virtual void HandleStatusRequest(HttpServerRequest& Request) override;
@@ -116,19 +115,20 @@ private:
inline void SetNewState(ServerState NewState) { m_CurrentState = NewState; }
static std::string_view ToString(ServerState Value);
- StatsReporter m_StatsReporter;
- Ref<HttpServer> m_Http;
- std::unique_ptr<AuthMgr> m_AuthMgr;
- std::unique_ptr<HttpAuthService> m_AuthService;
- HttpStatusService m_StatusService;
- HttpStatsService m_StatsService;
- GcManager m_GcManager;
- GcScheduler m_GcScheduler{m_GcManager};
- std::unique_ptr<CidStore> m_CidStore;
- Ref<ZenCacheStore> m_CacheStore;
- std::unique_ptr<OpenProcessCache> m_OpenProcessCache;
- HttpTestService m_TestService;
- std::unique_ptr<BuildStore> m_BuildStore;
+ StatsReporter m_StatsReporter;
+ Ref<HttpServer> m_Http;
+ std::unique_ptr<AuthMgr> m_AuthMgr;
+ std::unique_ptr<HttpAuthService> m_AuthService;
+ HttpStatusService m_StatusService;
+ HttpStatsService m_StatsService;
+ GcManager m_GcManager;
+ GcScheduler m_GcScheduler{m_GcManager};
+ tsl::robin_map<std::string, std::unique_ptr<CidStore>> m_CidStores;
+ Ref<ZenCacheStore> m_CacheStore;
+ std::unique_ptr<OpenProcessCache> m_OpenProcessCache;
+ HttpTestService m_TestService;
+ std::unique_ptr<CidStore> m_BuildCidStore;
+ std::unique_ptr<BuildStore> m_BuildStore;
#if ZEN_WITH_TESTS
HttpTestingService m_TestingService;
diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp
index 20dc55bca..1b2cf036b 100644
--- a/src/zenstore/buildstore/buildstore.cpp
+++ b/src/zenstore/buildstore/buildstore.cpp
@@ -3,6 +3,7 @@
#include <zenstore/buildstore/buildstore.h>
#include <zencore/compactbinarybuilder.h>
+#include <zencore/compress.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/memory/llm.h>
@@ -20,7 +21,6 @@ ZEN_THIRD_PARTY_INCLUDES_END
#if ZEN_WITH_TESTS
# include <zencore/compactbinarybuilder.h>
-# include <zencore/compress.h>
# include <zencore/testing.h>
# include <zencore/testutils.h>
# include <zenutil/workerpools.h>
@@ -45,7 +45,7 @@ namespace blobstore::impl {
const char* LogExtension = ".slog";
const char* AccessTimeExtension = ".zacs";
- const uint32_t ManifestVersion = (1 << 16) | (0 << 8) | (0);
+ const uint32_t ManifestVersion = (2 << 16) | (0 << 8) | (0);
std::filesystem::path GetManifestPath(const std::filesystem::path& RootDirectory)
{
@@ -106,13 +106,11 @@ namespace blobstore::impl {
} // namespace blobstore::impl
-BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc)
+BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc, CidStore& BlobStore)
: m_Log(logging::Get("builds"))
, m_Config(Config)
, m_Gc(Gc)
-, m_LargeBlobStore(m_Gc)
-, m_SmallBlobStore(Gc)
-, m_MetadataBlockStore()
+, m_BlobStore(BlobStore)
{
ZEN_TRACE_CPU("BuildStore::BuildStore");
ZEN_MEMSCOPE(GetBuildstoreTag());
@@ -170,75 +168,16 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc)
ManifestWriter.AddDateTime("createdAt", DateTime::Now());
TemporaryFile::SafeWriteFile(ManifestPath, ManifestWriter.Save().GetBuffer().AsIoBuffer());
}
- m_LargeBlobStore.Initialize(Config.RootDirectory / "file_cas", IsNew);
- m_SmallBlobStore.Initialize(Config.RootDirectory,
- "blob_cas",
- m_Config.SmallBlobBlockStoreMaxBlockSize,
- m_Config.SmallBlobBlockStoreAlignement,
- IsNew);
- m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20);
-
- BlockStore::BlockIndexSet KnownBlocks;
- for (const BlobEntry& Blob : m_BlobEntries)
- {
- if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex)
- {
- const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex];
- KnownBlocks.insert(Metadata.Location.BlockIndex);
- }
- }
- BlockStore::BlockIndexSet MissingBlocks = m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite);
m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite);
- if (!MissingBlocks.empty())
- {
- std::vector<MetadataDiskEntry> MissingMetadatas;
- for (auto& It : m_BlobLookup)
- {
- const IoHash& BlobHash = It.first;
- const BlobIndex ReadBlobIndex = It.second;
- const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
- if (ReadBlobEntry.Metadata)
- {
- const MetadataEntry& MetaData = m_MetadataEntries[ReadBlobEntry.Metadata];
- if (MissingBlocks.contains(MetaData.Location.BlockIndex))
- {
- MissingMetadatas.push_back(
- MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = BlobHash});
- MissingMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone;
- m_MetadataEntries[ReadBlobEntry.Metadata] = {};
- m_BlobEntries[ReadBlobIndex].Metadata = {};
- }
- }
- }
- ZEN_ASSERT(!MissingMetadatas.empty());
-
- for (const MetadataDiskEntry& Entry : MissingMetadatas)
- {
- auto It = m_BlobLookup.find(Entry.BlobHash);
- ZEN_ASSERT(It != m_BlobLookup.end());
-
- const BlobIndex ReadBlobIndex = It->second;
- const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
- if (!ReadBlobEntry.Payload)
- {
- m_BlobLookup.erase(It);
- }
- }
- m_MetadatalogFile.Append(MissingMetadatas);
- CompactState();
- }
-
m_Gc.AddGcReferencer(*this);
m_Gc.AddGcReferenceLocker(*this);
- m_Gc.AddGcStorage(this);
}
catch (const std::exception& Ex)
{
ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what());
- m_Gc.RemoveGcStorage(this);
m_Gc.RemoveGcReferenceLocker(*this);
m_Gc.RemoveGcReferencer(*this);
}
@@ -249,7 +188,6 @@ BuildStore::~BuildStore()
try
{
ZEN_TRACE_CPU("BuildStore::~BuildStore");
- m_Gc.RemoveGcStorage(this);
m_Gc.RemoveGcReferenceLocker(*this);
m_Gc.RemoveGcReferencer(*this);
Flush();
@@ -280,21 +218,12 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload)
}
}
- uint64_t PayloadSize = Payload.GetSize();
- PayloadEntry Entry;
- if (Payload.GetSize() > m_Config.SmallBlobBlockStoreMaxBlockEmbedSize)
- {
- CasStore::InsertResult Result = m_LargeBlobStore.InsertChunk(Payload, BlobHash);
- ZEN_UNUSED(Result);
- Entry = PayloadEntry(PayloadEntry::kStandalone, PayloadSize);
- }
- else
- {
- CasStore::InsertResult Result = m_SmallBlobStore.InsertChunk(Payload, BlobHash);
- ZEN_UNUSED(Result);
- Entry = PayloadEntry(0, PayloadSize);
- }
+ uint64_t PayloadSize = Payload.GetSize();
+ CidStore::InsertResult Result = m_BlobStore.AddChunk(Payload, BlobHash);
+ PayloadEntry Entry = PayloadEntry(0, PayloadSize);
+ ;
+ IoHash MetadataHash;
{
RwLock::ExclusiveLockScope _(m_Lock);
if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
@@ -310,6 +239,10 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload)
Blob.Payload = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
m_PayloadEntries.push_back(Entry);
}
+ if (Blob.Metadata)
+ {
+ MetadataHash = m_MetadataEntries[Blob.Metadata].MetadataHash;
+ }
Blob.LastAccessTime = GcClock::TickCount();
}
else
@@ -322,6 +255,16 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload)
m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
m_BlobLookup.insert({BlobHash, NewBlobIndex});
}
+
+ m_LastAccessTimeUpdateCount++;
+ if (m_TrackedBlobKeys)
+ {
+ m_TrackedBlobKeys->push_back(BlobHash);
+ if (MetadataHash != IoHash::Zero)
+ {
+ m_TrackedBlobKeys->push_back(BlobHash);
+ }
+ }
}
m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
m_LastAccessTimeUpdateCount++;
@@ -340,21 +283,9 @@ BuildStore::GetBlob(const IoHash& BlobHash)
Blob.LastAccessTime = GcClock::TickCount();
if (Blob.Payload)
{
- const PayloadEntry& Entry = m_PayloadEntries[Blob.Payload];
- const bool IsStandalone = (Entry.GetFlags() & PayloadEntry::kStandalone) != 0;
Lock.ReleaseNow();
+ IoBuffer Chunk = m_BlobStore.FindChunkByCid(BlobHash);
- IoBuffer Chunk;
- if (IsStandalone)
- {
- ZEN_TRACE_CPU("GetLarge");
- Chunk = m_LargeBlobStore.FindChunk(BlobHash);
- }
- else
- {
- ZEN_TRACE_CPU("GetSmall");
- Chunk = m_SmallBlobStore.FindChunk(BlobHash);
- }
if (Chunk)
{
Chunk.SetContentType(ZenContentType::kCompressedBinary);
@@ -362,7 +293,7 @@ BuildStore::GetBlob(const IoHash& BlobHash)
}
else
{
- ZEN_WARN("Inconsistencies in build store, {} is in index but not {}", BlobHash, IsStandalone ? "on disk" : "in block");
+ ZEN_WARN("Inconsistencies in build store, {} is in index but not in blob store", BlobHash);
}
}
}
@@ -381,10 +312,10 @@ BuildStore::BlobsExists(std::span<const IoHash> BlobHashes)
{
if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
{
- const BlobIndex ExistingBlobIndex = It->second;
- BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex];
- bool HasPayload = !!Blob.Payload;
- bool HasMetadata = !!Blob.Metadata;
+ const BlobIndex ExistingBlobIndex = It->second;
+ const BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex];
+ bool HasPayload = !!Blob.Payload;
+ bool HasMetadata = !!Blob.Metadata;
Result.push_back(BlobExistsResult{.HasBody = HasPayload, .HasMetadata = HasMetadata});
}
else
@@ -396,20 +327,82 @@ BuildStore::BlobsExists(std::span<const IoHash> BlobHashes)
}
void
-BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas)
+BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> Metadatas, WorkerThreadPool* OptionalWorkerPool)
{
ZEN_TRACE_CPU("BuildStore::PutMetadatas");
ZEN_MEMSCOPE(GetBuildstoreTag());
- size_t WriteBlobIndex = 0;
- m_MetadataBlockStore.WriteChunks(MetaDatas, m_Config.MetadataBlockStoreAlignement, [&](std::span<BlockStoreLocation> Locations) {
+ std::vector<IoHash> MetadataHashes;
+ std::vector<IoBuffer> CompressedMetadataBuffers;
+
+ auto CompressOne = [&BlobHashes, &MetadataHashes, &CompressedMetadataBuffers](const IoBuffer& Buffer, size_t Index) {
+ if (Buffer.GetContentType() == ZenContentType::kCompressedBinary)
+ {
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (!CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize))
+ {
+ throw std::runtime_error(
+ fmt::format("Invalid compressed buffer provided when storing metadata for blob {}", BlobHashes[Index]));
+ }
+ else
+ {
+ CompressedMetadataBuffers[Index] = Buffer;
+ MetadataHashes[Index] = RawHash;
+ }
+ }
+ else
+ {
+ CompressedBuffer Compressed =
+ CompressedBuffer::Compress(SharedBuffer(Buffer), OodleCompressor::Mermaid, OodleCompressionLevel::None);
+ MetadataHashes[Index] = Compressed.DecodeRawHash();
+ CompressedMetadataBuffers[Index] = std::move(Compressed.GetCompressed()).Flatten().AsIoBuffer();
+ CompressedMetadataBuffers[Index].SetContentType(ZenContentType::kCompressedBinary);
+ }
+ };
+
+ MetadataHashes.resize(Metadatas.size());
+ CompressedMetadataBuffers.resize(Metadatas.size());
+ if (OptionalWorkerPool)
+ {
+ std::atomic<bool> AbortFlag;
+ std::atomic<bool> PauseFlag;
+ ParallelWork Work(AbortFlag, PauseFlag);
+ for (size_t Index = 0; Index < Metadatas.size(); Index++)
+ {
+ Work.ScheduleWork(
+ *OptionalWorkerPool,
+ [Index, &BlobHashes, &Metadatas, &MetadataHashes, &CompressedMetadataBuffers, &CompressOne](std::atomic<bool>&) {
+ const IoBuffer& Buffer = Metadatas[Index];
+ CompressOne(Buffer, Index);
+ },
+ {});
+ }
+ Work.Wait();
+ }
+ else
+ {
+ for (size_t Index = 0; Index < Metadatas.size(); Index++)
+ {
+ const IoBuffer& Buffer = Metadatas[Index];
+ CompressOne(Buffer, Index);
+ }
+ }
+
+ std::vector<MetadataDiskEntry> AddedMetadataEntries;
+ AddedMetadataEntries.reserve(MetadataHashes.size());
+
+ std::vector<CidStore::InsertResult> InsertResults = m_BlobStore.AddChunks(CompressedMetadataBuffers, MetadataHashes);
+ ZEN_UNUSED(InsertResults);
+ {
RwLock::ExclusiveLockScope _(m_Lock);
- for (size_t LocationIndex = 0; LocationIndex < Locations.size(); LocationIndex++)
+ for (size_t Index = 0; Index < BlobHashes.size(); Index++)
{
- const IoBuffer& Data = MetaDatas[WriteBlobIndex];
- const IoHash& BlobHash = BlobHashes[WriteBlobIndex];
- const BlockStoreLocation& Location = Locations[LocationIndex];
+ const ZenContentType ContentType = Metadatas[Index].GetContentType();
+ const IoHash& BlobHash = BlobHashes[Index];
+ const IoHash& MetadataHash = MetadataHashes[Index];
+ const IoBuffer& Metadata = CompressedMetadataBuffers[Index];
- MetadataEntry Entry = {.Location = Location, .ContentType = Data.GetContentType(), .Flags = 0};
+ MetadataEntry Entry(MetadataHash, Metadata.GetSize(), ContentType, 0);
if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
{
@@ -435,17 +428,16 @@ BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoB
m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
m_BlobLookup.insert({BlobHash, NewBlobIndex});
}
-
- m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
-
m_LastAccessTimeUpdateCount++;
- WriteBlobIndex++;
- if (m_TrackedCacheKeys)
+ if (m_TrackedBlobKeys)
{
- m_TrackedCacheKeys->insert(BlobHash);
+ m_TrackedBlobKeys->push_back(BlobHash);
+ m_TrackedBlobKeys->push_back(MetadataHash);
}
+ AddedMetadataEntries.push_back(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
}
- });
+ }
+ m_MetadatalogFile.Append(AddedMetadataEntries);
}
std::vector<IoBuffer>
@@ -453,9 +445,9 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O
{
ZEN_TRACE_CPU("BuildStore::GetMetadatas");
ZEN_MEMSCOPE(GetBuildstoreTag());
- std::vector<BlockStoreLocation> MetaLocations;
- std::vector<size_t> MetaLocationResultIndexes;
- MetaLocations.reserve(BlobHashes.size());
+ std::vector<IoHash> MetadataHashes;
+ std::vector<size_t> MetaLocationResultIndexes;
+ MetadataHashes.reserve(BlobHashes.size());
MetaLocationResultIndexes.reserve(BlobHashes.size());
tsl::robin_set<uint32_t> ReferencedBlocks;
@@ -475,10 +467,9 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O
if (ExistingBlobEntry.Metadata)
{
const MetadataEntry& ExistingMetadataEntry = m_MetadataEntries[ExistingBlobEntry.Metadata];
- MetaLocations.push_back(ExistingMetadataEntry.Location);
+ MetadataHashes.push_back(ExistingMetadataEntry.MetadataHash);
MetaLocationResultIndexes.push_back(Index);
- ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex);
- ResultContentTypes[Index] = ExistingMetadataEntry.ContentType;
+ ResultContentTypes[Index] = ExistingMetadataEntry.GetContentType();
}
ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount());
m_LastAccessTimeUpdateCount++;
@@ -486,100 +477,35 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O
}
}
- auto DoOneBlock = [this](std::span<const BlockStoreLocation> MetaLocations,
- std::span<const size_t> MetaLocationResultIndexes,
- std::span<const size_t> ChunkIndexes,
- std::vector<IoBuffer>& Result) {
- if (ChunkIndexes.size() < 4)
- {
- for (size_t ChunkIndex : ChunkIndexes)
+ m_BlobStore.IterateChunks(
+ MetadataHashes,
+ [this, &BlobHashes, &MetadataHashes, &MetaLocationResultIndexes, &Result](size_t Index, const IoBuffer& Payload) {
+ if (Payload)
{
- IoBuffer Chunk = m_MetadataBlockStore.TryGetChunk(MetaLocations[ChunkIndex]);
- if (Chunk)
- {
- size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex];
- Result[ResultIndex] = std::move(Chunk);
- }
- }
- return true;
- }
- return m_MetadataBlockStore.IterateBlock(
- MetaLocations,
- ChunkIndexes,
- [&MetaLocationResultIndexes, &Result](size_t ChunkIndex, const void* Data, uint64_t Size) {
- if (Data != nullptr)
+ size_t ResultIndex = MetaLocationResultIndexes[Index];
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer CompressedBuffer = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize);
+ if (CompressedBuffer)
{
- size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex];
- Result[ResultIndex] = IoBuffer(IoBuffer::Clone, Data, Size);
- }
- return true;
- },
- [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
- size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex];
- Result[ResultIndex] = File.GetChunk(Offset, Size);
- return true;
- },
- 8u * 1024u);
- };
-
- if (!MetaLocations.empty())
- {
- std::atomic<bool> AbortFlag;
- std::atomic<bool> PauseFlag;
- ParallelWork Work(AbortFlag, PauseFlag);
-
- try
- {
- m_MetadataBlockStore.IterateChunks(
- MetaLocations,
- [this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock](
- uint32_t BlockIndex,
- std::span<const size_t> ChunkIndexes) -> bool {
- ZEN_UNUSED(BlockIndex);
- if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1)
+ IoBuffer Decompressed = CompressedBuffer.DecompressToComposite().Flatten().AsIoBuffer();
+ if (Decompressed)
{
- return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result);
+ Result[ResultIndex] = std::move(Decompressed);
}
else
{
- ZEN_ASSERT(OptionalWorkerPool != nullptr);
- std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end());
- Work.ScheduleWork(
- *OptionalWorkerPool,
- [this,
- &Result,
- &MetaLocations,
- &MetaLocationResultIndexes,
- DoOneBlock,
- ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic<bool>& AbortFlag) {
- if (AbortFlag)
- {
- return;
- }
- try
- {
- if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result))
- {
- AbortFlag.store(true);
- }
- }
- catch (const std::exception& Ex)
- {
- ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what());
- }
- });
- return !Work.IsAborted();
+ ZEN_WARN("Metadata {} for blob {} is malformed (not a compressed binary format)",
+ MetadataHashes[ResultIndex],
+ BlobHashes[ResultIndex]);
}
- });
- }
- catch (const std::exception& Ex)
- {
- AbortFlag.store(true);
- ZEN_WARN("Failed iterating block metadata chunks in {}. Reason: '{}'", m_Config.RootDirectory, Ex.what());
- }
+ }
+ }
+ return true;
+ },
+ OptionalWorkerPool,
+ 8u * 1024u);
- Work.Wait();
- }
for (size_t Index = 0; Index < Result.size(); Index++)
{
if (Result[Index])
@@ -600,9 +526,7 @@ BuildStore::Flush()
const auto _ = MakeGuard(
[&] { ZEN_INFO("Flushed build store at {} in {}", m_Config.RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
- m_LargeBlobStore.Flush();
- m_SmallBlobStore.Flush();
- m_MetadataBlockStore.Flush(false);
+ m_BlobStore.Flush();
m_PayloadlogFile.Flush();
m_MetadatalogFile.Flush();
@@ -636,22 +560,14 @@ BuildStore::GetStorageStats() const
{
const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload];
uint64_t Size = Payload.GetSize();
- if ((Payload.GetFlags() & PayloadEntry::kStandalone) != 0)
- {
- Result.LargeBlobCount++;
- Result.LargeBlobBytes += Size;
- }
- else
- {
- Result.SmallBlobCount++;
- Result.SmallBlobBytes += Size;
- }
+ Result.BlobCount++;
+ Result.BlobBytes += Size;
}
if (ReadBlobEntry.Metadata)
{
const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata];
Result.MetadataCount++;
- Result.MetadataByteCount += Metadata.Location.Size;
+ Result.MetadataByteCount += Metadata.GetSize();
}
}
}
@@ -882,10 +798,9 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy
CasLog.Replay(
[&](const MetadataDiskEntry& Record) {
std::string InvalidEntryReason;
- if (Record.Entry.Flags & MetadataEntry::kTombStone)
+ if (Record.Entry.GetFlags() & MetadataEntry::kTombStone)
{
// Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
- // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end())
{
if (!m_BlobEntries[ExistingIt->second].Payload)
@@ -1058,7 +973,7 @@ BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string&
OutReason = fmt::format("Invalid blob hash {}", Entry.BlobHash.ToHexString());
return false;
}
- if (Entry.Entry.GetFlags() & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone))
+ if (Entry.Entry.GetFlags() & ~(PayloadEntry::kTombStone))
{
OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.GetFlags(), Entry.BlobHash.ToHexString());
return false;
@@ -1083,30 +998,20 @@ BuildStore::ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::strin
OutReason = fmt::format("Invalid blob hash {} for meta entry", Entry.BlobHash.ToHexString());
return false;
}
- if (Entry.Entry.Location.Size == 0)
+ if (Entry.Entry.GetSize() == 0)
{
- OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.Location.Size);
+ OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.GetSize());
return false;
}
- if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0)
- {
- OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString());
- return false;
- }
- if (Entry.Entry.Flags & MetadataEntry::kTombStone)
+ if (Entry.Entry.GetFlags() & MetadataEntry::kTombStone)
{
return true;
}
- if (Entry.Entry.ContentType == ZenContentType::kCOUNT)
+ if (Entry.Entry.GetContentType() == ZenContentType::kCOUNT)
{
OutReason = fmt::format("Invalid content type for meta entry {}", Entry.BlobHash.ToHexString());
return false;
}
- if (Entry.Reserved1 != 0 || Entry.Reserved2 != 0 || Entry.Reserved3 != 0 || Entry.Reserved4 != 0)
- {
- OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString());
- return false;
- }
return true;
}
@@ -1114,31 +1019,76 @@ class BuildStoreGcReferenceChecker : public GcReferenceChecker
{
public:
BuildStoreGcReferenceChecker(BuildStore& Store) : m_Store(Store) {}
+ ~BuildStoreGcReferenceChecker()
+ {
+ try
+ {
+ m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedBlobKeys.reset(); });
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("~BuildStoreGcReferenceChecker threw exception: '{}'", Ex.what());
+ }
+ }
virtual std::string GetGcName(GcCtx& Ctx) override
{
ZEN_UNUSED(Ctx);
return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string());
}
- virtual void PreCache(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); }
-
- virtual void UpdateLockedState(GcCtx& Ctx) override
+ virtual void PreCache(GcCtx& Ctx) override
{
- ZEN_TRACE_CPU("Builds::UpdateLockedState");
- ZEN_MEMSCOPE(GetBuildstoreTag());
+ ZEN_TRACE_CPU("Builds::PreCache");
auto Log = [&Ctx]() { return Ctx.Logger; };
- m_References.reserve(m_Store.m_BlobLookup.size());
- for (const auto& It : m_Store.m_BlobLookup)
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: builds [PRECACHE] '{}': found {} references in {}",
+ m_Store.m_Config.RootDirectory,
+ m_PrecachedReferences.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
+
+ m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedBlobKeys = std::make_unique<std::vector<IoHash>>(); });
+
{
- const BuildStore::BlobIndex ExistingBlobIndex = It.second;
- if (m_Store.m_BlobEntries[ExistingBlobIndex].Payload)
+ m_PrecachedReferences.reserve(m_Store.m_BlobLookup.size());
+ RwLock::SharedLockScope __(m_Store.m_Lock);
+ for (const auto& It : m_Store.m_BlobLookup)
{
- m_References.push_back(It.first);
+ const BuildStore::BlobIndex ExistingBlobIndex = It.second;
+ const BuildStore::BlobEntry& Entry = m_Store.m_BlobEntries[ExistingBlobIndex];
+ if (Entry.Payload)
+ {
+ m_PrecachedReferences.push_back(It.first);
+ }
+ if (Entry.Metadata)
+ {
+ const BuildStore::MetadataEntry& MetadataEntry = m_Store.m_MetadataEntries[Entry.Metadata];
+ m_PrecachedReferences.push_back(MetadataEntry.MetadataHash);
+ }
}
}
- FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", "buildstore"), m_References);
+ FilterReferences(Ctx, fmt::format("buildstore [PRECACHE] '{}'", m_Store.m_Config.RootDirectory), m_PrecachedReferences);
+ }
+
+ virtual void UpdateLockedState(GcCtx& Ctx) override
+ {
+ ZEN_TRACE_CPU("Builds::UpdateLockedState");
+ ZEN_MEMSCOPE(GetBuildstoreTag());
+
+ auto Log = [&Ctx]() { return Ctx.Logger; };
+
+ ZEN_ASSERT(m_Store.m_TrackedBlobKeys);
+
+ m_AddedReferences = std::move(*m_Store.m_TrackedBlobKeys);
+
+ FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", m_Store.m_Config.RootDirectory), m_AddedReferences);
}
virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
@@ -1165,14 +1115,16 @@ public:
NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
- std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids);
+ std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_PrecachedReferences, IoCids);
+ UnusedReferences = KeepUnusedReferences(m_AddedReferences, UnusedReferences);
UsedCount = IoCids.size() - UnusedReferences.size();
return UnusedReferences;
}
private:
BuildStore& m_Store;
- std::vector<IoHash> m_References;
+ std::vector<IoHash> m_PrecachedReferences;
+ std::vector<IoHash> m_AddedReferences;
};
std::string
@@ -1184,201 +1136,6 @@ BuildStore::GetGcName(GcCtx& Ctx)
return fmt::format("buildstore: '{}'", m_Config.RootDirectory.string());
}
-class BuildStoreGcCompator : public GcStoreCompactor
-{
- using BlobEntry = BuildStore::BlobEntry;
- using PayloadEntry = BuildStore::PayloadEntry;
- using MetadataEntry = BuildStore::MetadataEntry;
- using MetadataDiskEntry = BuildStore::MetadataDiskEntry;
- using BlobIndex = BuildStore::BlobIndex;
- using PayloadIndex = BuildStore::PayloadIndex;
- using MetadataIndex = BuildStore::MetadataIndex;
-
-public:
- BuildStoreGcCompator(BuildStore& Store, std::vector<IoHash>&& RemovedBlobs) : m_Store(Store), m_RemovedBlobs(std::move(RemovedBlobs)) {}
-
- virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override
- {
- ZEN_UNUSED(ClaimDiskReserveCallback);
- ZEN_TRACE_CPU("Builds::CompactStore");
- ZEN_MEMSCOPE(GetBuildstoreTag());
-
- auto Log = [&Ctx]() { return Ctx.Logger; };
-
- Stopwatch Timer;
- const auto _ = MakeGuard([&] {
- if (!Ctx.Settings.Verbose)
- {
- return;
- }
- ZEN_INFO("GCV2: buildstore [COMPACT] '{}': RemovedDisk: {} in {}",
- m_Store.m_Config.RootDirectory,
- NiceBytes(Stats.RemovedDisk),
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- });
-
- const auto __ = MakeGuard([&] { m_Store.Flush(); });
-
- if (!m_RemovedBlobs.empty())
- {
- if (Ctx.Settings.CollectSmallObjects)
- {
- m_Store.m_Lock.WithExclusiveLock([this]() { m_Store.m_TrackedCacheKeys = std::make_unique<HashSet>(); });
- auto __ = MakeGuard([this]() { m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedCacheKeys.reset(); }); });
-
- BlockStore::BlockUsageMap BlockUsage;
- {
- RwLock::SharedLockScope __(m_Store.m_Lock);
-
- for (auto LookupIt : m_Store.m_BlobLookup)
- {
- const BlobIndex ReadBlobIndex = LookupIt.second;
- const BlobEntry& ReadBlobEntry = m_Store.m_BlobEntries[ReadBlobIndex];
-
- if (ReadBlobEntry.Metadata)
- {
- const MetadataEntry& ReadMetadataEntry = m_Store.m_MetadataEntries[ReadBlobEntry.Metadata];
-
- uint32_t BlockIndex = ReadMetadataEntry.Location.BlockIndex;
- uint64_t ChunkSize = RoundUp(ReadMetadataEntry.Location.Size, m_Store.m_Config.MetadataBlockStoreAlignement);
-
- if (auto BlockUsageIt = BlockUsage.find(BlockIndex); BlockUsageIt != BlockUsage.end())
- {
- BlockStore::BlockUsageInfo& Info = BlockUsageIt.value();
- Info.EntryCount++;
- Info.DiskUsage += ChunkSize;
- }
- else
- {
- BlockUsage.insert_or_assign(BlockIndex,
- BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
- }
- }
- }
- }
-
- BlockStore::BlockEntryCountMap BlocksToCompact = m_Store.m_MetadataBlockStore.GetBlocksToCompact(BlockUsage, 90);
- BlockStoreCompactState BlockCompactState;
- std::vector<IoHash> BlockCompactStateKeys;
- BlockCompactState.IncludeBlocks(BlocksToCompact);
-
- if (BlocksToCompact.size() > 0)
- {
- {
- RwLock::SharedLockScope ___(m_Store.m_Lock);
- for (const auto& Entry : m_Store.m_BlobLookup)
- {
- BlobIndex Index = Entry.second;
-
- if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta)
- {
- if (BlockCompactState.AddKeepLocation(m_Store.m_MetadataEntries[Meta].Location))
- {
- BlockCompactStateKeys.push_back(Entry.first);
- }
- }
- }
- }
-
- if (Ctx.Settings.IsDeleteMode)
- {
- if (Ctx.Settings.Verbose)
- {
- ZEN_INFO("GCV2: buildstore [COMPACT] '{}': compacting {} blocks",
- m_Store.m_Config.RootDirectory,
- BlocksToCompact.size());
- }
-
- m_Store.m_MetadataBlockStore.CompactBlocks(
- BlockCompactState,
- m_Store.m_Config.MetadataBlockStoreAlignement,
- [&](const BlockStore::MovedChunksArray& MovedArray,
- const BlockStore::ChunkIndexArray& ScrubbedArray,
- uint64_t FreedDiskSpace) {
- std::vector<MetadataDiskEntry> MovedEntries;
- MovedEntries.reserve(MovedArray.size());
- RwLock::ExclusiveLockScope _(m_Store.m_Lock);
- for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
- {
- size_t ChunkIndex = Moved.first;
- const IoHash& Key = BlockCompactStateKeys[ChunkIndex];
-
- ZEN_ASSERT(m_Store.m_TrackedCacheKeys);
- if (m_Store.m_TrackedCacheKeys->contains(Key))
- {
- continue;
- }
-
- if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end())
- {
- const BlobIndex Index = It->second;
-
- if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta)
- {
- m_Store.m_MetadataEntries[Meta].Location = Moved.second;
- MovedEntries.push_back(
- MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key});
- }
- }
- }
-
- for (size_t Scrubbed : ScrubbedArray)
- {
- const IoHash& Key = BlockCompactStateKeys[Scrubbed];
- if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end())
- {
- const BlobIndex Index = It->second;
-
- if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta)
- {
- MovedEntries.push_back(
- MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key});
- MovedEntries.back().Entry.Flags |= MetadataEntry::kTombStone;
- m_Store.m_MetadataEntries[Meta] = {};
- m_Store.m_BlobEntries[Index].Metadata = {};
- }
- }
- }
-
- m_Store.m_MetadatalogFile.Append(MovedEntries);
-
- Stats.RemovedDisk += FreedDiskSpace;
- if (Ctx.IsCancelledFlag.load())
- {
- return false;
- }
- return true;
- },
- ClaimDiskReserveCallback,
- fmt::format("GCV2: buildstore [COMPACT] '{}': ", m_Store.m_Config.RootDirectory));
- }
- else
- {
- if (Ctx.Settings.Verbose)
- {
- ZEN_INFO("GCV2: buildstore [COMPACT] '{}': skipped compacting of {} eligible blocks",
- m_Store.m_Config.RootDirectory,
- BlocksToCompact.size());
- }
- }
- }
- }
- }
- }
-
- virtual std::string GetGcName(GcCtx& Ctx) override
- {
- ZEN_UNUSED(Ctx);
- ZEN_MEMSCOPE(GetBuildstoreTag());
-
- return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string());
- }
-
-private:
- BuildStore& m_Store;
- const std::vector<IoHash> m_RemovedBlobs;
-};
-
GcStoreCompactor*
BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
{
@@ -1413,10 +1170,9 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
uint64_t BlobSize = 0;
};
- bool DiskSizeExceeded = false;
- const uint64_t CurrentDiskSize =
- m_LargeBlobStore.StorageSize().DiskSize + m_SmallBlobStore.StorageSize().DiskSize + m_MetadataBlockStore.TotalSize();
- if (CurrentDiskSize > m_Config.MaxDiskSpaceLimit)
+ bool DiskSizeExceeded = false;
+ const uint64_t CurrentBlobsDiskSize = m_BlobStore.TotalSize().TotalSize;
+ if ((m_Config.MaxDiskSpaceLimit > 0) && (CurrentBlobsDiskSize > m_Config.MaxDiskSpaceLimit))
{
DiskSizeExceeded = true;
}
@@ -1444,14 +1200,14 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
if (ReadBlobEntry.Metadata)
{
const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata];
- Size += Metadata.Location.Size;
+ Size += Metadata.GetSize();
}
const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime;
if (AccessTick < ExpireTicks)
{
ExpiredBlobs.push_back(It.first);
- ExpiredDataSize += ExpiredDataSize;
+ ExpiredDataSize += Size;
}
else if (DiskSizeExceeded)
{
@@ -1469,7 +1225,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
const uint64_t NewSizeLimit =
m_Config.MaxDiskSpaceLimit -
(m_Config.MaxDiskSpaceLimit >> 4); // Remove a bit more than just below the limit so we have some space to grow
- if ((CurrentDiskSize - ExpiredDataSize) > NewSizeLimit)
+ if ((CurrentBlobsDiskSize - ExpiredDataSize) > NewSizeLimit)
{
std::vector<size_t> NonExpiredOrder;
NonExpiredOrder.resize(NonExpiredBlobSizeInfos.size());
@@ -1487,7 +1243,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
while (It != NonExpiredOrder.end())
{
const SizeInfo& Info = NonExpiredBlobSizeInfos[*It];
- if ((CurrentDiskSize - ExpiredDataSize) < NewSizeLimit)
+ if ((CurrentBlobsDiskSize - ExpiredDataSize) < NewSizeLimit)
{
break;
}
@@ -1539,7 +1295,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
{
RemoveMetadatas.push_back(
MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = ExpiredBlob});
- RemoveMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone;
+ RemoveMetadatas.back().Entry.AddFlag(MetadataEntry::kTombStone);
m_MetadataEntries[ReadBlobEntry.Metadata] = {};
m_BlobEntries[ReadBlobIndex].Metadata = {};
}
@@ -1568,7 +1324,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
CompactState();
}
- return new BuildStoreGcCompator(*this, std::move(RemovedBlobs));
+ return nullptr;
}
std::vector<GcReferenceChecker*>
@@ -1595,21 +1351,6 @@ BuildStore::LockState(GcCtx& Ctx)
return Locks;
}
-void
-BuildStore::ScrubStorage(ScrubContext& ScrubCtx)
-{
- ZEN_UNUSED(ScrubCtx);
- // TODO
-}
-
-GcStorageSize
-BuildStore::StorageSize() const
-{
- GcStorageSize Result;
- Result.DiskSize = m_MetadataBlockStore.TotalSize();
- return Result;
-}
-
/*
___________ __
\__ ___/___ _______/ |_ ______
@@ -1630,8 +1371,10 @@ TEST_CASE("BuildStore.Blobs")
std::vector<IoHash> CompressedBlobsHashes;
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (size_t I = 0; I < 5; I++)
{
@@ -1658,10 +1401,13 @@ TEST_CASE("BuildStore.Blobs")
IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer();
CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
}
+ BlobStore.Flush();
}
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (const IoHash& RawHash : CompressedBlobsHashes)
{
IoBuffer Payload = Store.GetBlob(RawHash);
@@ -1689,8 +1435,10 @@ TEST_CASE("BuildStore.Blobs")
}
}
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (const IoHash& RawHash : CompressedBlobsHashes)
{
IoBuffer Payload = Store.GetBlob(RawHash);
@@ -1709,7 +1457,7 @@ TEST_CASE("BuildStore.Blobs")
}
namespace blockstore::testing {
- IoBuffer MakeMetaData(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues)
+ IoBuffer MakeMetadata(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues)
{
CbObjectWriter Writer;
Writer.AddHash("rawHash"sv, BlobHash);
@@ -1740,16 +1488,18 @@ TEST_CASE("BuildStore.Metadata")
std::vector<IoHash> BlobHashes;
std::vector<IoBuffer> MetaPayloads;
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (size_t I = 0; I < 5; I++)
{
BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I)));
- MetaPayloads.push_back(MakeMetaData(BlobHashes.back(), {{"index", fmt::format("{}", I)}}));
+ MetaPayloads.push_back(MakeMetadata(BlobHashes.back(), {{"index", fmt::format("{}", I)}}));
MetaPayloads.back().SetContentType(ZenContentType::kCbObject);
}
- Store.PutMetadatas(BlobHashes, MetaPayloads);
+ Store.PutMetadatas(BlobHashes, MetaPayloads, &WorkerPool);
std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool);
CHECK(ValidateMetaPayloads.size() == MetaPayloads.size());
@@ -1760,8 +1510,10 @@ TEST_CASE("BuildStore.Metadata")
}
}
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool);
CHECK(ValidateMetaPayloads.size() == MetaPayloads.size());
for (size_t I = 0; I < ValidateMetaPayloads.size(); I++)
@@ -1776,8 +1528,10 @@ TEST_CASE("BuildStore.Metadata")
}
std::vector<IoHash> CompressedBlobsHashes;
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (size_t I = 0; I < 5; I++)
{
IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7);
@@ -1805,14 +1559,16 @@ TEST_CASE("BuildStore.Metadata")
std::vector<IoBuffer> BlobMetaPayloads;
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (const IoHash& BlobHash : CompressedBlobsHashes)
{
- BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
+ BlobMetaPayloads.push_back(MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
}
- Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+ Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, &WorkerPool);
std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool);
CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
@@ -1824,8 +1580,10 @@ TEST_CASE("BuildStore.Metadata")
}
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool);
CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
@@ -1847,14 +1605,16 @@ TEST_CASE("BuildStore.Metadata")
for (const IoHash& BlobHash : CompressedBlobsHashes)
{
BlobMetaPayloads.push_back(
- MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}}));
+ MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}}));
BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
}
- Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+ Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, &WorkerPool);
}
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool);
CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
@@ -1886,8 +1646,10 @@ TEST_CASE("BuildStore.GC")
std::vector<IoHash> CompressedBlobsHashes;
std::vector<IoBuffer> BlobMetaPayloads;
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (size_t I = 0; I < 5; I++)
{
IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7);
@@ -1900,14 +1662,16 @@ TEST_CASE("BuildStore.GC")
}
for (const IoHash& BlobHash : CompressedBlobsHashes)
{
- BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
+ BlobMetaPayloads.push_back(MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
}
- Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+ Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, nullptr);
}
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
{
GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
@@ -1967,8 +1731,10 @@ TEST_CASE("BuildStore.SizeLimit")
std::vector<IoHash> CompressedBlobsHashes;
std::vector<IoBuffer> BlobMetaPayloads;
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
for (size_t I = 0; I < 64; I++)
{
IoBuffer Blob = CreateSemiRandomBlob(65537 + I * 7);
@@ -1981,10 +1747,10 @@ TEST_CASE("BuildStore.SizeLimit")
}
for (const IoHash& BlobHash : CompressedBlobsHashes)
{
- BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
+ BlobMetaPayloads.push_back(MakeMetadata(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
}
- Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+ Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads, nullptr);
{
for (size_t I = 0; I < 64; I++)
@@ -1997,8 +1763,10 @@ TEST_CASE("BuildStore.SizeLimit")
}
}
{
- GcManager Gc;
- BuildStore Store(Config, Gc);
+ GcManager Gc;
+ CidStore BlobStore(Gc);
+ BlobStore.Initialize({.RootDirectory = _.Path() / "build_cas"});
+ BuildStore Store(Config, Gc, BlobStore);
{
GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
@@ -2023,7 +1791,7 @@ TEST_CASE("BuildStore.SizeLimit")
CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash);
}
}
- CHECK(DeletedBlobs == 50);
+ CHECK(DeletedBlobs == 53);
std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr);
CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 15a1c9650..cacbbd966 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -233,6 +233,28 @@ using namespace std::literals;
namespace zen::cache::impl {
+static bool
+UpdateValueWithRawSizeAndHash(ZenCacheValue& Value)
+{
+ if ((Value.RawSize == 0) && (Value.RawHash == IoHash::Zero))
+ {
+ if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary)
+ {
+ return CompressedBuffer::ValidateCompressedHeader(Value.Value, Value.RawHash, Value.RawSize);
+ }
+ else
+ {
+ Value.RawSize = Value.Value.GetSize();
+ Value.RawHash = IoHash::HashBuffer(Value.Value);
+ return true;
+ }
+ }
+ else
+ {
+ return true;
+ }
+}
+
class BucketManifestSerializer
{
using MetaDataIndex = ZenCacheDiskLayer::CacheBucket::MetaDataIndex;
@@ -348,11 +370,20 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck
Stopwatch Timer;
const auto _ = MakeGuard([&] { ZEN_INFO("parsed store manifest '{}' in {}", ManifestPath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
- const uint64_t Count = Manifest["Count"sv].AsUInt64(0);
+ const uint64_t Count = Manifest["Count"sv].AsUInt64(0);
+ CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView();
+ if (KeyArray.Num() != Count)
+ {
+ ZEN_WARN("Mismatch in size between 'Keys' ({}) array size and 'Count' ({}) in {}, skipping metadata",
+ KeyArray.Num(),
+ Count,
+ ManifestPath);
+ return;
+ }
+
std::vector<PayloadIndex> KeysIndexes;
KeysIndexes.reserve(Count);
- CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView();
for (CbFieldView& KeyView : KeyArray)
{
if (auto It = Index.find(KeyView.AsHash()); It != Index.end())
@@ -367,19 +398,43 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck
size_t KeyIndexOffset = 0;
CbArrayView TimeStampArray = Manifest["Timestamps"].AsArrayView();
- for (CbFieldView& TimeStampView : TimeStampArray)
+ if (KeysIndexes.size() != TimeStampArray.Num())
{
- const PayloadIndex KeyIndex = KeysIndexes[KeyIndexOffset++];
- if (KeyIndex)
+ ZEN_WARN("Mismatch in size between 'Keys' ({}) and 'Timestamps' ({}) arrays in {}, skipping timestamps",
+ KeysIndexes.size(),
+ TimeStampArray.Num(),
+ ManifestPath);
+ }
+ else
+ {
+ for (CbFieldView& TimeStampView : TimeStampArray)
{
- AccessTimes[KeyIndex] = TimeStampView.AsInt64();
+ const PayloadIndex KeyIndex = KeysIndexes[KeyIndexOffset++];
+ if (KeyIndex)
+ {
+ AccessTimes[KeyIndex] = TimeStampView.AsInt64();
+ }
}
}
KeyIndexOffset = 0;
CbArrayView RawHashArray = Manifest["RawHash"].AsArrayView();
CbArrayView RawSizeArray = Manifest["RawSize"].AsArrayView();
- if (RawHashArray.Num() == RawSizeArray.Num())
+ if (RawHashArray.Num() != KeysIndexes.size())
+ {
+ ZEN_WARN("Mismatch in size between 'Keys' ({}) and 'RawHash' ({}) arrays in {}, skipping meta data",
+ KeysIndexes.size(),
+ RawHashArray.Num(),
+ ManifestPath);
+ }
+ else if (RawSizeArray.Num() != KeysIndexes.size())
+ {
+ ZEN_WARN("Mismatch in size between 'Keys' ({}) and 'RawSize' ({}) arrays in {}, skipping meta data",
+ KeysIndexes.size(),
+ RawSizeArray.Num(),
+ ManifestPath);
+ }
+ else
{
auto RawHashIt = RawHashArray.CreateViewIterator();
auto RawSizeIt = RawSizeArray.CreateViewIterator();
@@ -404,10 +459,6 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck
RawSizeIt++;
}
}
- else
- {
- ZEN_WARN("Mismatch in size between 'RawHash' and 'RawSize' arrays in {}, skipping meta data", ManifestPath);
- }
}
Oid
@@ -747,6 +798,7 @@ ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager& Gc,
m_Configuration.LargeObjectThreshold = Max(m_Configuration.LargeObjectThreshold, IoStoreDDCOverrideSize);
}
m_Gc.AddGcReferencer(*this);
+ m_Gc.AddGcStorage(this);
}
ZenCacheDiskLayer::CacheBucket::~CacheBucket()
@@ -761,6 +813,7 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket()
{
ZEN_ERROR("~CacheBucket() failed with: ", Ex.what());
}
+ m_Gc.RemoveGcStorage(this);
m_Gc.RemoveGcReferencer(*this);
}
@@ -1286,20 +1339,21 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc,
struct ZenCacheDiskLayer::CacheBucket::PutBatchHandle
{
- PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {}
+ PutBatchHandle(std::vector<ZenCacheDiskLayer::PutResult>& OutResults) : OutResults(OutResults) {}
struct Entry
{
std::vector<IoHash> HashKeyAndReferences;
+ bool Overwrite;
};
- std::vector<IoBuffer> Buffers;
- std::vector<Entry> Entries;
- std::vector<size_t> EntryResultIndexes;
+ std::vector<ZenCacheValue> Buffers;
+ std::vector<Entry> Entries;
+ std::vector<size_t> EntryResultIndexes;
- std::vector<bool>& OutResults;
+ std::vector<ZenCacheDiskLayer::PutResult>& OutResults;
};
ZenCacheDiskLayer::CacheBucket::PutBatchHandle*
-ZenCacheDiskLayer::CacheBucket::BeginPutBatch(std::vector<bool>& OutResults)
+ZenCacheDiskLayer::CacheBucket::BeginPutBatch(std::vector<ZenCacheDiskLayer::PutResult>& OutResults)
{
ZEN_TRACE_CPU("Z$::Bucket::BeginPutBatch");
return new PutBatchHandle(OutResults);
@@ -1315,23 +1369,40 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept
ZEN_ASSERT(Batch);
if (!Batch->Buffers.empty())
{
- std::vector<uint8_t> EntryFlags;
- for (const IoBuffer& Buffer : Batch->Buffers)
+ ZEN_ASSERT(Batch->Buffers.size() == Batch->Entries.size());
+ std::vector<uint8_t> EntryFlags;
+ std::vector<size_t> BufferToEntryIndexes;
+ std::vector<IoBuffer> BuffersToCommit;
+ BuffersToCommit.reserve(Batch->Buffers.size());
+ for (size_t Index = 0; Index < Batch->Entries.size(); Index++)
{
- uint8_t Flags = 0;
- if (Buffer.GetContentType() == ZenContentType::kCbObject)
+ const std::vector<IoHash>& HashKeyAndReferences = Batch->Entries[Index].HashKeyAndReferences;
+ ZEN_ASSERT(HashKeyAndReferences.size() >= 1);
+
+ ZenCacheValue& Value = Batch->Buffers[Index];
+ std::span<const IoHash> ReferenceSpan(HashKeyAndReferences.begin() + 1, HashKeyAndReferences.end());
+ PutResult& OutResult = Batch->OutResults[Batch->EntryResultIndexes[Index]];
+ OutResult = PutResult{zen::PutStatus::Success};
+ if (!ShouldRejectPut(HashKeyAndReferences[0], Value, Batch->Entries[Index].Overwrite, OutResult))
{
- Flags |= DiskLocation::kStructured;
- }
- else if (Buffer.GetContentType() == ZenContentType::kCompressedBinary)
- {
- Flags |= DiskLocation::kCompressed;
+ BufferToEntryIndexes.push_back(Index);
+ BuffersToCommit.push_back(Value.Value);
+
+ uint8_t Flags = 0;
+ if (Value.Value.GetContentType() == ZenContentType::kCbObject)
+ {
+ Flags |= DiskLocation::kStructured;
+ }
+ else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary)
+ {
+ Flags |= DiskLocation::kCompressed;
+ }
+ EntryFlags.push_back(Flags);
}
- EntryFlags.push_back(Flags);
}
size_t IndexOffset = 0;
- m_BlockStore.WriteChunks(Batch->Buffers, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) {
+ m_BlockStore.WriteChunks(BuffersToCommit, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) {
ZEN_MEMSCOPE(GetCacheDiskTag());
std::vector<DiskIndexEntry> DiskEntries;
{
@@ -1339,8 +1410,9 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept
for (size_t Index = 0; Index < Locations.size(); Index++)
{
DiskLocation Location(Locations[Index], m_Configuration.PayloadAlignment, EntryFlags[IndexOffset + Index]);
- const std::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences;
- ZEN_ASSERT(HashKeyAndReferences.size() > 1);
+ const std::vector<IoHash>& HashKeyAndReferences =
+ Batch->Entries[BufferToEntryIndexes[IndexOffset + Index]].HashKeyAndReferences;
+ ZEN_ASSERT(HashKeyAndReferences.size() >= 1);
const IoHash HashKey = HashKeyAndReferences[0];
DiskEntries.push_back({.Key = HashKey, .Location = Location});
if (m_TrackedCacheKeys)
@@ -1375,12 +1447,6 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept
}
}
m_SlogFile.Append(DiskEntries);
- for (size_t Index = 0; Index < Locations.size(); Index++)
- {
- size_t ResultIndex = Batch->EntryResultIndexes[IndexOffset + Index];
- ZEN_ASSERT(ResultIndex < Batch->OutResults.size());
- Batch->OutResults[ResultIndex] = true;
- }
IndexOffset += Locations.size();
});
}
@@ -1876,30 +1942,128 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
}
}
-void
+bool
+ZenCacheDiskLayer::CacheBucket::ShouldRejectPut(const IoHash& HashKey,
+ ZenCacheValue& InOutValue,
+ bool Overwrite,
+ ZenCacheDiskLayer::PutResult& OutPutResult)
+{
+ const bool CheckExisting = m_Configuration.LimitOverwrites && !Overwrite;
+ if (CheckExisting)
+ {
+ RwLock::SharedLockScope IndexLock(m_IndexLock);
+ auto It = m_Index.find(HashKey);
+ if (It != m_Index.end())
+ {
+ const PayloadIndex EntryIndex = It.value();
+ m_AccessTimes[EntryIndex] = GcClock::TickCount();
+ const DiskLocation Location = m_Payloads[EntryIndex].Location;
+
+ const BucketPayload* Payload = &m_Payloads[EntryIndex];
+ if (Payload->MetaData)
+ {
+ const BucketMetaData MetaData = m_MetaDatas[Payload->MetaData];
+ if (MetaData)
+ {
+ IndexLock.ReleaseNow();
+ if (!cache::impl::UpdateValueWithRawSizeAndHash(InOutValue))
+ {
+ OutPutResult = PutResult{zen::PutStatus::Fail, "Value provided is of bad format"};
+ return true;
+ }
+ else if (MetaData.RawSize != InOutValue.RawSize || MetaData.RawHash != InOutValue.RawHash)
+ {
+ OutPutResult = PutResult{
+ zen::PutStatus::Conflict,
+ fmt::format("Value exists with different size '{}' or hash '{}'", MetaData.RawSize, MetaData.RawHash)};
+ return true;
+ }
+ return false;
+ }
+ }
+
+ ZenCacheValue ExistingValue;
+ if (Payload->MemCached)
+ {
+ ExistingValue.Value = m_MemCachedPayloads[Payload->MemCached].Payload;
+ IndexLock.ReleaseNow();
+ }
+ else
+ {
+ IndexLock.ReleaseNow();
+
+ if (Location.IsFlagSet(DiskLocation::kStandaloneFile))
+ {
+ ExistingValue.Value = GetStandaloneCacheValue(Location, HashKey);
+ }
+ else
+ {
+ ExistingValue.Value = GetInlineCacheValue(Location);
+ }
+ }
+
+ if (ExistingValue.Value)
+ {
+ if (cache::impl::UpdateValueWithRawSizeAndHash(ExistingValue))
+ {
+ if (!cache::impl::UpdateValueWithRawSizeAndHash(InOutValue))
+ {
+ OutPutResult = PutResult{zen::PutStatus::Fail, "Value provided is of bad format"};
+ return true;
+ }
+
+ if (ExistingValue.RawSize != InOutValue.RawSize || ExistingValue.RawHash != InOutValue.RawHash)
+ {
+ OutPutResult = PutResult{zen::PutStatus::Conflict,
+ fmt::format("Value exists with different size '{}' or hash '{}'",
+ ExistingValue.RawSize,
+ ExistingValue.RawHash)};
+ return true;
+ }
+ }
+ }
+ }
+ }
+ return false;
+}
+
+ZenCacheDiskLayer::PutResult
ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey,
const ZenCacheValue& Value,
std::span<IoHash> References,
+ bool Overwrite,
PutBatchHandle* OptionalBatchHandle)
{
ZEN_TRACE_CPU("Z$::Bucket::Put");
metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
+ PutResult Result{zen::PutStatus::Success};
+
if (Value.Value.Size() >= m_Configuration.LargeObjectThreshold)
{
- PutStandaloneCacheValue(HashKey, Value, References);
+ ZenCacheValue AcceptedValue = Value;
+ if (ShouldRejectPut(HashKey, AcceptedValue, Overwrite, Result))
+ {
+ if (OptionalBatchHandle)
+ {
+ OptionalBatchHandle->OutResults.push_back(Result);
+ }
+ return Result;
+ }
+ PutStandaloneCacheValue(HashKey, AcceptedValue, References);
if (OptionalBatchHandle)
{
- OptionalBatchHandle->OutResults.push_back(true);
+ OptionalBatchHandle->OutResults.push_back({zen::PutStatus::Success});
}
}
else
{
- PutInlineCacheValue(HashKey, Value, References, OptionalBatchHandle);
+ Result = PutInlineCacheValue(HashKey, Value, References, Overwrite, OptionalBatchHandle);
}
m_DiskWriteCount++;
+ return Result;
}
uint64_t
@@ -2425,7 +2589,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
ZenCacheDiskLayer::BucketStats
ZenCacheDiskLayer::CacheBucket::Stats()
{
- GcStorageSize Size = StorageSize();
+ CacheStoreSize Size = TotalSize();
return ZenCacheDiskLayer::BucketStats{.DiskSize = Size.DiskSize,
.MemorySize = Size.MemorySize,
.DiskHitCount = m_DiskHitCount,
@@ -2748,38 +2912,49 @@ ZenCacheDiskLayer::CacheBucket::GetMetaData(RwLock::SharedLockScope&, const Buck
return {};
}
-void
+ZenCacheDiskLayer::PutResult
ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey,
const ZenCacheValue& Value,
std::span<IoHash> References,
+ bool Overwrite,
PutBatchHandle* OptionalBatchHandle)
{
ZEN_TRACE_CPU("Z$::Bucket::PutInlineCacheValue");
+ PutResult Result{zen::PutStatus::Success};
if (OptionalBatchHandle != nullptr)
{
- OptionalBatchHandle->Buffers.push_back(Value.Value);
+ OptionalBatchHandle->Buffers.push_back(Value);
OptionalBatchHandle->Entries.push_back({});
OptionalBatchHandle->EntryResultIndexes.push_back(OptionalBatchHandle->OutResults.size());
- OptionalBatchHandle->OutResults.push_back(false);
+ OptionalBatchHandle->OutResults.push_back(PutResult{zen::PutStatus::Fail});
+ PutBatchHandle::Entry& CurrentEntry = OptionalBatchHandle->Entries.back();
+ CurrentEntry.Overwrite = Overwrite;
std::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences;
- HashKeyAndReferences.reserve(1 + HashKeyAndReferences.size());
+ HashKeyAndReferences.reserve(1 + References.size());
HashKeyAndReferences.push_back(HashKey);
- HashKeyAndReferences.insert(HashKeyAndReferences.end(), HashKeyAndReferences.begin(), HashKeyAndReferences.end());
- return;
+ HashKeyAndReferences.insert(HashKeyAndReferences.end(), References.begin(), References.end());
+ return Result;
}
+
+ ZenCacheValue AcceptedValue = Value;
+ if (ShouldRejectPut(HashKey, AcceptedValue, Overwrite, Result))
+ {
+ return Result;
+ }
+
uint8_t EntryFlags = 0;
- if (Value.Value.GetContentType() == ZenContentType::kCbObject)
+ if (AcceptedValue.Value.GetContentType() == ZenContentType::kCbObject)
{
EntryFlags |= DiskLocation::kStructured;
}
- else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary)
+ else if (AcceptedValue.Value.GetContentType() == ZenContentType::kCompressedBinary)
{
EntryFlags |= DiskLocation::kCompressed;
}
- m_BlockStore.WriteChunk(Value.Value.Data(),
- Value.Value.Size(),
+ m_BlockStore.WriteChunk(AcceptedValue.Value.Data(),
+ AcceptedValue.Value.Size(),
m_Configuration.PayloadAlignment,
[&](const BlockStoreLocation& BlockStoreLocation) {
ZEN_MEMSCOPE(GetCacheDiskTag());
@@ -2816,6 +2991,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey,
}
m_SlogFile.Append({.Key = HashKey, .Location = Location});
});
+ return Result;
}
std::string
@@ -3752,7 +3928,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket)
struct ZenCacheDiskLayer::PutBatchHandle
{
- PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {}
+ PutBatchHandle(std::vector<ZenCacheDiskLayer::PutResult>& OutResults) : OutResults(OutResults) {}
struct BucketHandle
{
CacheBucket* Bucket;
@@ -3811,13 +3987,13 @@ struct ZenCacheDiskLayer::PutBatchHandle
return NewBucketHandle;
}
- RwLock Lock;
- std::vector<BucketHandle> BucketHandles;
- std::vector<bool>& OutResults;
+ RwLock Lock;
+ std::vector<BucketHandle> BucketHandles;
+ std::vector<ZenCacheDiskLayer::PutResult>& OutResults;
};
ZenCacheDiskLayer::PutBatchHandle*
-ZenCacheDiskLayer::BeginPutBatch(std::vector<bool>& OutResults)
+ZenCacheDiskLayer::BeginPutBatch(std::vector<PutResult>& OutResults)
{
return new PutBatchHandle(OutResults);
}
@@ -3954,21 +4130,23 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, GetBatc
}
}
-void
+ZenCacheDiskLayer::PutResult
ZenCacheDiskLayer::Put(std::string_view InBucket,
const IoHash& HashKey,
const ZenCacheValue& Value,
std::span<IoHash> References,
+ bool Overwrite,
PutBatchHandle* OptionalBatchHandle)
{
ZEN_TRACE_CPU("Z$::Put");
-
+ PutResult RetVal = {zen::PutStatus::Fail};
if (CacheBucket* Bucket = GetOrCreateBucket(InBucket); Bucket != nullptr)
{
CacheBucket::PutBatchHandle* BucketBatchHandle = OptionalBatchHandle == nullptr ? nullptr : OptionalBatchHandle->GetHandle(Bucket);
- Bucket->Put(HashKey, Value, References, BucketBatchHandle);
+ RetVal = Bucket->Put(HashKey, Value, References, Overwrite, BucketBatchHandle);
TryMemCacheTrim();
}
+ return RetVal;
}
void
@@ -4241,8 +4419,9 @@ ZenCacheDiskLayer::Flush()
}
}
+#if ZEN_WITH_TESTS
void
-ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
+ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
{
ZEN_TRACE_CPU("Z$::ScrubStorage");
@@ -4253,13 +4432,13 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
for (auto& Kv : m_Buckets)
{
-#if 1
+# if 1
Results.push_back(Ctx.ThreadPool().EnqueueTask(
std::packaged_task<void()>{[this, Bucket = Kv.second.get(), &Ctx] { Bucket->ScrubStorage(Ctx); }}));
-#else
+# else
CacheBucket& Bucket = *Kv.second;
Bucket.ScrubStorage(Ctx);
-#endif
+# endif
}
for (auto& Result : Results)
@@ -4275,16 +4454,17 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
}
}
}
+#endif // ZEN_WITH_TESTS
-GcStorageSize
-ZenCacheDiskLayer::StorageSize() const
+CacheStoreSize
+ZenCacheDiskLayer::TotalSize() const
{
- GcStorageSize StorageSize{};
+ CacheStoreSize StorageSize{};
RwLock::SharedLockScope _(m_Lock);
for (auto& Kv : m_Buckets)
{
- GcStorageSize BucketSize = Kv.second->StorageSize();
+ CacheStoreSize BucketSize = Kv.second->TotalSize();
StorageSize.DiskSize += BucketSize.DiskSize;
StorageSize.MemorySize += BucketSize.MemorySize;
}
@@ -4295,7 +4475,7 @@ ZenCacheDiskLayer::StorageSize() const
ZenCacheDiskLayer::DiskStats
ZenCacheDiskLayer::Stats() const
{
- GcStorageSize Size = StorageSize();
+ CacheStoreSize Size = TotalSize();
ZenCacheDiskLayer::DiskStats Stats = {.DiskSize = Size.DiskSize, .MemorySize = Size.MemorySize};
{
RwLock::SharedLockScope _(m_Lock);
@@ -4319,7 +4499,7 @@ ZenCacheDiskLayer::GetInfo() const
{
Info.BucketNames.push_back(Kv.first);
Info.EntryCount += Kv.second->EntryCount();
- GcStorageSize BucketSize = Kv.second->StorageSize();
+ CacheStoreSize BucketSize = Kv.second->TotalSize();
Info.StorageSize.DiskSize += BucketSize.DiskSize;
Info.StorageSize.MemorySize += BucketSize.MemorySize;
}
@@ -4334,7 +4514,7 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
{
- return ZenCacheDiskLayer::BucketInfo{.EntryCount = It->second->EntryCount(), .StorageSize = It->second->StorageSize()};
+ return ZenCacheDiskLayer::BucketInfo{.EntryCount = It->second->EntryCount(), .StorageSize = It->second->TotalSize()};
}
return {};
}
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index de4b0a37c..ff21d1ede 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -153,13 +153,13 @@ CacheRpcHandler::CacheRpcHandler(LoggerRef InLog,
CacheStats& InCacheStats,
UpstreamCacheClient& InUpstreamCache,
ZenCacheStore& InCacheStore,
- CidStore& InCidStore,
+ GetCidStoreFunc&& InGetCidStore,
const DiskWriteBlocker* InDiskWriteBlocker)
: m_Log(InLog)
, m_CacheStats(InCacheStats)
, m_UpstreamCache(InUpstreamCache)
, m_CacheStore(InCacheStore)
-, m_CidStore(InCidStore)
+, m_GetCidStore(std::move(InGetCidStore))
, m_DiskWriteBlocker(InDiskWriteBlocker)
{
}
@@ -174,6 +174,12 @@ CacheRpcHandler::AreDiskWritesAllowed() const
return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed());
}
+CidStore&
+CacheRpcHandler::GetCidStore(std::string_view Namespace)
+{
+ return m_GetCidStore(Namespace);
+}
+
CacheRpcHandler::RpcResponseCode
CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context,
std::string_view UriNamespace,
@@ -334,13 +340,13 @@ CacheRpcHandler::HandleRpcPutCacheRecords(const CacheRequestContext& Context, co
.Policy = std::move(Policy),
.Context = Context};
- PutResult Result = PutCacheRecord(PutRequest, &BatchRequest);
+ PutStatus Result = PutCacheRecord(PutRequest, &BatchRequest);
- if (Result == PutResult::Invalid)
+ if (Result == PutStatus::Invalid)
{
return CbPackage{};
}
- Results.push_back(Result == PutResult::Success);
+ Results.push_back(Result == PutStatus::Success);
}
if (Results.empty())
{
@@ -360,7 +366,7 @@ CacheRpcHandler::HandleRpcPutCacheRecords(const CacheRequestContext& Context, co
return RpcResponse;
}
-PutResult
+PutStatus
CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Package)
{
CbObjectView Record = Request.RecordObject;
@@ -381,9 +387,12 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
Stopwatch Timer;
+ CidStore& ChunkStore = m_GetCidStore(Request.Namespace);
+
Request.RecordObject.IterateAttachments([this,
&Request,
Package,
+ &ChunkStore,
&WriteAttachmentBuffers,
&WriteRawHashes,
&ValidAttachments,
@@ -412,7 +421,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
Count.Invalid++;
}
}
- else if (m_CidStore.ContainsChunk(ValueHash))
+ else if (ChunkStore.ContainsChunk(ValueHash))
{
ValidAttachments.emplace_back(ValueHash);
Count.Valid++;
@@ -422,19 +431,33 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
if (Count.Invalid > 0)
{
- return PutResult::Invalid;
+ return PutStatus::Invalid;
}
ZenCacheValue CacheValue;
CacheValue.Value = IoBuffer(Record.GetSize());
Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize()));
CacheValue.Value.SetContentType(ZenContentType::kCbObject);
- m_CacheStore.Put(Request.Context, Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue, ReferencedAttachments, nullptr);
+ bool Overwrite = EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::StoreLocal) &&
+ !EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::QueryLocal);
+ // TODO: Propagation for rejected PUTs
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Request.Context,
+ Request.Namespace,
+ Request.Key.Bucket,
+ Request.Key.Hash,
+ CacheValue,
+ ReferencedAttachments,
+ Overwrite,
+ nullptr);
+ if (PutResult.Status != zen::PutStatus::Success)
+ {
+ return PutResult.Status;
+ }
m_CacheStats.WriteCount++;
if (!WriteAttachmentBuffers.empty())
{
- std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
+ std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
for (size_t Index = 0; Index < InsertResults.size(); Index++)
{
if (InsertResults[Index].New)
@@ -461,12 +484,14 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
if (HasUpstream && EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::StoreRemote) && !IsPartialRecord)
{
- m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage,
- .Namespace = Request.Namespace,
- .Key = Request.Key,
- .ValueContentIds = std::move(ValidAttachments)});
+ m_UpstreamCache.EnqueueUpstream(
+ {.Type = ZenContentType::kCbPackage,
+ .Namespace = Request.Namespace,
+ .Key = Request.Key,
+ .ValueContentIds = std::move(ValidAttachments)},
+ [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
}
- return PutResult::Success;
+ return PutStatus::Success;
}
CbPackage
@@ -507,6 +532,8 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
return CbPackage{};
}
+ CidStore& ChunkStore = m_GetCidStore(Namespace.value());
+
const bool HasUpstream = m_UpstreamCache.IsActive();
eastl::fixed_vector<RecordRequestData, 16> Requests;
@@ -606,7 +633,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
else if (EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData))
{
- if (m_CidStore.ContainsChunk(Value.ContentId))
+ if (ChunkStore.ContainsChunk(Value.ContentId))
{
Value.Exists = true;
}
@@ -627,7 +654,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
else
{
- if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Value.ContentId))
+ if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Value.ContentId))
{
if (Chunk.GetSize() > 0)
{
@@ -651,7 +678,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
if (!RequestValueIndexes.empty())
{
- m_CidStore.IterateChunks(
+ ChunkStore.IterateChunks(
CidHashes,
[this, &Request, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool {
try
@@ -744,7 +771,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
}
- const auto OnCacheRecordGetComplete = [this, Namespace, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) {
+ const auto OnCacheRecordGetComplete = [this, Namespace, &ChunkStore, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) {
if (!Params.Record)
{
return;
@@ -765,18 +792,24 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal) && AreDiskWritesAllowed();
if (StoreLocal)
{
+ bool Overwrite = !EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::QueryLocal);
std::vector<IoHash> ReferencedAttachments;
ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
const IoHash ValueHash = HashView.AsHash();
ReferencedAttachments.push_back(ValueHash);
});
- m_CacheStore.Put(Context,
- *Namespace,
- Key.Bucket,
- Key.Hash,
- {.Value = {Request.RecordCacheValue}},
- ReferencedAttachments,
- nullptr);
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Context,
+ *Namespace,
+ Key.Bucket,
+ Key.Hash,
+ {.Value = {Request.RecordCacheValue}},
+ ReferencedAttachments,
+ Overwrite,
+ nullptr);
+ if (PutResult.Status != zen::PutStatus::Success)
+ {
+ return;
+ }
m_CacheStats.WriteCount++;
}
ParseValues(Request);
@@ -807,7 +840,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
Value.Exists = true;
if (StoreLocal)
{
- m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash());
+ ChunkStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash());
}
if (!EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData))
{
@@ -924,10 +957,12 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con
const bool HasUpstream = m_UpstreamCache.IsActive();
CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
- std::vector<bool> BatchResults;
- eastl::fixed_vector<size_t, 32> BatchResultIndexes;
- eastl::fixed_vector<bool, 32> Results;
- eastl::fixed_vector<CacheKey, 32> UpstreamCacheKeys;
+ CidStore& ChunkStore = m_GetCidStore(Namespace.value());
+
+ std::vector<ZenCacheStore::PutResult> BatchResults;
+ eastl::fixed_vector<size_t, 32> BatchResultIndexes;
+ eastl::fixed_vector<ZenCacheStore::PutResult, 32> Results;
+ eastl::fixed_vector<CacheKey, 32> UpstreamCacheKeys;
uint64_t RequestCount = RequestsArray.Num();
{
@@ -977,34 +1012,39 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con
if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal))
{
- IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer();
+ bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal);
+ IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer();
Value.SetContentType(ZenContentType::kCompressedBinary);
if (RawSize == 0)
{
RawSize = Chunk.DecodeRawSize();
}
- m_CacheStore.Put(Context,
- *Namespace,
- Key.Bucket,
- Key.Hash,
- {.Value = Value, .RawSize = RawSize, .RawHash = RawHash},
- {},
- Batch.get());
- m_CacheStats.WriteCount++;
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Context,
+ *Namespace,
+ Key.Bucket,
+ Key.Hash,
+ {.Value = Value, .RawSize = RawSize, .RawHash = RawHash},
+ {},
+ Overwrite,
+ Batch.get());
+ if (PutResult.Status == zen::PutStatus::Success)
+ {
+ m_CacheStats.WriteCount++;
+ }
if (Batch)
{
BatchResultIndexes.push_back(Results.size());
- Results.push_back(false);
+ Results.push_back({zen::PutStatus::Fail});
}
else
{
- Results.push_back(true);
+ Results.push_back(PutResult);
}
TransferredSize = Chunk.GetCompressedSize();
}
else
{
- Results.push_back(true);
+ Results.push_back({zen::PutStatus::Success});
}
Valid = true;
}
@@ -1020,12 +1060,12 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con
if (m_CacheStore.Get(Context, *Namespace, Key.Bucket, Key.Hash, ExistingValue) &&
IsCompressedBinary(ExistingValue.Value.GetContentType()))
{
- Results.push_back(true);
+ Results.push_back({zen::PutStatus::Success});
Valid = true;
}
else
{
- Results.push_back(false);
+ Results.push_back({zen::PutStatus::Fail, fmt::format("Missing attachment with raw hash {}", RawHash)});
}
}
// We do not search the Upstream. No data in a put means the caller is probing for whether they need to do a heavy put.
@@ -1060,27 +1100,49 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con
{
size_t BatchResultIndex = BatchResultIndexes[Index];
ZEN_ASSERT(BatchResultIndex < Results.size());
- ZEN_ASSERT(Results[BatchResultIndex] == false);
+ ZEN_ASSERT(Results[BatchResultIndex].Status != zen::PutStatus::Success);
Results[BatchResultIndex] = BatchResults[Index];
}
for (std::size_t Index = 0; Index < Results.size(); Index++)
{
- if (Results[Index] && UpstreamCacheKeys[Index] != CacheKey::Empty)
+ if ((Results[Index].Status == zen::PutStatus::Success) && UpstreamCacheKeys[Index] != CacheKey::Empty)
{
m_UpstreamCache.EnqueueUpstream(
- {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]});
+ {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]},
+ [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
}
}
{
ZEN_TRACE_CPU("Z$::RpcPutCacheValues::Response");
CbObjectWriter ResponseObject{1024};
ResponseObject.BeginArray("Result"sv);
- for (bool Value : Results)
+ bool bAnyErrors = false;
+ for (const ZenCacheStore::PutResult& Value : Results)
{
- ResponseObject.AddBool(Value);
+ if (Value.Status == zen::PutStatus::Success)
+ {
+ ResponseObject.AddBool(true);
+ }
+ else
+ {
+ bAnyErrors = true;
+ ResponseObject.AddBool(false);
+ }
}
ResponseObject.EndArray();
+ if (bAnyErrors)
+ {
+ ResponseObject.BeginArray("ErrorMessages"sv);
+ for (const ZenCacheStore::PutResult& Value : Results)
+ {
+ if (Value.Status != zen::PutStatus::Success)
+ {
+ ResponseObject.AddString(Value.Message);
+ }
+ }
+ ResponseObject.EndArray();
+ }
CbPackage RpcResponse;
RpcResponse.SetObject(ResponseObject.Save());
@@ -1239,6 +1301,7 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO
const bool HasData = IsCompressedBinary(Params.Value.GetContentType());
const bool SkipData = EnumHasAllFlags(Request.Policy, CachePolicy::SkipData);
const bool StoreData = EnumHasAllFlags(Request.Policy, CachePolicy::StoreLocal) && AreDiskWritesAllowed();
+ const bool Overwrite = StoreData && !EnumHasAllFlags(Request.Policy, CachePolicy::QueryLocal);
const bool IsHit = SkipData || HasData;
if (IsHit)
{
@@ -1249,14 +1312,19 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO
if (HasData && StoreData)
{
- m_CacheStore.Put(Context,
- *Namespace,
- Request.Key.Bucket,
- Request.Key.Hash,
- ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash},
- {},
- nullptr);
- m_CacheStats.WriteCount++;
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(
+ Context,
+ *Namespace,
+ Request.Key.Bucket,
+ Request.Key.Hash,
+ ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash},
+ {},
+ Overwrite,
+ nullptr);
+ if (PutResult.Status == zen::PutStatus::Success)
+ {
+ m_CacheStats.WriteCount++;
+ }
}
ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}",
@@ -1494,6 +1562,8 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
using namespace cache::detail;
const bool HasUpstream = m_UpstreamCache.IsActive();
+ CidStore& ChunkStore = m_GetCidStore(Namespace);
+
// TODO: BatchGet records?
std::vector<CacheKeyRequest*> UpstreamRecordRequests;
for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex)
@@ -1527,36 +1597,48 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
if (!UpstreamRecordRequests.empty())
{
- const auto OnCacheRecordGetComplete = [this, Namespace, &RecordKeys, &Records, &RecordRequests, Context](
- CacheRecordGetCompleteParams&& Params) {
- if (!Params.Record)
- {
- return;
- }
- CacheKeyRequest& RecordKey = Params.Request;
- size_t RecordIndex = std::distance(RecordKeys.data(), &RecordKey);
- RecordRequests[RecordIndex]->ElapsedTimeUs += static_cast<uint64_t>(Params.ElapsedSeconds * 1000000.0);
- RecordBody& Record = Records[RecordIndex];
-
- const CacheKey& Key = RecordKey.Key;
- Record.Exists = true;
- CbObject ObjectBuffer = CbObject::Clone(Params.Record);
- Record.CacheValue = ObjectBuffer.GetBuffer().AsIoBuffer();
- Record.CacheValue.SetContentType(ZenContentType::kCbObject);
- Record.Source = Params.Source;
-
- bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed();
- if (StoreLocal)
- {
- std::vector<IoHash> ReferencedAttachments;
- ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
- const IoHash ValueHash = HashView.AsHash();
- ReferencedAttachments.push_back(ValueHash);
- });
- m_CacheStore.Put(Context, Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}, ReferencedAttachments, nullptr);
- m_CacheStats.WriteCount++;
- }
- };
+ const auto OnCacheRecordGetComplete =
+ [this, Namespace, &RecordKeys, &Records, &RecordRequests, Context](CacheRecordGetCompleteParams&& Params) {
+ if (!Params.Record)
+ {
+ return;
+ }
+ CacheKeyRequest& RecordKey = Params.Request;
+ size_t RecordIndex = std::distance(RecordKeys.data(), &RecordKey);
+ RecordRequests[RecordIndex]->ElapsedTimeUs += static_cast<uint64_t>(Params.ElapsedSeconds * 1000000.0);
+ RecordBody& Record = Records[RecordIndex];
+
+ const CacheKey& Key = RecordKey.Key;
+ Record.Exists = true;
+ CbObject ObjectBuffer = CbObject::Clone(Params.Record);
+ Record.CacheValue = ObjectBuffer.GetBuffer().AsIoBuffer();
+ Record.CacheValue.SetContentType(ZenContentType::kCbObject);
+ Record.Source = Params.Source;
+
+ bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed();
+ if (StoreLocal)
+ {
+ bool Overwrite = !EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal);
+ std::vector<IoHash> ReferencedAttachments;
+ ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
+ const IoHash ValueHash = HashView.AsHash();
+ ReferencedAttachments.push_back(ValueHash);
+ });
+ ZenCacheStore::PutResult PutResult = m_CacheStore.Put(Context,
+ Namespace,
+ Key.Bucket,
+ Key.Hash,
+ {.Value = Record.CacheValue},
+ ReferencedAttachments,
+ Overwrite,
+ nullptr);
+ if (PutResult.Status != zen::PutStatus::Success)
+ {
+ return;
+ }
+ m_CacheStats.WriteCount++;
+ }
+ };
m_UpstreamCache.GetCacheRecords(Namespace, UpstreamRecordRequests, std::move(OnCacheRecordGetComplete));
}
@@ -1620,12 +1702,12 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
{
if (EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData) && Request->RawSizeKnown)
{
- if (m_CidStore.ContainsChunk(Request->Key->ChunkId))
+ if (ChunkStore.ContainsChunk(Request->Key->ChunkId))
{
Request->Exists = true;
}
}
- else if (IoBuffer Payload = m_CidStore.FindChunkByCid(Request->Key->ChunkId))
+ else if (IoBuffer Payload = ChunkStore.FindChunkByCid(Request->Key->ChunkId))
{
if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
{
@@ -1758,6 +1840,8 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context,
return;
}
+ CidStore& ChunkStore = m_GetCidStore(Namespace);
+
CacheChunkRequest& Key = Params.Request;
size_t RequestIndex = std::distance(RequestKeys.data(), &Key);
ChunkRequest& Request = Requests[RequestIndex];
@@ -1774,20 +1858,26 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context,
bool StoreLocal = EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed();
if (StoreLocal)
{
+ bool Overwrite = !EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::QueryLocal);
if (Request.IsRecordRequest)
{
- m_CidStore.AddChunk(Params.Value, Params.RawHash);
+ ChunkStore.AddChunk(Params.Value, Params.RawHash);
}
else
{
- m_CacheStore.Put(Context,
- Namespace,
- Key.Key.Bucket,
- Key.Key.Hash,
- {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash},
- {},
- nullptr);
- m_CacheStats.WriteCount++;
+ ZenCacheStore::PutResult PutResult =
+ m_CacheStore.Put(Context,
+ Namespace,
+ Key.Key.Bucket,
+ Key.Key.Hash,
+ {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash},
+ {},
+ Overwrite,
+ nullptr);
+ if (PutResult.Status == zen::PutStatus::Success)
+ {
+ m_CacheStats.WriteCount++;
+ }
}
}
if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index d956384ca..1f2d6c37f 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -139,13 +139,10 @@ ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const st
CreateDirectories(m_RootDir);
m_DiskLayer.DiscoverBuckets();
-
- m_Gc.AddGcStorage(this);
}
ZenCacheNamespace::~ZenCacheNamespace()
{
- m_Gc.RemoveGcStorage(this);
}
struct ZenCacheNamespace::PutBatchHandle
@@ -154,7 +151,7 @@ struct ZenCacheNamespace::PutBatchHandle
};
ZenCacheNamespace::PutBatchHandle*
-ZenCacheNamespace::BeginPutBatch(std::vector<bool>& OutResult)
+ZenCacheNamespace::BeginPutBatch(std::vector<PutResult>& OutResult)
{
ZenCacheNamespace::PutBatchHandle* Handle = new ZenCacheNamespace::PutBatchHandle;
Handle->DiskLayerHandle = m_DiskLayer.BeginPutBatch(OutResult);
@@ -252,11 +249,12 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, GetBatc
return;
}
-void
+ZenCacheNamespace::PutResult
ZenCacheNamespace::Put(std::string_view InBucket,
const IoHash& HashKey,
const ZenCacheValue& Value,
std::span<IoHash> References,
+ bool Overwrite,
PutBatchHandle* OptionalBatchHandle)
{
ZEN_TRACE_CPU(OptionalBatchHandle ? "Z$::Namespace::Put(Batched)" : "Z$::Namespace::Put");
@@ -268,8 +266,12 @@ ZenCacheNamespace::Put(std::string_view InBucket,
ZEN_ASSERT(Value.Value.Size());
ZenCacheDiskLayer::PutBatchHandle* DiskLayerBatchHandle = OptionalBatchHandle ? OptionalBatchHandle->DiskLayerHandle : nullptr;
- m_DiskLayer.Put(InBucket, HashKey, Value, References, DiskLayerBatchHandle);
- m_WriteCount++;
+ PutResult RetVal = m_DiskLayer.Put(InBucket, HashKey, Value, References, Overwrite, DiskLayerBatchHandle);
+ if (RetVal.Status == zen::PutStatus::Success)
+ {
+ m_WriteCount++;
+ }
+ return RetVal;
}
bool
@@ -297,7 +299,6 @@ ZenCacheNamespace::EnumerateBucketContents(std::string_view
std::function<void()>
ZenCacheNamespace::Drop()
{
- m_Gc.RemoveGcStorage(this);
return m_DiskLayer.Drop();
}
@@ -307,25 +308,19 @@ ZenCacheNamespace::Flush()
m_DiskLayer.Flush();
}
+#if ZEN_WITH_TESTS
void
-ZenCacheNamespace::ScrubStorage(ScrubContext& Ctx)
+ZenCacheNamespace::Scrub(ScrubContext& Ctx)
{
- if (m_LastScrubTime == Ctx.ScrubTimestamp())
- {
- return;
- }
-
ZEN_INFO("scrubbing '{}'", m_RootDir);
-
- m_LastScrubTime = Ctx.ScrubTimestamp();
-
- m_DiskLayer.ScrubStorage(Ctx);
+ m_DiskLayer.Scrub(Ctx);
}
+#endif // ZEN_WITH_TESTS
-GcStorageSize
-ZenCacheNamespace::StorageSize() const
+CacheStoreSize
+ZenCacheNamespace::TotalSize() const
{
- return m_DiskLayer.StorageSize();
+ return m_DiskLayer.TotalSize();
}
ZenCacheNamespace::Info
@@ -557,7 +552,7 @@ ZenCacheStore::LogWorker()
}
}
-ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<bool>& OutResult)
+ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<PutResult>& OutResult)
: m_CacheStore(CacheStore)
{
ZEN_MEMSCOPE(GetCacheStoreTag());
@@ -720,13 +715,14 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
m_MissCount++;
}
-void
+ZenCacheStore::PutResult
ZenCacheStore::Put(const CacheRequestContext& Context,
std::string_view Namespace,
std::string_view Bucket,
const IoHash& HashKey,
const ZenCacheValue& Value,
std::span<IoHash> References,
+ bool Overwrite,
PutBatch* OptionalBatchHandle)
{
// Ad hoc rejection of known bad usage patterns for DDC bucket names
@@ -734,7 +730,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
if (IsKnownBadBucketName(Bucket))
{
m_RejectedWriteCount++;
- return;
+ return PutResult{zen::PutStatus::Invalid, "Bad bucket name"};
}
ZEN_MEMSCOPE(GetCacheStoreTag());
@@ -764,9 +760,16 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
{
ZenCacheNamespace::PutBatchHandle* BatchHandle = OptionalBatchHandle ? OptionalBatchHandle->m_NamespaceBatchHandle : nullptr;
- Store->Put(Bucket, HashKey, Value, References, BatchHandle);
- m_WriteCount++;
- return;
+ PutResult RetVal = Store->Put(Bucket, HashKey, Value, References, Overwrite, BatchHandle);
+ if (RetVal.Status == zen::PutStatus::Success)
+ {
+ m_WriteCount++;
+ }
+ else
+ {
+ m_RejectedWriteCount++;
+ }
+ return RetVal;
}
ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put [{}] bucket '{}', key '{}'",
@@ -774,6 +777,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
Namespace,
Bucket,
HashKey.ToHexString());
+ return PutResult{zen::PutStatus::Fail, fmt::format("Unknown namespace '{}'", Namespace)};
}
bool
@@ -822,11 +826,13 @@ ZenCacheStore::Flush()
IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Flush(); });
}
+#if ZEN_WITH_TESTS
void
-ZenCacheStore::ScrubStorage(ScrubContext& Ctx)
+ZenCacheStore::Scrub(ScrubContext& Ctx)
{
- IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.ScrubStorage(Ctx); });
+ IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); });
}
+#endif // ZEN_WITH_TESTS
CacheValueDetails
ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter,
@@ -951,12 +957,12 @@ ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Names
}
}
-GcStorageSize
-ZenCacheStore::StorageSize() const
+CacheStoreSize
+ZenCacheStore::TotalSize() const
{
- GcStorageSize Size;
+ CacheStoreSize Size;
IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) {
- GcStorageSize StoreSize = Store.StorageSize();
+ CacheStoreSize StoreSize = Store.TotalSize();
Size.MemorySize += StoreSize.MemorySize;
Size.DiskSize += StoreSize.DiskSize;
});
@@ -1026,7 +1032,7 @@ ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig)
ZenCacheStore::Info
ZenCacheStore::GetInfo() const
{
- ZenCacheStore::Info Info = {.Config = m_Configuration, .StorageSize = StorageSize()};
+ ZenCacheStore::Info Info = {.Config = m_Configuration, .StorageSize = TotalSize()};
IterateNamespaces([&Info](std::string_view NamespaceName, ZenCacheNamespace& Namespace) {
Info.NamespaceNames.push_back(std::string(NamespaceName));
@@ -1378,7 +1384,7 @@ TEST_CASE("cachestore.store")
Value.Value = Obj.GetBuffer().AsIoBuffer();
Value.Value.SetContentType(ZenContentType::kCbObject);
- Zcs.Put("test_bucket"sv, Key, Value, {});
+ Zcs.Put("test_bucket"sv, Key, Value, {}, false);
}
for (int i = 0; i < kIterationCount; ++i)
@@ -1414,7 +1420,7 @@ TEST_CASE("cachestore.size")
const size_t Count = 16;
ScopedTemporaryDirectory TempDir;
- GcStorageSize CacheSize;
+ CacheStoreSize CacheSize;
{
GcManager Gc;
@@ -1432,10 +1438,10 @@ TEST_CASE("cachestore.size")
const size_t Bucket = Key % 4;
std::string BucketName = fmt::format("test_bucket-{}", Bucket);
IoHash Hash = IoHash::HashBuffer(&Key, sizeof(uint32_t));
- Zcs.Put(BucketName, Hash, ZenCacheValue{.Value = Buffer}, {});
+ Zcs.Put(BucketName, Hash, ZenCacheValue{.Value = Buffer}, {}, false);
Keys.push_back({BucketName, Hash});
}
- CacheSize = Zcs.StorageSize();
+ CacheSize = Zcs.TotalSize();
CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize);
CHECK_EQ(0, CacheSize.MemorySize);
@@ -1445,7 +1451,7 @@ TEST_CASE("cachestore.size")
Zcs.Get(Key.first, Key.second, _);
}
- CacheSize = Zcs.StorageSize();
+ CacheSize = Zcs.TotalSize();
CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize);
CHECK_LE(CacheValue.GetSize() * Count, CacheSize.MemorySize);
}
@@ -1454,7 +1460,7 @@ TEST_CASE("cachestore.size")
GcManager Gc;
ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
- const GcStorageSize SerializedSize = Zcs.StorageSize();
+ const CacheStoreSize SerializedSize = Zcs.TotalSize();
CHECK_EQ(SerializedSize.MemorySize, 0);
CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize);
@@ -1462,8 +1468,8 @@ TEST_CASE("cachestore.size")
{
Zcs.DropBucket(fmt::format("test_bucket-{}", Bucket));
}
- CHECK_EQ(0, Zcs.StorageSize().DiskSize);
- CHECK_EQ(0, Zcs.StorageSize().MemorySize);
+ CHECK_EQ(0, Zcs.TotalSize().DiskSize);
+ CHECK_EQ(0, Zcs.TotalSize().MemorySize);
}
}
@@ -1472,7 +1478,7 @@ TEST_CASE("cachestore.size")
const size_t Count = 16;
ScopedTemporaryDirectory TempDir;
- GcStorageSize CacheSize;
+ CacheStoreSize CacheSize;
{
GcManager Gc;
@@ -1486,10 +1492,10 @@ TEST_CASE("cachestore.size")
for (size_t Key = 0; Key < Count; ++Key)
{
const size_t Bucket = Key % 4;
- Zcs.Put(fmt::format("test_bucket-{}", Bucket), IoHash::HashBuffer(&Key, sizeof(uint32_t)), {.Value = Buffer}, {});
+ Zcs.Put(fmt::format("test_bucket-{}", Bucket), IoHash::HashBuffer(&Key, sizeof(uint32_t)), {.Value = Buffer}, {}, false);
}
- CacheSize = Zcs.StorageSize();
+ CacheSize = Zcs.TotalSize();
CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize);
CHECK_EQ(0, CacheSize.MemorySize);
}
@@ -1498,7 +1504,7 @@ TEST_CASE("cachestore.size")
GcManager Gc;
ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
- const GcStorageSize SerializedSize = Zcs.StorageSize();
+ const CacheStoreSize SerializedSize = Zcs.TotalSize();
CHECK_EQ(SerializedSize.MemorySize, 0);
CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize);
@@ -1506,7 +1512,7 @@ TEST_CASE("cachestore.size")
{
Zcs.DropBucket(fmt::format("test_bucket-{}", Bucket));
}
- CHECK_EQ(0, Zcs.StorageSize().DiskSize);
+ CHECK_EQ(0, Zcs.TotalSize().DiskSize);
}
}
}
@@ -1569,7 +1575,7 @@ TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true))
for (const auto& Chunk : Chunks)
{
ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, &Chunk]() {
- Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {});
+ Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {}, false);
WorkCompleted.fetch_add(1);
});
}
@@ -1599,7 +1605,7 @@ TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true))
GcChunkHashes.swap(RemainingChunkHashes);
};
- const uint64_t TotalSize = Zcs.StorageSize().DiskSize;
+ const uint64_t TotalSize = Zcs.TotalSize().DiskSize;
CHECK_LE(kChunkSize * Chunks.size(), TotalSize);
{
@@ -1650,7 +1656,7 @@ TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true))
for (const auto& Chunk : NewChunks)
{
ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, Chunk, &AddedChunkCount]() {
- Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {});
+ Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {}, false);
AddedChunkCount.fetch_add(1);
WorkCompleted.fetch_add(1);
});
@@ -1755,14 +1761,14 @@ TEST_CASE("cachestore.namespaces")
Buffer.SetContentType(ZenContentType::kCbObject);
ZenCacheValue PutValue = {.Value = Buffer};
- Zcs.Put(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue, {});
+ Zcs.Put(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue, {}, false);
ZenCacheValue GetValue;
CHECK(Zcs.Get(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue));
CHECK(!Zcs.Get(Context, CustomNamespace, Bucket, Key1, GetValue));
// This should just be dropped as we don't allow creating of namespaces on the fly
- Zcs.Put(Context, CustomNamespace, Bucket, Key1, PutValue, {});
+ Zcs.Put(Context, CustomNamespace, Bucket, Key1, PutValue, {}, false);
CHECK(!Zcs.Get(Context, CustomNamespace, Bucket, Key1, GetValue));
}
@@ -1778,7 +1784,7 @@ TEST_CASE("cachestore.namespaces")
IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer();
Buffer2.SetContentType(ZenContentType::kCbObject);
ZenCacheValue PutValue2 = {.Value = Buffer2};
- Zcs.Put(Context, CustomNamespace, Bucket, Key2, PutValue2, {});
+ Zcs.Put(Context, CustomNamespace, Bucket, Key2, PutValue2, {}, false);
ZenCacheValue GetValue;
CHECK(!Zcs.Get(Context, ZenCacheStore::DefaultNamespace, Bucket, Key2, GetValue));
@@ -1820,7 +1826,7 @@ TEST_CASE("cachestore.drop.bucket")
Buffer.SetContentType(ZenContentType::kCbObject);
ZenCacheValue PutValue = {.Value = Buffer};
- Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {});
+ Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {}, false);
return Key;
};
auto GetValue = [&Context](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) {
@@ -1893,7 +1899,7 @@ TEST_CASE("cachestore.drop.namespace")
Buffer.SetContentType(ZenContentType::kCbObject);
ZenCacheValue PutValue = {.Value = Buffer};
- Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {});
+ Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {}, false);
return Key;
};
auto GetValue = [&Context](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) {
@@ -1957,8 +1963,6 @@ TEST_CASE("cachestore.blocked.disklayer.put")
{
ScopedTemporaryDirectory TempDir;
- GcStorageSize CacheSize;
-
const auto CreateCacheValue = [](size_t Size) -> CbObject {
std::vector<uint8_t> Buf;
Buf.resize(Size, Size & 0xff);
@@ -1979,7 +1983,7 @@ TEST_CASE("cachestore.blocked.disklayer.put")
size_t Key = Buffer.Size();
IoHash HashKey = IoHash::HashBuffer(&Key, sizeof(uint32_t));
- Zcs.Put("test_bucket", HashKey, {.Value = Buffer}, {});
+ Zcs.Put("test_bucket", HashKey, {.Value = Buffer}, {}, false);
ZenCacheValue BufferGet;
CHECK(Zcs.Get("test_bucket", HashKey, BufferGet));
@@ -1989,7 +1993,7 @@ TEST_CASE("cachestore.blocked.disklayer.put")
Buffer2.SetContentType(ZenContentType::kCbObject);
// We should be able to overwrite even if the file is open for read
- Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}, {});
+ Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}, {}, false);
MemoryView OldView = BufferGet.Value.GetView();
@@ -2080,7 +2084,7 @@ TEST_CASE("cachestore.scrub")
AttachmentHashes.push_back(Attachment.DecodeRawHash());
CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back());
}
- Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes);
+ Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes, false);
}
};
@@ -2094,8 +2098,8 @@ TEST_CASE("cachestore.scrub")
WorkerThreadPool ThreadPool{1};
ScrubContext ScrubCtx{ThreadPool};
- Zcs.ScrubStorage(ScrubCtx);
- CidStore.ScrubStorage(ScrubCtx);
+ Zcs.Scrub(ScrubCtx);
+ CidStore.Scrub(ScrubCtx);
CHECK(ScrubCtx.ScrubbedChunks() == (StructuredCids.size() + StructuredCids.size() * AttachmentSizes.size()) + UnstructuredCids.size());
CHECK(ScrubCtx.BadCids().GetSize() == 0);
}
@@ -2129,7 +2133,8 @@ TEST_CASE("cachestore.newgc.basics")
{.Value = Record.second,
.RawSize = Record.second.GetSize(),
.RawHash = IoHash::HashBuffer(Record.second.GetData(), Record.second.GetSize())},
- AttachmentKeys);
+ AttachmentKeys,
+ false);
for (const auto& Attachment : Attachments)
{
CidStore.AddChunk(Attachment.second.GetCompressed().Flatten().AsIoBuffer(), Attachment.second.DecodeRawHash());
@@ -2145,7 +2150,8 @@ TEST_CASE("cachestore.newgc.basics")
{.Value = CacheValue.second,
.RawSize = CacheValue.second.GetSize(),
.RawHash = IoHash::HashBuffer(CacheValue.second.GetData(), CacheValue.second.GetSize())},
- {});
+ {},
+ false);
CacheEntries.insert({Key, CacheEntry{CacheValue.second, {}}});
return Key;
};
diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp
index 460f0e10d..6b89beb3d 100644
--- a/src/zenstore/cas.cpp
+++ b/src/zenstore/cas.cpp
@@ -73,9 +73,12 @@ public:
WorkerThreadPool* OptionalWorkerPool,
uint64_t LargeSizeLimit) override;
virtual void Flush() override;
- virtual void ScrubStorage(ScrubContext& Ctx) override;
virtual CidStoreSize TotalSize() const override;
+#if ZEN_WITH_TESTS
+ virtual void Scrub(ScrubContext& Ctx) override;
+#endif // ZEN_WITH_TESTS
+
private:
CasContainerStrategy m_TinyStrategy;
CasContainerStrategy m_SmallStrategy;
@@ -195,7 +198,7 @@ CasImpl::OpenOrCreateManifest()
}
else
{
- ZEN_WARN("Store manifest validation failed: {:#x}, will generate new manifest to recover", uint32_t(ValidationResult));
+ ZEN_WARN("Store manifest validation failed: {}, will generate new manifest to recover", ToString(ValidationResult));
}
if (ManifestIsOk)
@@ -463,24 +466,19 @@ CasImpl::Flush()
m_LargeStrategy.Flush();
}
+#if ZEN_WITH_TESTS
void
-CasImpl::ScrubStorage(ScrubContext& Ctx)
+CasImpl::Scrub(ScrubContext& Ctx)
{
ZEN_MEMSCOPE(GetCasTag());
ZEN_TRACE_CPU("Cas::ScrubStorage");
- if (m_LastScrubTime == Ctx.ScrubTimestamp())
- {
- return;
- }
-
- m_LastScrubTime = Ctx.ScrubTimestamp();
-
m_SmallStrategy.ScrubStorage(Ctx);
m_TinyStrategy.ScrubStorage(Ctx);
m_LargeStrategy.ScrubStorage(Ctx);
}
+#endif // ZEN_WITH_TESTS
CidStoreSize
CasImpl::TotalSize() const
@@ -523,7 +521,7 @@ TEST_CASE("CasStore")
WorkerThreadPool ThreadPool{1};
ScrubContext Ctx{ThreadPool};
- Store->ScrubStorage(Ctx);
+ Store->Scrub(Ctx);
IoBuffer Value1{16};
memcpy(Value1.MutableData(), "1234567890123456", 16);
diff --git a/src/zenstore/cas.h b/src/zenstore/cas.h
index e279dd2cc..0f6e2ba9d 100644
--- a/src/zenstore/cas.h
+++ b/src/zenstore/cas.h
@@ -50,12 +50,13 @@ public:
WorkerThreadPool* OptionalWorkerPool,
uint64_t LargeSizeLimit) = 0;
virtual void Flush() = 0;
- virtual void ScrubStorage(ScrubContext& Ctx) = 0;
virtual CidStoreSize TotalSize() const = 0;
+#if ZEN_WITH_TESTS
+ virtual void Scrub(ScrubContext& Ctx) = 0;
+#endif // ZEN_WITH_TESTS
protected:
CidStoreConfiguration m_Config;
- uint64_t m_LastScrubTime = 0;
};
ZENCORE_API std::unique_ptr<CasStore> CreateCasStore(GcManager& Gc);
diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp
index 2ab769d04..ae1b59dc0 100644
--- a/src/zenstore/cidstore.cpp
+++ b/src/zenstore/cidstore.cpp
@@ -127,17 +127,9 @@ struct CidStore::Impl
void Flush() { m_CasStore.Flush(); }
- void ScrubStorage(ScrubContext& Ctx)
- {
- if (Ctx.ScrubTimestamp() == m_LastScrubTime)
- {
- return;
- }
-
- m_LastScrubTime = Ctx.ScrubTimestamp();
-
- m_CasStore.ScrubStorage(Ctx);
- }
+#if ZEN_WITH_TESTS
+ void Scrub(ScrubContext& Ctx) { m_CasStore.Scrub(Ctx); }
+#endif // ZEN_WITH_TESTS
CidStoreStats Stats()
{
@@ -236,11 +228,13 @@ CidStore::Flush()
m_Impl->Flush();
}
+#if ZEN_WITH_TESTS
void
-CidStore::ScrubStorage(ScrubContext& Ctx)
+CidStore::Scrub(ScrubContext& Ctx)
{
- m_Impl->ScrubStorage(Ctx);
+ m_Impl->Scrub(Ctx);
}
+#endif // ZEN_WITH_TESTS
CidStoreSize
CidStore::TotalSize() const
diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h
index 15e4cbf81..32c256a42 100644
--- a/src/zenstore/compactcas.h
+++ b/src/zenstore/compactcas.h
@@ -68,10 +68,10 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore
void Flush();
// GcStorage
-
virtual void ScrubStorage(ScrubContext& ScrubCtx) override;
virtual GcStorageSize StorageSize() const override;
+ // GcReferenceStore
virtual std::string GetGcName(GcCtx& Ctx) override;
virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override;
diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h
index adf48dc26..87b7dd812 100644
--- a/src/zenstore/include/zenstore/buildstore/buildstore.h
+++ b/src/zenstore/include/zenstore/buildstore/buildstore.h
@@ -6,9 +6,8 @@
#include <zencore/iohash.h>
#include <zenstore/accesstime.h>
#include <zenstore/caslog.h>
+#include <zenstore/cidstore.h>
#include <zenstore/gc.h>
-#include "../compactcas.h"
-#include "../filecas.h"
ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
@@ -19,18 +18,13 @@ namespace zen {
struct BuildStoreConfig
{
std::filesystem::path RootDirectory;
- uint32_t SmallBlobBlockStoreMaxBlockSize = 256 * 1024 * 1024;
- uint64_t SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024;
- uint32_t SmallBlobBlockStoreAlignement = 16;
- uint32_t MetadataBlockStoreMaxBlockSize = 64 * 1024 * 1024;
- uint32_t MetadataBlockStoreAlignement = 8;
- uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB
+ uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB
};
-class BuildStore : public GcReferencer, public GcReferenceLocker, public GcStorage
+class BuildStore : public GcReferencer, public GcReferenceLocker
{
public:
- explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc);
+ explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc, CidStore& BlobStore);
virtual ~BuildStore();
void PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload);
@@ -44,7 +38,7 @@ public:
std::vector<BlobExistsResult> BlobsExists(std::span<const IoHash> BlobHashes);
- void PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas);
+ void PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas, WorkerThreadPool* OptionalWorkerPool);
std::vector<IoBuffer> GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool);
void Flush();
@@ -52,10 +46,8 @@ public:
struct StorageStats
{
uint64_t EntryCount = 0;
- uint64_t LargeBlobCount = 0;
- uint64_t LargeBlobBytes = 0;
- uint64_t SmallBlobCount = 0;
- uint64_t SmallBlobBytes = 0;
+ uint64_t BlobCount = 0;
+ uint64_t BlobBytes = 0;
uint64_t MetadataCount = 0;
uint64_t MetadataByteCount = 0;
};
@@ -86,23 +78,18 @@ private:
//////// GcReferenceLocker
virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override;
- //////// GcStorage
- virtual void ScrubStorage(ScrubContext& ScrubCtx) override;
- virtual GcStorageSize StorageSize() const override;
-
#pragma pack(push)
#pragma pack(1)
struct PayloadEntry
{
PayloadEntry() {}
- PayloadEntry(uint64_t Flags, uint64_t Size)
+ PayloadEntry(uint8_t Flags, uint64_t Size)
{
ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size);
- ZEN_ASSERT((Flags & (kTombStone | kStandalone)) == Flags);
+ ZEN_ASSERT((Flags & (kTombStone)) == Flags);
FlagsAndSize = (Size << 8) | Flags;
}
- static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value
- static const uint8_t kStandalone = 0x20u; // This payload is stored as a standalone value
+ static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value
uint64_t FlagsAndSize = 0;
uint64_t GetSize() const { return FlagsAndSize >> 8; }
@@ -126,27 +113,47 @@ private:
struct MetadataEntry
{
- BlockStoreLocation Location; // 12 bytes
+ IoHash MetadataHash; // 20 bytes
+
+ MetadataEntry() {}
- ZenContentType ContentType = ZenContentType::kCOUNT; // 1 byte
- static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value
- uint8_t Flags = 0; // 1 byte
+ MetadataEntry(const IoHash& Hash, uint64_t Size, ZenContentType ContentType, uint8_t Flags)
+ {
+ ZEN_ASSERT((Size & 0x0000ffffffffffffu) == Size);
+ ZEN_ASSERT((Flags & kTombStone) == Flags);
+ FlagsContentTypeAndSize = (Size << 16) | ((uint64_t)ContentType << 8) | Flags;
+ MetadataHash = Hash;
+ }
+
+ static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value
+
+ uint64_t GetSize() const { return FlagsContentTypeAndSize >> 16; }
+ void SetSize(uint64_t Size)
+ {
+ ZEN_ASSERT((Size & 0x0000ffffffffffffu) == Size);
+ FlagsContentTypeAndSize = (Size << 16) | (FlagsContentTypeAndSize & 0xffff);
+ }
+
+ uint8_t GetFlags() const { return uint8_t(FlagsContentTypeAndSize & 0xff); }
+ void AddFlag(uint8_t Flag) { FlagsContentTypeAndSize |= Flag; }
+ void SetFlags(uint8_t Flags) { FlagsContentTypeAndSize = (FlagsContentTypeAndSize & 0xffffffffffffff00u) | Flags; }
+
+ ZenContentType GetContentType() const { return ZenContentType((FlagsContentTypeAndSize >> 8) & 0xff); }
+ void SetContentType(ZenContentType ContentType)
+ {
+ FlagsContentTypeAndSize = (FlagsContentTypeAndSize & 0xffffffffffff00ffu) | (uint16_t(ContentType) << 8);
+ }
- uint8_t Reserved1 = 0;
- uint8_t Reserved2 = 0;
+ uint64_t FlagsContentTypeAndSize = ((uint64_t)ZenContentType::kCOUNT << 8);
};
- static_assert(sizeof(MetadataEntry) == 16);
+ static_assert(sizeof(MetadataEntry) == 28);
struct MetadataDiskEntry
{
- MetadataEntry Entry; // 16 bytes
+ MetadataEntry Entry; // 28 bytes
IoHash BlobHash; // 20 bytes
- uint8_t Reserved1 = 0;
- uint8_t Reserved2 = 0;
- uint8_t Reserved3 = 0;
- uint8_t Reserved4 = 0;
};
- static_assert(sizeof(MetadataDiskEntry) == 40);
+ static_assert(sizeof(MetadataDiskEntry) == 48);
#pragma pack(pop)
@@ -206,17 +213,15 @@ private:
std::vector<BlobEntry> m_BlobEntries;
tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> m_BlobLookup;
- FileCasStrategy m_LargeBlobStore;
- CasContainerStrategy m_SmallBlobStore;
- BlockStore m_MetadataBlockStore;
+ CidStore& m_BlobStore;
TCasLogFile<PayloadDiskEntry> m_PayloadlogFile;
TCasLogFile<MetadataDiskEntry> m_MetadatalogFile;
uint64_t m_BlobLogFlushPosition = 0;
uint64_t m_MetaLogFlushPosition = 0;
- std::unique_ptr<HashSet> m_TrackedCacheKeys;
- std::atomic<uint64_t> m_LastAccessTimeUpdateCount;
+ std::unique_ptr<std::vector<IoHash>> m_TrackedBlobKeys;
+ std::atomic<uint64_t> m_LastAccessTimeUpdateCount;
friend class BuildStoreGcReferenceChecker;
friend class BuildStoreGcReferencePruner;
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 3cd2d6423..49c52f847 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -106,6 +106,12 @@ static_assert(sizeof(DiskIndexEntry) == 32);
//////////////////////////////////////////////////////////////////////////
+struct CacheStoreSize
+{
+ uint64_t DiskSize = 0;
+ uint64_t MemorySize = 0;
+};
+
class ZenCacheDiskLayer
{
public:
@@ -115,6 +121,7 @@ public:
uint32_t PayloadAlignment = 1u << 4;
uint64_t MemCacheSizeThreshold = 1 * 1024;
uint64_t LargeObjectThreshold = 128 * 1024;
+ bool LimitOverwrites = false;
};
struct Configuration
@@ -130,8 +137,8 @@ public:
struct BucketInfo
{
- uint64_t EntryCount = 0;
- GcStorageSize StorageSize;
+ uint64_t EntryCount = 0;
+ CacheStoreSize StorageSize;
};
struct Info
@@ -140,7 +147,7 @@ public:
Configuration Config;
std::vector<std::string> BucketNames;
uint64_t EntryCount = 0;
- GcStorageSize StorageSize;
+ CacheStoreSize StorageSize;
};
struct BucketStats
@@ -170,6 +177,12 @@ public:
uint64_t MemorySize;
};
+ struct PutResult
+ {
+ zen::PutStatus Status;
+ std::string Message;
+ };
+
explicit ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config);
~ZenCacheDiskLayer();
@@ -180,22 +193,22 @@ public:
void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& BatchHandle);
struct PutBatchHandle;
- PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult);
+ PutBatchHandle* BeginPutBatch(std::vector<PutResult>& OutResult);
void EndPutBatch(PutBatchHandle* Batch) noexcept;
- void Put(std::string_view Bucket,
+ PutResult Put(std::string_view Bucket,
const IoHash& HashKey,
const ZenCacheValue& Value,
std::span<IoHash> References,
+ bool Overwrite,
PutBatchHandle* OptionalBatchHandle);
std::function<void()> Drop();
std::function<void()> DropBucket(std::string_view Bucket);
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
- void DiscoverBuckets();
- GcStorageSize StorageSize() const;
- DiskStats Stats() const;
+ void DiscoverBuckets();
+ CacheStoreSize TotalSize() const;
+ DiskStats Stats() const;
Info GetInfo() const;
std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
@@ -212,6 +225,7 @@ public:
#if ZEN_WITH_TESTS
void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
+ void Scrub(ScrubContext& Ctx);
#endif // ZEN_WITH_TESTS
bool GetContentStats(std::string_view BucketName, CacheContentStats& OutContentStats) const;
@@ -219,7 +233,7 @@ public:
/** A cache bucket manages a single directory containing
metadata and data for that bucket
*/
- struct CacheBucket : public GcReferencer
+ struct CacheBucket : public GcReferencer, public GcStorage
{
CacheBucket(GcManager& Gc,
std::atomic_uint64_t& OuterCacheMemoryUsage,
@@ -236,13 +250,22 @@ public:
void Get(const IoHash& HashKey, GetBatchHandle& BatchHandle);
struct PutBatchHandle;
- PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult);
- void EndPutBatch(PutBatchHandle* Batch) noexcept;
- void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, PutBatchHandle* OptionalBatchHandle);
- uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime);
- std::function<void()> Drop();
- void Flush();
- void ScrubStorage(ScrubContext& Ctx);
+ PutBatchHandle* BeginPutBatch(std::vector<ZenCacheDiskLayer::PutResult>& OutResult);
+ void EndPutBatch(PutBatchHandle* Batch) noexcept;
+ PutResult Put(const IoHash& HashKey,
+ const ZenCacheValue& Value,
+ std::span<IoHash> References,
+ bool Overwrite,
+ PutBatchHandle* OptionalBatchHandle);
+ uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime);
+ std::function<void()> Drop();
+ void Flush();
+ inline CacheStoreSize TotalSize() const
+ {
+ return {.DiskSize = m_StandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(),
+ .MemorySize = m_MemCachedSize.load(std::memory_order::relaxed)};
+ }
+
RwLock::SharedLockScope GetGcReferencerLock();
struct ReferencesStats
@@ -265,11 +288,6 @@ public:
std::span<const std::size_t> ChunkIndexes,
std::vector<IoHash>& OutReferences) const;
- inline GcStorageSize StorageSize() const
- {
- return {.DiskSize = m_StandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(),
- .MemorySize = m_MemCachedSize.load(std::memory_order::relaxed)};
- }
uint64_t EntryCount() const;
BucketStats Stats();
@@ -281,6 +299,20 @@ public:
void SetAccessTime(const IoHash& HashKey, GcClock::TimePoint Time);
#endif // ZEN_WITH_TESTS
+ // GcReferencer
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
+ virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
+ virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override;
+
+ // GcStorage
+ virtual void ScrubStorage(ScrubContext& Ctx) override;
+ virtual GcStorageSize StorageSize() const override
+ {
+ CacheStoreSize Size = TotalSize();
+ return {.DiskSize = Size.DiskSize, .MemorySize = Size.MemorySize};
+ }
+
private:
#pragma pack(push)
#pragma pack(1)
@@ -379,19 +411,16 @@ public:
std::atomic_uint64_t m_StandaloneSize{};
std::atomic_uint64_t m_MemCachedSize{};
- virtual std::string GetGcName(GcCtx& Ctx) override;
- virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
- virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
- virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override;
-
- void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
- void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
- IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
- void PutInlineCacheValue(const IoHash& HashKey,
- const ZenCacheValue& Value,
- std::span<IoHash> References,
- PutBatchHandle* OptionalBatchHandle = nullptr);
- IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
+ void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
+ bool ShouldRejectPut(const IoHash& HashKey, ZenCacheValue& InOutValue, bool Overwrite, ZenCacheDiskLayer::PutResult& OutPutResult);
+ void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
+ IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
+ PutResult PutInlineCacheValue(const IoHash& HashKey,
+ const ZenCacheValue& Value,
+ std::span<IoHash> References,
+ bool Overwrite,
+ PutBatchHandle* OptionalBatchHandle = nullptr);
+ IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
CacheValueDetails::ValueDetails GetValueDetails(RwLock::SharedLockScope&, const IoHash& Key, PayloadIndex Index) const;
void SetMetaData(RwLock::ExclusiveLockScope&,
diff --git a/src/zenstore/include/zenstore/cache/cacherpc.h b/src/zenstore/include/zenstore/cache/cacherpc.h
index da8cf69fe..80340d72c 100644
--- a/src/zenstore/include/zenstore/cache/cacherpc.h
+++ b/src/zenstore/include/zenstore/cache/cacherpc.h
@@ -4,6 +4,7 @@
#include <zencore/iobuffer.h>
#include <zencore/logging.h>
+#include <zenstore/cache/cacheshared.h>
#include <zenutil/cache/cache.h>
#include <atomic>
@@ -56,13 +57,6 @@ struct CacheStats
std::atomic_uint64_t RpcChunkBatchRequests{};
};
-enum class PutResult
-{
- Success,
- Fail,
- Invalid,
-};
-
/** Recognize both kBinary and kCompressedBinary as kCompressedBinary for structured cache value keys.
We need this until the content type is preserved for kCompressedBinary when passing to and from upstream servers.
@@ -76,11 +70,13 @@ IsCompressedBinary(ZenContentType Type)
struct CacheRpcHandler
{
+ typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
+
CacheRpcHandler(LoggerRef InLog,
CacheStats& InCacheStats,
UpstreamCacheClient& InUpstreamCache,
ZenCacheStore& InCacheStore,
- CidStore& InCidStore,
+ GetCidStoreFunc&& InGetCidStore,
const DiskWriteBlocker* InDiskWriteBlocker);
~CacheRpcHandler();
@@ -100,6 +96,8 @@ struct CacheRpcHandler
int& OutTargetProcessId,
CbPackage& OutPackage);
+ CidStore& GetCidStore(std::string_view Namespace);
+
private:
CbPackage HandleRpcPutCacheRecords(const CacheRequestContext& Context, const CbPackage& BatchRequest);
CbPackage HandleRpcGetCacheRecords(const CacheRequestContext& Context, CbObjectView BatchRequest);
@@ -107,7 +105,7 @@ private:
CbPackage HandleRpcGetCacheValues(const CacheRequestContext& Context, CbObjectView BatchRequest);
CbPackage HandleRpcGetCacheChunks(const CacheRequestContext& Context, RpcAcceptOptions AcceptOptions, CbObjectView BatchRequest);
- PutResult PutCacheRecord(PutRequestData& Request, const CbPackage* Package);
+ PutStatus PutCacheRecord(PutRequestData& Request, const CbPackage* Package);
/** HandleRpcGetCacheChunks Helper: Parse the Body object into RecordValue Requests and Value Requests. */
bool ParseGetCacheChunksRequest(std::string& Namespace,
@@ -148,7 +146,7 @@ private:
CacheStats& m_CacheStats;
UpstreamCacheClient& m_UpstreamCache;
ZenCacheStore& m_CacheStore;
- CidStore& m_CidStore;
+ GetCidStoreFunc m_GetCidStore;
const DiskWriteBlocker* m_DiskWriteBlocker = nullptr;
bool AreDiskWritesAllowed() const;
diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h
index ef1b803de..8f40ae727 100644
--- a/src/zenstore/include/zenstore/cache/cacheshared.h
+++ b/src/zenstore/include/zenstore/cache/cacheshared.h
@@ -69,6 +69,14 @@ struct CacheContentStats
std::vector<IoHash> Attachments;
};
+enum class PutStatus
+{
+ Success,
+ Fail,
+ Conflict,
+ Invalid,
+};
+
bool IsKnownBadBucketName(std::string_view BucketName);
bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer);
diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h
index 48fc17960..c51d7312c 100644
--- a/src/zenstore/include/zenstore/cache/structuredcachestore.h
+++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h
@@ -49,7 +49,7 @@ class JobQueue;
*/
-class ZenCacheNamespace final : public GcStorage
+class ZenCacheNamespace final
{
public:
struct Configuration
@@ -78,24 +78,27 @@ public:
ZenCacheDiskLayer::DiskStats DiskStats;
};
+ using PutResult = ZenCacheDiskLayer::PutResult;
+
ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config);
~ZenCacheNamespace();
struct PutBatchHandle;
- PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResults);
+ PutBatchHandle* BeginPutBatch(std::vector<PutResult>& OutResults);
void EndPutBatch(PutBatchHandle* Batch) noexcept;
struct GetBatchHandle;
GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResults);
void EndGetBatch(GetBatchHandle* Batch) noexcept;
- bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
- void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& OptionalBatchHandle);
- void Put(std::string_view Bucket,
- const IoHash& HashKey,
- const ZenCacheValue& Value,
- std::span<IoHash> References,
- PutBatchHandle* OptionalBatchHandle = nullptr);
+ bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& OptionalBatchHandle);
+ PutResult Put(std::string_view Bucket,
+ const IoHash& HashKey,
+ const ZenCacheValue& Value,
+ std::span<IoHash> References,
+ bool Overwrite,
+ PutBatchHandle* OptionalBatchHandle = nullptr);
bool DropBucket(std::string_view Bucket);
void EnumerateBucketContents(std::string_view Bucket,
@@ -104,10 +107,7 @@ public:
std::function<void()> Drop();
void Flush();
- // GcStorage
- virtual void ScrubStorage(ScrubContext& ScrubCtx) override;
- virtual GcStorageSize StorageSize() const override;
-
+ CacheStoreSize TotalSize() const;
Configuration GetConfig() const { return m_Configuration; }
Info GetInfo() const;
std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
@@ -124,6 +124,7 @@ public:
#if ZEN_WITH_TESTS
void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
+ void Scrub(ScrubContext& ScrubCtx);
#endif // ZEN_WITH_TESTS
private:
@@ -137,7 +138,6 @@ private:
std::atomic<uint64_t> m_WriteCount{};
metrics::RequestStats m_PutOps;
metrics::RequestStats m_GetOps;
- uint64_t m_LastScrubTime = 0;
ZenCacheNamespace(const ZenCacheNamespace&) = delete;
ZenCacheNamespace& operator=(const ZenCacheNamespace&) = delete;
@@ -175,7 +175,7 @@ public:
Configuration Config;
std::vector<std::string> NamespaceNames;
uint64_t DiskEntryCount = 0;
- GcStorageSize StorageSize;
+ CacheStoreSize StorageSize;
};
struct NamedNamespaceStats
@@ -196,6 +196,8 @@ public:
std::vector<NamedNamespaceStats> NamespaceStats;
};
+ using PutResult = ZenCacheNamespace::PutResult;
+
ZenCacheStore(GcManager& Gc,
JobQueue& JobQueue,
const std::filesystem::path& BasePath,
@@ -206,7 +208,7 @@ public:
class PutBatch
{
public:
- PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector<bool>& OutResult);
+ PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector<PutResult>& OutResult);
~PutBatch();
private:
@@ -243,24 +245,24 @@ public:
const IoHash& HashKey,
GetBatch& BatchHandle);
- void Put(const CacheRequestContext& Context,
- std::string_view Namespace,
- std::string_view Bucket,
- const IoHash& HashKey,
- const ZenCacheValue& Value,
- std::span<IoHash> References,
- PutBatch* OptionalBatchHandle = nullptr);
+ PutResult Put(const CacheRequestContext& Context,
+ std::string_view Namespace,
+ std::string_view Bucket,
+ const IoHash& HashKey,
+ const ZenCacheValue& Value,
+ std::span<IoHash> References,
+ bool Overwrite,
+ PutBatch* OptionalBatchHandle = nullptr);
bool DropBucket(std::string_view Namespace, std::string_view Bucket);
bool DropNamespace(std::string_view Namespace);
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
CacheValueDetails GetValueDetails(const std::string_view NamespaceFilter,
const std::string_view BucketFilter,
const std::string_view ValueFilter) const;
- GcStorageSize StorageSize() const;
+ CacheStoreSize TotalSize() const;
CacheStoreStats Stats(bool IncludeNamespaceStats = true);
Configuration GetConfiguration() const { return m_Configuration; }
@@ -290,6 +292,10 @@ public:
bool GetContentStats(std::string_view Namespace, std::string_view BucketName, CacheContentStats& OutContentStats) const;
+#if ZEN_WITH_TESTS
+ void Scrub(ScrubContext& Ctx);
+#endif // ZEN_WITH_TESTS
+
private:
const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const;
ZenCacheNamespace* GetNamespace(std::string_view Namespace);
diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
index 152031c3a..c3993c028 100644
--- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
+++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
@@ -113,7 +113,7 @@ public:
std::span<CacheChunkRequest*> CacheChunkRequests,
OnCacheChunksGetComplete&& OnComplete) = 0;
- virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) = 0;
+ virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) = 0;
};
} // namespace zen
diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h
index b3d00fec0..8918b119f 100644
--- a/src/zenstore/include/zenstore/cidstore.h
+++ b/src/zenstore/include/zenstore/cidstore.h
@@ -87,12 +87,15 @@ public:
bool ContainsChunk(const IoHash& DecompressedId);
void FilterChunks(HashKeySet& InOutChunks);
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
CidStoreSize TotalSize() const;
CidStoreStats Stats() const;
virtual void ReportMetrics(StatsMetrics& Statsd) override;
+#if ZEN_WITH_TESTS
+ void Scrub(ScrubContext& Ctx);
+#endif // ZEN_WITH_TESTS
+
private:
struct Impl;
std::unique_ptr<CasStore> m_CasStore;
diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp
index 88238effd..2171f4d62 100644
--- a/src/zenutil/buildstoragecache.cpp
+++ b/src/zenutil/buildstoragecache.cpp
@@ -378,6 +378,13 @@ private:
m_Stats.TotalBytesRead += Result.DownloadedBytes;
m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0);
m_Stats.TotalRequestCount++;
+ SetAtomicMax(m_Stats.PeakSentBytes, Result.UploadedBytes);
+ SetAtomicMax(m_Stats.PeakReceivedBytes, Result.DownloadedBytes);
+ if (Result.ElapsedSeconds > 0.0)
+ {
+ uint64_t BytesPerSec = uint64_t((Result.UploadedBytes + Result.DownloadedBytes) / Result.ElapsedSeconds);
+ SetAtomicMax(m_Stats.PeakBytesPerSec, BytesPerSec);
+ }
}
HttpClient& m_HttpClient;
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index c2cc5ab3c..f75fe403f 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -40,19 +40,23 @@ public:
ZEN_TRACE_CPU("FileBuildStorage::ListNamespaces");
ZEN_UNUSED(bRecursive);
- SimulateLatency(0, 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
CbObjectWriter Writer;
Writer.BeginArray("results");
{
}
Writer.EndArray(); // results
- Writer.Save();
- SimulateLatency(Writer.GetSaveSize(), 0);
+
+ Writer.Finalize();
+ ReceivedBytes = Writer.GetSaveSize();
return Writer.Save();
}
@@ -61,11 +65,14 @@ public:
ZEN_TRACE_CPU("FileBuildStorage::ListBuilds");
ZEN_UNUSED(Query);
- SimulateLatency(Query.GetSize(), 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = Query.GetSize();
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BuildFolder = GetBuildsFolder();
DirectoryContent Content;
@@ -88,19 +95,23 @@ public:
}
}
Writer.EndArray(); // results
- Writer.Save();
- SimulateLatency(Writer.GetSaveSize(), 0);
+
+ Writer.Finalize();
+ ReceivedBytes = Writer.GetSaveSize();
return Writer.Save();
}
virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
{
ZEN_TRACE_CPU("FileBuildStorage::PutBuild");
- SimulateLatency(MetaData.GetSize(), 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = MetaData.GetSize();
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
CbObjectWriter BuildObject;
BuildObject.AddObject("metadata", MetaData);
@@ -109,35 +120,41 @@ public:
CbObjectWriter BuildResponse;
BuildResponse.AddInteger("chunkSize"sv, 32u * 1024u * 1024u);
- BuildResponse.Save();
-
- SimulateLatency(0, BuildResponse.GetSaveSize());
+ BuildResponse.Finalize();
+ ReceivedBytes = BuildResponse.GetSaveSize();
return BuildResponse.Save();
}
virtual CbObject GetBuild(const Oid& BuildId) override
{
ZEN_TRACE_CPU("FileBuildStorage::GetBuild");
- SimulateLatency(0, 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
CbObject Build = ReadBuild(BuildId);
- SimulateLatency(0, Build.GetSize());
+ ReceivedBytes = Build.GetSize();
return Build;
}
virtual void FinalizeBuild(const Oid& BuildId) override
{
ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuild");
- SimulateLatency(0, 0);
- Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
-
ZEN_UNUSED(BuildId);
- SimulateLatency(0, 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
}
virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid& BuildId,
@@ -146,10 +163,14 @@ public:
const CbObject& MetaData) override
{
ZEN_TRACE_CPU("FileBuildStorage::PutBuildPart");
- SimulateLatency(MetaData.GetSize(), 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = MetaData.GetSize();
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
CreateDirectories(BuildPartDataPath.parent_path());
@@ -184,7 +205,7 @@ public:
std::vector<IoHash> NeededAttachments = GetNeededAttachments(MetaData);
- SimulateLatency(0, sizeof(IoHash) * NeededAttachments.size());
+ ReceivedBytes = sizeof(IoHash) * NeededAttachments.size();
return std::make_pair(RawHash, std::move(NeededAttachments));
}
@@ -192,22 +213,24 @@ public:
virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
{
ZEN_TRACE_CPU("FileBuildStorage::GetBuildPart");
- SimulateLatency(0, 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten();
- m_Stats.TotalBytesRead += Payload.GetSize();
ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None);
CbObject BuildPartObject = CbObject(SharedBuffer(Payload));
- SimulateLatency(0, BuildPartObject.GetSize());
+ ReceivedBytes = BuildPartObject.GetSize();
return BuildPartObject;
}
@@ -215,15 +238,18 @@ public:
virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
{
ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuildPart");
- SimulateLatency(0, 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten();
- m_Stats.TotalBytesRead += Payload.GetSize();
+
IoHash RawHash = IoHash::HashBuffer(Payload.GetView());
if (RawHash != PartHash)
{
@@ -234,7 +260,7 @@ public:
CbObject BuildPartObject = CbObject(SharedBuffer(Payload));
std::vector<IoHash> NeededAttachments(GetNeededAttachments(BuildPartObject));
- SimulateLatency(0, NeededAttachments.size() * sizeof(IoHash));
+ ReceivedBytes = NeededAttachments.size() * sizeof(IoHash);
return NeededAttachments;
}
@@ -247,13 +273,14 @@ public:
ZEN_TRACE_CPU("FileBuildStorage::PutBuildBlob");
ZEN_UNUSED(BuildId);
ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
- SimulateLatency(Payload.GetSize(), 0);
ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, Payload));
- Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = Payload.GetSize();
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
if (!IsFile(BlockPath))
@@ -261,8 +288,8 @@ public:
CreateDirectories(BlockPath.parent_path());
TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView());
}
- m_Stats.TotalBytesWritten += Payload.GetSize();
- SimulateLatency(0, 0);
+
+ ReceivedBytes = Payload.GetSize();
}
virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId,
@@ -275,10 +302,15 @@ public:
ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob");
ZEN_UNUSED(BuildId);
ZEN_UNUSED(ContentType);
- SimulateLatency(0, 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
if (!IsFile(BlockPath))
@@ -314,7 +346,15 @@ public:
WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() {
ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob_Work");
IoBuffer PartPayload = Workload->Transmitter(Offset, Size);
- SimulateLatency(PartPayload.GetSize(), 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = PartPayload.GetSize();
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
std::error_code Ec;
Workload->TempFile.Write(PartPayload, Offset, Ec);
@@ -325,8 +365,7 @@ public:
Ec.message(),
Ec.value()));
}
- uint64_t BytesWritten = PartPayload.GetSize();
- m_Stats.TotalBytesWritten += BytesWritten;
+
const bool IsLastPart = Workload->PartsLeft.fetch_sub(1) == 1;
if (IsLastPart)
{
@@ -342,18 +381,14 @@ public:
Ec.value()));
}
}
- Workload->OnSentBytes(BytesWritten, IsLastPart);
- SimulateLatency(0, 0);
+ Workload->OnSentBytes(SentBytes, IsLastPart);
});
Offset += Size;
}
Workload->PartsLeft.store(WorkItems.size());
-
- SimulateLatency(0, 0);
return WorkItems;
}
- SimulateLatency(0, 0);
return {};
}
@@ -361,10 +396,15 @@ public:
{
ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlob");
ZEN_UNUSED(BuildId);
- SimulateLatency(0, 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
if (IsFile(BlockPath))
@@ -382,11 +422,9 @@ public:
ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload))));
}
Payload.SetContentType(ZenContentType::kCompressedBinary);
- m_Stats.TotalBytesRead += Payload.GetSize();
- SimulateLatency(0, Payload.GetSize());
+ ReceivedBytes = Payload.GetSize();
return Payload;
}
- SimulateLatency(0, 0);
return IoBuffer{};
}
@@ -398,10 +436,15 @@ public:
{
ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob");
ZEN_UNUSED(BuildId);
- SimulateLatency(0, 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
if (IsFile(BlockPath))
@@ -429,22 +472,29 @@ public:
uint64_t Size = Min(ChunkSize, BlobSize - Offset);
WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() {
ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob_Work");
- SimulateLatency(0, 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
+
IoBuffer PartPayload(Size);
Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset);
- m_Stats.TotalBytesRead += PartPayload.GetSize();
+ ReceivedBytes = PartPayload.GetSize();
Workload->OnReceive(Offset, PartPayload);
uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size);
if (ByteRemaning == Size)
{
Workload->OnComplete();
}
- SimulateLatency(Size, PartPayload.GetSize());
});
Offset += Size;
}
- SimulateLatency(0, 0);
return WorkItems;
}
return {};
@@ -455,18 +505,19 @@ public:
ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata");
ZEN_UNUSED(BuildId);
- SimulateLatency(MetaData.GetSize(), 0);
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = MetaData.GetSize();
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
const std::filesystem::path BlockMetaDataPath = GetBlobMetadataPath(BlockRawHash);
CreateDirectories(BlockMetaDataPath.parent_path());
TemporaryFile::SafeWriteFile(BlockMetaDataPath, MetaData.GetView());
- m_Stats.TotalBytesWritten += MetaData.GetSize();
WriteAsJson(BlockMetaDataPath, MetaData);
- SimulateLatency(0, 0);
return true;
}
@@ -474,10 +525,15 @@ public:
{
ZEN_TRACE_CPU("FileBuildStorage::FindBlocks");
ZEN_UNUSED(BuildId);
- SimulateLatency(sizeof(BuildId), 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
uint64_t FoundCount = 0;
@@ -495,8 +551,6 @@ public:
{
IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
- m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
-
CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
Writer.AddObject(BlockObject);
FoundCount++;
@@ -508,19 +562,25 @@ public:
}
}
Writer.EndArray(); // blocks
- CbObject Result = Writer.Save();
- SimulateLatency(0, Result.GetSize());
- return Result;
+
+ Writer.Finalize();
+ ReceivedBytes = Writer.GetSaveSize();
+ return Writer.Save();
}
virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
{
ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata");
ZEN_UNUSED(BuildId);
- SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0);
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = sizeof(Oid) + sizeof(IoHash) * BlockHashes.size();
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
Stopwatch ExecutionTimer;
- auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- m_Stats.TotalRequestCount++;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
CbObjectWriter Writer;
Writer.BeginArray("blocks");
@@ -532,22 +592,29 @@ public:
{
IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
- m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
-
CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
Writer.AddObject(BlockObject);
}
}
Writer.EndArray(); // blocks
- CbObject Result = Writer.Save();
- SimulateLatency(0, Result.GetSize());
- return Result;
+ Writer.Finalize();
+ ReceivedBytes = Writer.GetSaveSize();
+ return Writer.Save();
}
virtual void PutBuildPartStats(const Oid& BuildId,
const Oid& BuildPartId,
const tsl::robin_map<std::string, double>& FloatStats) override
{
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = 0;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
+
CbObjectWriter Request;
Request.BeginObject("floatStats"sv);
for (auto It : FloatStats)
@@ -555,17 +622,16 @@ public:
Request.AddFloat(It.first, It.second);
}
Request.EndObject();
- CbObject Payload = Request.Save();
-
- SimulateLatency(Payload.GetSize(), 0);
+ Request.Finalize();
+ SentBytes = Request.GetSaveSize();
const std::filesystem::path BuildPartStatsDataPath = GetBuildPartStatsPath(BuildId, BuildPartId);
CreateDirectories(BuildPartStatsDataPath.parent_path());
+ CbObject Payload = Request.Save();
+
TemporaryFile::SafeWriteFile(BuildPartStatsDataPath, Payload.GetView());
WriteAsJson(BuildPartStatsDataPath, Payload);
-
- SimulateLatency(0, 0);
}
protected:
@@ -629,7 +695,6 @@ protected:
const std::filesystem::path BuildDataPath = GetBuildPath(BuildId);
CreateDirectories(BuildDataPath.parent_path());
TemporaryFile::SafeWriteFile(BuildDataPath, Data.GetView());
- m_Stats.TotalBytesWritten += Data.GetSize();
WriteAsJson(BuildDataPath, Data);
}
@@ -646,7 +711,6 @@ protected:
Content.ErrorCode.value()));
}
IoBuffer Payload = Content.Flatten();
- m_Stats.TotalBytesRead += Payload.GetSize();
ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None);
CbObject BuildObject = CbObject(SharedBuffer(Payload));
return BuildObject;
@@ -704,6 +768,22 @@ protected:
}
private:
+ void AddStatistic(Stopwatch& ExecutionTimer, uint64_t UploadedBytes, uint64_t DownloadedBytes)
+ {
+ const uint64_t ElapsedUs = ExecutionTimer.GetElapsedTimeUs();
+ m_Stats.TotalBytesWritten += UploadedBytes;
+ m_Stats.TotalBytesRead += DownloadedBytes;
+ m_Stats.TotalRequestTimeUs += ElapsedUs;
+ m_Stats.TotalRequestCount++;
+ SetAtomicMax(m_Stats.PeakSentBytes, UploadedBytes);
+ SetAtomicMax(m_Stats.PeakReceivedBytes, DownloadedBytes);
+ if (ElapsedUs > 0)
+ {
+ uint64_t BytesPerSec = ((UploadedBytes + DownloadedBytes) * 1000000u) / ElapsedUs;
+ SetAtomicMax(m_Stats.PeakBytesPerSec, BytesPerSec);
+ }
+ }
+
const std::filesystem::path m_StoragePath;
BuildStorage::Statistics& m_Stats;
const bool m_EnableJsonOutput = false;
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
index f49d4b42a..46ecd0a11 100644
--- a/src/zenutil/include/zenutil/buildstorage.h
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -21,6 +21,9 @@ public:
std::atomic<uint64_t> TotalRequestCount = 0;
std::atomic<uint64_t> TotalRequestTimeUs = 0;
std::atomic<uint64_t> TotalExecutionTimeUs = 0;
+ std::atomic<uint64_t> PeakSentBytes = 0;
+ std::atomic<uint64_t> PeakReceivedBytes = 0;
+ std::atomic<uint64_t> PeakBytesPerSec = 0;
};
virtual ~BuildStorage() {}
diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h
index e1fb73fd4..a0690a16a 100644
--- a/src/zenutil/include/zenutil/buildstoragecache.h
+++ b/src/zenutil/include/zenutil/buildstoragecache.h
@@ -22,6 +22,9 @@ public:
std::atomic<uint64_t> TotalRequestCount = 0;
std::atomic<uint64_t> TotalRequestTimeUs = 0;
std::atomic<uint64_t> TotalExecutionTimeUs = 0;
+ std::atomic<uint64_t> PeakSentBytes = 0;
+ std::atomic<uint64_t> PeakReceivedBytes = 0;
+ std::atomic<uint64_t> PeakBytesPerSec = 0;
};
virtual ~BuildStorageCache() {}
diff --git a/src/zenutil/include/zenutil/wildcard.h b/src/zenutil/include/zenutil/wildcard.h
new file mode 100644
index 000000000..9f402e100
--- /dev/null
+++ b/src/zenutil/include/zenutil/wildcard.h
@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <string_view>
+
+namespace zen {
+
+bool MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive);
+
+void wildcard_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 9974725ff..c9278acb4 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -17,6 +17,16 @@ namespace zen {
using namespace std::literals;
+namespace {
+ void ThrowFromJupiterResult(const JupiterResult& Result, std::string_view Prefix)
+ {
+ int Error = Result.ErrorCode < (int)HttpResponseCode::Continue ? Result.ErrorCode : 0;
+ HttpResponseCode Status =
+ Result.ErrorCode >= int(HttpResponseCode::Continue) ? HttpResponseCode(Result.ErrorCode) : HttpResponseCode::ImATeapot;
+ throw HttpClientError(fmt::format("{}: {} ({})", Prefix, Result.Reason, Result.ErrorCode), Error, Status);
+ }
+} // namespace
+
class JupiterBuildStorage : public BuildStorage
{
public:
@@ -46,7 +56,7 @@ public:
AddStatistic(ListResult);
if (!ListResult.Success)
{
- throw std::runtime_error(fmt::format("Failed listing namespaces: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+ ThrowFromJupiterResult(ListResult, "Failed listing namespaces");
}
CbObject NamespaceResponse = PayloadToCbObject("Failed listing namespaces"sv, ListResult.Response);
@@ -66,10 +76,10 @@ public:
AddStatistic(BucketsResult);
if (!BucketsResult.Success)
{
- throw std::runtime_error(
- fmt::format("Failed listing namespaces: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode));
+ ThrowFromJupiterResult(BucketsResult, fmt::format("Failed listing buckets in namespace {}", Namespace));
}
- CbObject BucketResponse = PayloadToCbObject("Failed listing namespaces"sv, BucketsResult.Response);
+ CbObject BucketResponse =
+ PayloadToCbObject(fmt::format("Failed listing buckets in namespace {}", Namespace), BucketsResult.Response);
Response.BeginArray("items");
for (CbFieldView BucketField : BucketResponse["buckets"])
@@ -103,7 +113,7 @@ public:
AddStatistic(ListResult);
if (!ListResult.Success)
{
- throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+ ThrowFromJupiterResult(ListResult, "Failed listing builds"sv);
}
return PayloadToCbObject("Failed listing builds"sv, ListResult.Response);
}
@@ -120,7 +130,7 @@ public:
AddStatistic(PutResult);
if (!PutResult.Success)
{
- throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode));
+ ThrowFromJupiterResult(PutResult, "Failed creating build"sv);
}
return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
}
@@ -135,7 +145,7 @@ public:
AddStatistic(GetBuildResult);
if (!GetBuildResult.Success)
{
- throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode));
+ ThrowFromJupiterResult(GetBuildResult, "Failed fetching build"sv);
}
return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
}
@@ -150,8 +160,7 @@ public:
AddStatistic(FinalizeBuildResult);
if (!FinalizeBuildResult.Success)
{
- throw std::runtime_error(
- fmt::format("Failed finalizing build part: {} ({})", FinalizeBuildResult.Reason, FinalizeBuildResult.ErrorCode));
+ ThrowFromJupiterResult(FinalizeBuildResult, "Failed finalizing build"sv);
}
}
@@ -170,7 +179,7 @@ public:
AddStatistic(PutPartResult);
if (!PutPartResult.Success)
{
- throw std::runtime_error(fmt::format("Failed creating build part: {} ({})", PutPartResult.Reason, PutPartResult.ErrorCode));
+ ThrowFromJupiterResult(PutPartResult, "Failed creating build part"sv);
}
return std::make_pair(PutPartResult.RawHash, std::move(PutPartResult.Needs));
}
@@ -185,10 +194,7 @@ public:
AddStatistic(GetBuildPartResult);
if (!GetBuildPartResult.Success)
{
- throw std::runtime_error(fmt::format("Failed fetching build part {}: {} ({})",
- BuildPartId,
- GetBuildPartResult.Reason,
- GetBuildPartResult.ErrorCode));
+ ThrowFromJupiterResult(GetBuildPartResult, "Failed fetching build part"sv);
}
return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
}
@@ -203,8 +209,7 @@ public:
AddStatistic(FinalizePartResult);
if (!FinalizePartResult.Success)
{
- throw std::runtime_error(
- fmt::format("Failed finalizing build part: {} ({})", FinalizePartResult.Reason, FinalizePartResult.ErrorCode));
+ ThrowFromJupiterResult(FinalizePartResult, "Failed finalizing build part"sv);
}
return std::move(FinalizePartResult.Needs);
}
@@ -222,7 +227,7 @@ public:
AddStatistic(PutBlobResult);
if (!PutBlobResult.Success)
{
- throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PutBlobResult.Reason, PutBlobResult.ErrorCode));
+ ThrowFromJupiterResult(PutBlobResult, "Failed putting build part"sv);
}
}
@@ -249,8 +254,7 @@ public:
AddStatistic(PutMultipartBlobResult);
if (!PutMultipartBlobResult.Success)
{
- throw std::runtime_error(
- fmt::format("Failed putting build part: {} ({})", PutMultipartBlobResult.Reason, PutMultipartBlobResult.ErrorCode));
+ ThrowFromJupiterResult(PutMultipartBlobResult, "Failed putting large build blob"sv);
}
OnSentBytes(PutMultipartBlobResult.SentBytes, WorkItems.empty());
@@ -265,7 +269,7 @@ public:
AddStatistic(PartResult);
if (!PartResult.Success)
{
- throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode));
+ ThrowFromJupiterResult(PartResult, "Failed putting large build blob"sv);
}
OnSentBytes(PartResult.SentBytes, IsComplete);
});
@@ -285,8 +289,7 @@ public:
AddStatistic(GetBuildBlobResult);
if (!GetBuildBlobResult.Success)
{
- throw std::runtime_error(
- fmt::format("Failed fetching build blob {}: {} ({})", RawHash, GetBuildBlobResult.Reason, GetBuildBlobResult.ErrorCode));
+ ThrowFromJupiterResult(GetBuildBlobResult, "Failed fetching build blob"sv);
}
return std::move(GetBuildBlobResult.Response);
}
@@ -314,8 +317,7 @@ public:
AddStatistic(GetMultipartBlobResult);
if (!GetMultipartBlobResult.Success)
{
- throw std::runtime_error(
- fmt::format("Failed getting build part: {} ({})", GetMultipartBlobResult.Reason, GetMultipartBlobResult.ErrorCode));
+ ThrowFromJupiterResult(GetMultipartBlobResult, "Failed getting large build part"sv);
}
std::vector<std::function<void()>> WorkList;
for (auto& WorkItem : WorkItems)
@@ -327,7 +329,7 @@ public:
AddStatistic(PartResult);
if (!PartResult.Success)
{
- throw std::runtime_error(fmt::format("Failed getting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode));
+ ThrowFromJupiterResult(PartResult, "Failed getting large build part"sv);
}
});
}
@@ -350,8 +352,7 @@ public:
{
return false;
}
- throw std::runtime_error(
- fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode));
+ ThrowFromJupiterResult(PutMetaResult, "Failed putting build block metadata"sv);
}
return true;
}
@@ -366,7 +367,7 @@ public:
AddStatistic(FindResult);
if (!FindResult.Success)
{
- throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode));
+ ThrowFromJupiterResult(FindResult, "Failed fetching known blocks"sv);
}
return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response);
}
@@ -392,8 +393,7 @@ public:
AddStatistic(GetBlockMetadataResult);
if (!GetBlockMetadataResult.Success)
{
- throw std::runtime_error(
- fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode));
+ ThrowFromJupiterResult(GetBlockMetadataResult, "Failed fetching block metadatas"sv);
}
return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response);
}
@@ -416,14 +416,12 @@ public:
AddStatistic(PutBuildPartStatsResult);
if (!PutBuildPartStatsResult.Success)
{
- throw std::runtime_error(fmt::format("Failed posting build part statistics: {} ({})",
- PutBuildPartStatsResult.Reason,
- PutBuildPartStatsResult.ErrorCode));
+ ThrowFromJupiterResult(PutBuildPartStatsResult, "Failed posting build part statistics"sv);
}
}
private:
- static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload)
+ static CbObject PayloadToCbObject(std::string_view ErrorContext, const IoBuffer& Payload)
{
if (Payload.GetContentType() == ZenContentType::kJSON)
{
@@ -443,7 +441,7 @@ private:
else
{
throw std::runtime_error(
- fmt::format("{}: {} ({})", "Unsupported response format", Context, ToString(Payload.GetContentType())));
+ fmt::format("{}: {} ({})", "Unsupported response format", ErrorContext, ToString(Payload.GetContentType())));
}
}
@@ -453,6 +451,14 @@ private:
m_Stats.TotalBytesRead += Result.ReceivedBytes;
m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0);
m_Stats.TotalRequestCount++;
+
+ SetAtomicMax(m_Stats.PeakSentBytes, Result.SentBytes);
+ SetAtomicMax(m_Stats.PeakReceivedBytes, Result.ReceivedBytes);
+ if (Result.ElapsedSeconds > 0.0)
+ {
+ uint64_t BytesPerSec = uint64_t((Result.SentBytes + Result.ReceivedBytes) / Result.ElapsedSeconds);
+ SetAtomicMax(m_Stats.PeakBytesPerSec, BytesPerSec);
+ }
}
JupiterSession m_Session;
diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp
index aa806438b..a571d1d11 100644
--- a/src/zenutil/parallelwork.cpp
+++ b/src/zenutil/parallelwork.cpp
@@ -33,18 +33,21 @@ ParallelWork::~ParallelWork()
"ParallelWork disposed without explicit wait for completion, likely caused by an exception, waiting for dispatched threads "
"to complete");
m_PendingWork.CountDown();
+ m_DispatchComplete = true;
+ }
+ const bool WaitSucceeded = m_PendingWork.Wait();
+ const ptrdiff_t RemainingWork = m_PendingWork.Remaining();
+ if (!WaitSucceeded)
+ {
+ ZEN_ERROR("ParallelWork::~ParallelWork(): waiting for latch failed, pending work count at {}", RemainingWork);
}
- m_PendingWork.Wait();
- ptrdiff_t RemainingWork = m_PendingWork.Remaining();
if (RemainingWork != 0)
{
void* Frames[8];
uint32_t FrameCount = GetCallstack(2, 8, Frames);
CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames);
- ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}",
- RemainingWork,
- CallstackToString(Callstack, " "));
- FreeCallstack(Callstack);
+ auto _ = MakeGuard([Callstack]() { FreeCallstack(Callstack); });
+ ZEN_WARN("ParallelWork::~ParallelWork(): waited for outstanding work but pending work count is {} instead of 0", RemainingWork);
uint32_t WaitedMs = 0;
while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000)
@@ -52,20 +55,27 @@ ParallelWork::~ParallelWork()
Sleep(50);
WaitedMs += 50;
}
- RemainingWork = m_PendingWork.Remaining();
- if (RemainingWork != 0)
+ ptrdiff_t RemainingWorkAfterSafetyWait = m_PendingWork.Remaining();
+ if (RemainingWorkAfterSafetyWait != 0)
{
- ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork)
+ ZEN_ERROR("ParallelWork::~ParallelWork(): safety wait for {} tasks failed, pending work count at {} after {}\n{}",
+ RemainingWork,
+ RemainingWorkAfterSafetyWait,
+ NiceLatencyNs(WaitedMs * 1000000u),
+ CallstackToString(Callstack, " "))
}
else
{
- ZEN_INFO("ParallelWork destructor safety wait succeeded");
+ ZEN_ERROR("ParallelWork::~ParallelWork(): safety wait for {} tasks completed after {}\n{}",
+ RemainingWork,
+ NiceLatencyNs(WaitedMs * 1000000u),
+ CallstackToString(Callstack, " "));
}
}
}
catch (const std::exception& Ex)
{
- ZEN_ERROR("Exception in ~ParallelWork: {}", Ex.what());
+ ZEN_ERROR("Exception in ParallelWork::~ParallelWork(): {}", Ex.what());
}
}
@@ -82,10 +92,9 @@ void
ParallelWork::Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback)
{
ZEN_ASSERT(!m_DispatchComplete);
- m_DispatchComplete = true;
-
ZEN_ASSERT(m_PendingWork.Remaining() > 0);
m_PendingWork.CountDown();
+ m_DispatchComplete = true;
while (!m_PendingWork.Wait(UpdateIntervalMS))
{
@@ -99,11 +108,20 @@ void
ParallelWork::Wait()
{
ZEN_ASSERT(!m_DispatchComplete);
- m_DispatchComplete = true;
-
ZEN_ASSERT(m_PendingWork.Remaining() > 0);
m_PendingWork.CountDown();
- m_PendingWork.Wait();
+ m_DispatchComplete = true;
+
+ const bool WaitSucceeded = m_PendingWork.Wait();
+ const ptrdiff_t RemainingWork = m_PendingWork.Remaining();
+ if (!WaitSucceeded)
+ {
+ ZEN_ERROR("ParallelWork::Wait(): waiting for latch failed, pending work count at {}", RemainingWork);
+ }
+ else if (RemainingWork != 0)
+ {
+ ZEN_ERROR("ParallelWork::Wait(): pending work count at {} after successful wait for latch", RemainingWork);
+ }
RethrowErrors();
}
diff --git a/src/zenutil/wildcard.cpp b/src/zenutil/wildcard.cpp
new file mode 100644
index 000000000..df69f6a5e
--- /dev/null
+++ b/src/zenutil/wildcard.cpp
@@ -0,0 +1,112 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/string.h>
+#include <zenutil/wildcard.h>
+
+#if ZEN_WITH_TESTS
+# include <zencore/testing.h>
+#endif // ZEN_WITH_TESTS
+
+namespace zen {
+
+bool
+MatchWildcard(std::string_view::const_iterator WildcardIt,
+ std::string_view::const_iterator WildcardEnd,
+ std::string_view::const_iterator StringIt,
+ std::string_view::const_iterator StringEnd)
+{
+ for (; WildcardIt != WildcardEnd; WildcardIt++)
+ {
+ switch (*WildcardIt)
+ {
+ case '?':
+ if (StringIt == StringEnd)
+ {
+ return false;
+ }
+ StringIt++;
+ break;
+ case '*':
+ {
+ if ((WildcardIt + 1) == WildcardEnd)
+ {
+ return true;
+ }
+ size_t Max = std::distance(StringIt, StringEnd);
+ for (size_t i = 0; i < Max; i++)
+ {
+ if (MatchWildcard(WildcardIt + 1, WildcardEnd, StringIt + i, StringEnd))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ default:
+ if (*StringIt != *WildcardIt)
+ {
+ return false;
+ }
+ ++StringIt;
+ }
+ }
+ return StringIt == StringEnd;
+}
+
+bool
+MatchWildcard(std::string_view Wildcard, std::string_view String, bool CaseSensitive)
+{
+ if (CaseSensitive)
+ {
+ return MatchWildcard(begin(Wildcard), end(Wildcard), begin(String), end(String));
+ }
+ else
+ {
+ std::string LowercaseWildcard = ToLower(Wildcard);
+ std::string LowercaseString = ToLower(String);
+ std::string_view LowercaseWildcardView(LowercaseWildcard);
+ std::string_view LowercaseStringView(LowercaseString);
+ return MatchWildcard(begin(LowercaseWildcardView),
+ end(LowercaseWildcardView),
+ begin(LowercaseStringView),
+ end(LowercaseStringView));
+ }
+}
+
+#if ZEN_WITH_TESTS
+
+void
+wildcard_forcelink()
+{
+}
+
+TEST_CASE("Wildcard")
+{
+ CHECK(MatchWildcard("*.*", "normal.txt", true));
+ CHECK(MatchWildcard("*.*", "normal.txt", false));
+
+ CHECK(!MatchWildcard("*.*", "normal", true));
+ CHECK(!MatchWildcard("*.*", "normal", false));
+
+ CHECK(MatchWildcard("*", "hey/normal", true));
+ CHECK(MatchWildcard("*", "hey/normal", false));
+
+ CHECK(MatchWildcard("hey/*.txt", "hey/normal.txt", true));
+ CHECK(MatchWildcard("*/?ormal.txt", "hey/normal.txt", true));
+ CHECK(!MatchWildcard("*/?rmal.txt", "hey/normal.txt", true));
+ CHECK(MatchWildcard("*/?ormal.*", "hey/normal.txt", true));
+ CHECK(MatchWildcard("*/?ormal", "hey/normal", true));
+
+ CHECK(MatchWildcard("hey/*.txt", "hey/normaL.txt", false));
+ CHECK(MatchWildcard("*/?ormal.TXT", "hey/normal.txt", false));
+ CHECK(MatchWildcard("*/?ORMAL.*", "hey/normal.txt", false));
+ CHECK(MatchWildcard("*/?ormal", "HEY/normal", false));
+
+ CHECK(!MatchWildcard("hey/*.txt", "heY/normal.txt", true));
+ CHECK(!MatchWildcard("*/?ormal.TXT", "hey/normal.txt", true));
+ CHECK(!MatchWildcard("*/?ORMAL.*", "hey/normal.txt", true));
+ CHECK(!MatchWildcard("*/?ormal", "hey/normaL", true));
+}
+
+#endif
+} // namespace zen
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
index fe23b00c1..37b229c49 100644
--- a/src/zenutil/zenutil.cpp
+++ b/src/zenutil/zenutil.cpp
@@ -9,6 +9,7 @@
# include <zenutil/chunkedfile.h>
# include <zenutil/commandlineoptions.h>
# include <zenutil/parallelwork.h>
+# include <zenutil/wildcard.h>
namespace zen {
@@ -21,6 +22,7 @@ zenutil_forcelinktests()
chunkedfile_forcelink();
commandlineoptions_forcelink();
parallellwork_forcelink();
+ wildcard_forcelink();
}
} // namespace zen