aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2023-10-30 18:29:09 +0100
committerGitHub <[email protected]>2023-10-30 18:29:09 +0100
commitcbdda104ada38108700f9da5b192867d83074119 (patch)
tree98c04b344e041c156fdc1a5c393672bef743be34 /src
parentfix changelog (diff)
downloadzen-cbdda104ada38108700f9da5b192867d83074119.tar.xz
zen-cbdda104ada38108700f9da5b192867d83074119.zip
individual gc stats (#506)
- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2 - Feature: New options for zen command `gc-status` - `--details` that enables the detailed output from the last GC operation when using GC V2
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/admin_cmd.cpp5
-rw-r--r--src/zen/cmds/admin_cmd.h1
-rw-r--r--src/zenserver/admin/admin.cpp130
-rw-r--r--src/zenserver/cache/cachedisklayer.cpp65
-rw-r--r--src/zenserver/cache/cachedisklayer.h3
-rw-r--r--src/zenserver/cache/structuredcachestore.cpp306
-rw-r--r--src/zenserver/projectstore/projectstore.cpp91
-rw-r--r--src/zenserver/projectstore/projectstore.h3
-rw-r--r--src/zenstore/compactcas.cpp89
-rw-r--r--src/zenstore/compactcas.h3
-rw-r--r--src/zenstore/filecas.cpp99
-rw-r--r--src/zenstore/filecas.h3
-rw-r--r--src/zenstore/gc.cpp631
-rw-r--r--src/zenstore/include/zenstore/gc.h90
14 files changed, 966 insertions, 553 deletions
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 209390e2a..8b8b416ba 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -183,6 +183,7 @@ GcStatusCommand::GcStatusCommand()
{
m_Options.add_options()("h,help", "Print help");
m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+ m_Options.add_option("", "d", "details", "Show detailed GC report", cxxopts::value(m_Details)->default_value("false"), "<details>");
}
GcStatusCommand::~GcStatusCommand()
@@ -209,6 +210,10 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
cpr::Session Session;
Session.SetHeader(cpr::Header{{"Accept", "application/json"}});
Session.SetUrl({fmt::format("{}/admin/gc", m_HostName)});
+ if (m_Details)
+ {
+ Session.SetParameters({{"details", "true"}});
+ }
cpr::Response Result = Session.Get();
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 4a9de89e2..4b57eccdb 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -57,6 +57,7 @@ public:
private:
cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"};
std::string m_HostName;
+ bool m_Details = false;
};
////////////////////////////////////////////
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index fb376f238..83f33d4ea 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -199,6 +199,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
[this](HttpRouterRequest& Req) {
const GcSchedulerState State = m_GcScheduler.GetState();
+ const HttpServerRequest::QueryParams Params = Req.ServerRequest().GetQueryParams();
+
+ bool Details = false;
+ if (auto Param = Params.GetValue("details"); Param == "true")
+ {
+ Details = true;
+ }
+
auto SecondsToString = [](std::chrono::seconds Secs) {
return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count()));
};
@@ -226,6 +234,101 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
Response << "DiskUsed" << NiceBytes(State.DiskUsed);
Response << "DiskFree" << NiceBytes(State.DiskFree);
+ auto WriteReferencerStats = [&](CbObjectWriter& Response, const GcReferencerStats& Stats) {
+ if (Stats.Count == 0)
+ {
+ return;
+ }
+ Response << "Count" << Stats.Count;
+ Response << "Expired" << Stats.Expired;
+ Response << "Deleted" << Stats.Deleted;
+
+ Response << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
+ Response << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
+
+ Response << "RemoveExpiredData" << NiceTimeSpanMs(Stats.RemoveExpiredDataMS.count());
+ Response << "CreateReferenceCheckers" << NiceTimeSpanMs(Stats.CreateReferenceCheckersMS.count());
+ Response << "LockState" << NiceTimeSpanMs(Stats.LockStateMS.count());
+ Response << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
+ };
+
+ auto WriteReferenceStoreStats = [&](CbObjectWriter& Response, const GcReferenceStoreStats& Stats) {
+ if (Stats.Count == 0)
+ {
+ return;
+ }
+ Response << "Count" << Stats.Count;
+ Response << "Pruned" << Stats.Pruned;
+ Response << "Compacted" << Stats.Compacted;
+
+ Response << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
+ Response << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
+
+ Response << "CreateReferencePruner" << NiceTimeSpanMs(Stats.CreateReferencePrunerMS.count());
+ Response << "RemoveUnreferencedData" << NiceTimeSpanMs(Stats.RemoveUnreferencedDataMS.count());
+ Response << "CompactReferenceStore" << NiceTimeSpanMs(Stats.CompactReferenceStoreMS.count());
+ Response << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
+ };
+
+ auto WriteGCResult = [&](CbObjectWriter& Response, const GcResult& Result) {
+ Response << "RemovedDisk" << NiceBytes(Result.RemovedDisk);
+ Response << "RemovedMemory" << NiceBytes(Result.RemovedMemory);
+ Response << "WriteBlock" << NiceTimeSpanMs(Result.WriteBlockMS.count());
+ Response << "Elapsed" << NiceTimeSpanMs(Result.ElapsedMS.count());
+
+ if (!Details)
+ {
+ return;
+ }
+
+ Response << "RemoveExpiredData" << NiceTimeSpanMs(Result.RemoveExpiredDataMS.count());
+ Response << "CreateReferenceCheckers" << NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count());
+ Response << "LockState" << NiceTimeSpanMs(Result.LockStateMS.count());
+
+ Response << "CreateReferencePruner" << NiceTimeSpanMs(Result.CreateReferencePrunerMS.count());
+ Response << "RemoveUnreferencedData" << NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count());
+ Response << "CompactReferenceStore" << NiceTimeSpanMs(Result.CompactReferenceStoreMS.count());
+
+ Response.BeginObject("ReferencerStats");
+ {
+ WriteReferencerStats(Response, Result.ReferencerStat);
+ }
+ Response.EndObject();
+
+ Response.BeginObject("ReferenceStoreStats");
+ {
+ WriteReferenceStoreStats(Response, Result.ReferenceStoreStat);
+ }
+ Response.EndObject();
+
+ if (!Result.ReferencerStats.empty())
+ {
+ Response.BeginArray("Referencers");
+ {
+ for (const std::pair<std::string, GcReferencerStats>& It : Result.ReferencerStats)
+ {
+ Response.BeginObject();
+ Response << "Name" << It.first;
+ WriteReferencerStats(Response, It.second);
+ Response.EndObject();
+ }
+ }
+ Response.EndArray();
+ }
+ if (!Result.ReferenceStoreStats.empty())
+ {
+ Response.BeginArray("ReferenceStores");
+ for (const std::pair<std::string, GcReferenceStoreStats>& It : Result.ReferenceStoreStats)
+ {
+ Response.BeginObject();
+ Response << "Name" << It.first;
+ WriteReferenceStoreStats(Response, It.second);
+ Response.EndObject();
+ }
+ Response.EndArray();
+ }
+ };
+
Response.BeginObject("FullGC");
{
Response << "LastTime" << fmt::format("{}", State.LastFullGcTime);
@@ -234,18 +337,33 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
{
Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC);
}
- Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count());
- Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
- Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
+ if (State.LastFullGCV2Result)
+ {
+ WriteGCResult(Response, State.LastFullGCV2Result.value());
+ }
+ else
+ {
+ Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count());
+ Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
+ Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
+ }
}
Response.EndObject();
Response.BeginObject("LightweightGC");
{
Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime);
Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc);
- Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count());
- Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
- Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
+
+ if (State.LastLightweightGCV2Result)
+ {
+ WriteGCResult(Response, State.LastLightweightGCV2Result.value());
+ }
+ else
+ {
+ Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count());
+ Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
+ Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
+ }
}
Response.EndObject();
Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 38cbf3a93..fa80ed414 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -2213,8 +2213,14 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
});
}
+std::string
+ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
+{
+ return fmt::format("cachebucket:'{}'", m_BucketDir.string());
+}
+
void
-ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
+ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
{
size_t TotalEntries = 0;
tsl::robin_set<IoHash, IoHash::Hasher> ExpiredInlineKeys;
@@ -2222,11 +2228,18 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc cache bucket '{}': removed {} expired keys out of {} in {}",
- m_BucketDir,
- ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size(),
- TotalEntries,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_BucketDir,
+ Stats.Count,
+ Stats.Expired,
+ Stats.Deleted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -2278,7 +2291,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
}
}
- Ctx.ExpiredItems.fetch_add(ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size());
+ Stats.Expired += ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size();
// Get all locations we need to keep for affected blocks
if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2312,7 +2325,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[It->second])
{
size_t PayloadSize = m_CachedPayloads[It->second].GetSize();
- Ctx.RemovedMemory.fetch_add(PayloadSize);
+ Stats.RemovedMemory += PayloadSize;
RemoveMemCacheUsage(PayloadSize);
}
m_Index.erase(It);
@@ -2321,7 +2334,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed);
}
}
- Ctx.Items.fetch_add(TotalEntries);
+ Stats.Count += TotalEntries;
if (ExpiredEntries.empty())
{
@@ -2333,7 +2346,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
return;
}
- Ctx.DeletedItems.fetch_add(ExpiredEntries.size());
+ Stats.Deleted += ExpiredEntries.size();
// Compact standalone items
ExtendablePathBuilder<256> Path;
@@ -2370,7 +2383,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
Ec.message());
continue;
}
- Ctx.RemovedDiskSpace.fetch_add(ExpiredKey.second);
+ Stats.RemovedDisk += ExpiredKey.second;
}
if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2405,7 +2418,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
}
}
m_SlogFile.Append(MovedEntries);
- Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+ Stats.RemovedDisk += FreedDiskSpace;
},
[&]() { return 0; });
}
@@ -2436,14 +2449,18 @@ public:
}
}
- virtual void LockState(GcCtx&) override
+ virtual void LockState(GcCtx& Ctx) override
{
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc cache bucket '{}': found {} references in {}",
- m_CacheBucket.m_BucketDir,
- m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}",
+ m_CacheBucket.m_BucketDir,
+ m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
@@ -2503,11 +2520,19 @@ public:
};
std::vector<GcReferenceChecker*>
-ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx&)
+ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
{
Stopwatch Timer;
- const auto _ = MakeGuard(
- [&] { ZEN_DEBUG("gc cache bucket '{}': refreshed reference cache in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': found {} references in {}",
+ m_BucketDir,
+ m_ReferenceCount,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
std::vector<IoHash> UpdateKeys;
std::vector<IoHash> StandaloneKeys;
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index d8f51c398..ded8f2e70 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -267,7 +267,8 @@ private:
std::atomic_uint64_t m_StandaloneSize{};
std::atomic_uint64_t m_MemCachedSize{};
- virtual void RemoveExpiredData(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 516532528..55ab6bf31 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -1982,18 +1982,20 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() - std::chrono::hours(1),
.ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
.CollectSmallObjects = false,
- .IsDeleteMode = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(0u, Result.ExpiredItems);
- CHECK_EQ(0u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(0u, Result.RemovedDiskSpace);
+ .IsDeleteMode = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(0u, Result.ReferencerStat.Expired);
+ CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(0u, Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2014,18 +2016,20 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = false,
- .IsDeleteMode = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems);
- CHECK_EQ(0u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(0u, Result.RemovedDiskSpace);
+ .IsDeleteMode = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired);
+ CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(0u, Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2046,18 +2050,20 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
- .IsDeleteMode = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(7u, Result.ExpiredItems);
- CHECK_EQ(0u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(0u, Result.RemovedDiskSpace);
+ .IsDeleteMode = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(7u, Result.ReferencerStat.Expired);
+ CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(0u, Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2078,19 +2084,21 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = false,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems);
- CHECK_EQ(1u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired);
+ CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2111,19 +2119,21 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(7u, Result.ExpiredItems);
- CHECK_EQ(7u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GE(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(7u, Result.ReferencerStat.Expired);
+ CHECK_EQ(7u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GE(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
@@ -2144,21 +2154,24 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = false,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems); // Only one cache value is pruned/deleted as that is the only large item in the cache (all other
- // large items as in cas)
- CHECK_EQ(1u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired); // Only one cache value is pruned/deleted as that is the only large item in the cache
+ // (all other large items as in cas)
+ CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
CHECK_EQ(0u,
- Result.PrunedReferences); // We won't remove any references since all referencers are small which retains all references
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+ Result.ReferenceStoreStat
+ .Pruned); // We won't remove any references since all referencers are small which retains all references
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2179,18 +2192,21 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .CollectSmallObjects = true,
- .IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(7u, Result.ExpiredItems);
- CHECK_EQ(7u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(5u, Result.PrunedReferences);
- CHECK_EQ(5u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .CollectSmallObjects = true,
+ .IsDeleteMode = true,
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(7u, Result.ReferencerStat.Expired);
+ CHECK_EQ(7u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
@@ -2212,20 +2228,23 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(6u, Result.ExpiredItems);
- CHECK_EQ(6u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(6u, Result.ReferencerStat.Expired);
+ CHECK_EQ(6u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2247,21 +2266,24 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(5u, Result.ExpiredItems);
- CHECK_EQ(5u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(5u, Result.ReferencerStat.Expired);
+ CHECK_EQ(5u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2283,22 +2305,25 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(4u, Result.ExpiredItems);
- CHECK_EQ(4u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(5u, Result.PrunedReferences);
- CHECK_EQ(5u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(4u, Result.ReferencerStat.Expired);
+ CHECK_EQ(4u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
@@ -2320,6 +2345,7 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
// Prime so we can check GC of memory layer
ZenCacheValue Dummy;
@@ -2331,22 +2357,23 @@ TEST_CASE("z$.newgc.basics")
Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[2], Dummy);
Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[3], Dummy);
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(4u, Result.ExpiredItems);
- CHECK_EQ(4u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(4u, Result.ReferencerStat.Expired);
+ CHECK_EQ(4u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() +
CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize();
CHECK_EQ(MemoryClean, Result.RemovedMemory);
@@ -2370,35 +2397,36 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
auto Attachments =
CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::hours(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::hours(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(8u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems);
- CHECK_EQ(1u, Result.DeletedItems);
- CHECK_EQ(9u, Result.References);
- CHECK_EQ(4u, Result.PrunedReferences);
- CHECK_EQ(4u, Result.CompactedReferences);
- CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(),
- Result.RemovedDiskSpace);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(8u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired);
+ CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(9u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(4u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(4u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(), Result.RemovedDisk);
uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize();
CHECK_EQ(MemoryClean, Result.RemovedMemory);
}
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 274876123..7da2f7695 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3036,8 +3036,14 @@ ProjectStore::AreDiskWritesAllowed() const
return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed());
}
+std::string
+ProjectStore::GetGcName(GcCtx&)
+{
+ return fmt::format("projectstore:'{}'", m_ProjectBasePath.string());
+}
+
void
-ProjectStore::RemoveExpiredData(GcCtx& Ctx)
+ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
{
size_t ProjectCount = 0;
size_t ExpiredProjectCount = 0;
@@ -3045,13 +3051,18 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
size_t ExpiredOplogCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc project store '{}': removed {} expired projects out of {}, {} expired oplogs out of {} in {}",
- m_ProjectBasePath,
- ExpiredProjectCount,
- ProjectCount,
- ExpiredOplogCount,
- OplogCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_ProjectBasePath,
+ Stats.Count,
+ Stats.Expired,
+ Stats.Deleted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<Ref<Project>> ExpiredProjects;
@@ -3097,14 +3108,14 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
OplogId,
Project->Identifier);
Project->DeleteOplog(OplogId);
- Ctx.RemovedDiskSpace.fetch_add(OplogSize);
+ Stats.RemovedDisk += OplogSize;
}
- Ctx.DeletedItems.fetch_add(ExpiredOplogs.size());
+ Stats.Deleted += ExpiredOplogs.size();
Project->Flush();
}
}
ProjectCount = Projects.size();
- Ctx.Items.fetch_add(ProjectCount + OplogCount);
+ Stats.Count += ProjectCount + OplogCount;
ExpiredProjectCount = ExpiredProjects.size();
if (ExpiredProjects.empty())
@@ -3150,20 +3161,34 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
DeleteDirectories(PathToRemove);
}
- Ctx.DeletedItems.fetch_add(ExpiredProjects.size());
+ Stats.Deleted += ExpiredProjects.size();
}
- Ctx.ExpiredItems.fetch_add(ExpiredOplogCount + ExpiredProjectCount);
+ Stats.Expired += ExpiredOplogCount + ExpiredProjectCount;
}
class ProjectStoreReferenceChecker : public GcReferenceChecker
{
public:
- ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner)
+ ProjectStoreReferenceChecker(GcCtx& Ctx, ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner)
{
if (PreCache)
{
- RwLock::SharedLockScope _(m_Oplog.m_OplogLock);
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': precached {} references in {} from {}/{}",
+ m_Oplog.m_BasePath,
+ m_UncachedReferences.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ m_Oplog.m_OuterProject->Identifier,
+ m_Oplog.OplogId());
+ });
+
+ RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
m_Oplog.IterateOplog([&](CbObjectView Op) {
Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
});
@@ -3173,16 +3198,20 @@ public:
virtual ~ProjectStoreReferenceChecker() {}
- virtual void LockState(GcCtx&) override
+ virtual void LockState(GcCtx& Ctx) override
{
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc project oplog '{}': found {} references in {} from {}/{}",
- m_Oplog.m_BasePath,
- m_UncachedReferences.size(),
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- m_Oplog.m_OuterProject->Identifier,
- m_Oplog.OplogId());
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
+ m_Oplog.m_BasePath,
+ m_UncachedReferences.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ m_Oplog.m_OuterProject->Identifier,
+ m_Oplog.OplogId());
});
m_OplogLock = std::make_unique<RwLock::SharedLockScope>(m_Oplog.m_OplogLock);
@@ -3211,18 +3240,22 @@ public:
};
std::vector<GcReferenceChecker*>
-ProjectStore::CreateReferenceCheckers(GcCtx&)
+ProjectStore::CreateReferenceCheckers(GcCtx& Ctx)
{
size_t ProjectCount = 0;
size_t OplogCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc project store '{}': opened {} projects and {} oplogs in {}",
- m_ProjectBasePath,
- ProjectCount,
- OplogCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [CREATE CHECKERS] '{}': opened {} projects and {} oplogs in {}",
+ m_ProjectBasePath,
+ ProjectCount,
+ OplogCount,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
DiscoverProjects();
@@ -3250,7 +3283,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx&)
ProjectStore::Oplog* Oplog = Project->OpenOplog(OpLogId);
GcClock::TimePoint Now = GcClock::Now();
bool TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5));
- Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache));
+ Checkers.emplace_back(new ProjectStoreReferenceChecker(Ctx, *Oplog, TryPreCache));
}
OplogCount += OpLogs.size();
}
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 94e697278..fbff1444d 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -293,7 +293,8 @@ public:
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override;
- virtual void RemoveExpiredData(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
CbArray GetProjectsList();
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index f93dafa21..655c0558d 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -567,15 +567,22 @@ public:
{
}
- virtual void CompactReferenceStore(GcCtx& Ctx)
+ virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- size_t CompactedCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc block store '{}': compacted {} cids in {}",
- m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
- CompactedCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: compactcas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
if (Ctx.Settings.IsDeleteMode && Ctx.Settings.CollectSmallObjects)
@@ -609,13 +616,12 @@ public:
}
}
m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
- Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+ Stats.RemovedDisk += FreedDiskSpace;
},
[&]() { return 0; });
- CompactedCount = m_PrunedKeys.size();
- Ctx.CompactedReferences.fetch_add(
- CompactedCount); // Slightly missleading, it might not be compacted if the block is the currently writing block
+ Stats.Compacted +=
+ m_PrunedKeys.size(); // Slightly missleading, it might not be compacted if the block is the currently writing block
}
}
@@ -634,21 +640,27 @@ public:
{
}
- virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences)
+ virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx,
+ GcReferenceStoreStats& Stats,
+ const GetUnusedReferencesFunc& GetUnusedReferences)
{
- size_t TotalCount = m_Cids.size();
- size_t PruneCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc block store '{}': removed {} unused cid out of {} in {}",
- m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
- PruneCount,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
- m_Cids.clear();
if (UnusedCids.empty())
{
@@ -725,8 +737,7 @@ public:
}
}
- PruneCount = UnusedKeys.size();
- Ctx.PrunedReferences.fetch_add(PruneCount);
+ Stats.Pruned += UnusedKeys.size();
return new CasContainerStoreCompactor(m_CasContainerStrategy,
std::move(CompactState),
std::move(CompactStateKeys),
@@ -738,33 +749,45 @@ private:
std::vector<IoHash> m_Cids;
};
+std::string
+CasContainerStrategy::GetGcName(GcCtx&)
+{
+ return fmt::format("compactcas:'{}'", (m_RootDirectory / m_ContainerBaseName).string());
+}
+
GcReferencePruner*
-CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx)
+CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- size_t TotalCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc block store '{}': found {} cid keys to check in {}",
- m_RootDirectory / m_ContainerBaseName,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: compactcas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_RootDirectory / m_ContainerBaseName,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> CidsToCheck;
{
RwLock::SharedLockScope __(m_LocationMapLock);
+ if (m_LocationMap.empty())
+ {
+ return {};
+ }
CidsToCheck.reserve(m_LocationMap.size());
for (const auto& It : m_LocationMap)
{
CidsToCheck.push_back(It.first);
}
}
- TotalCount = CidsToCheck.size();
- if (TotalCount == 0)
- {
- return {};
- }
- Ctx.References.fetch_add(TotalCount);
+ Stats.Count += CidsToCheck.size();
return new CasContainerReferencePruner(*this, std::move(CidsToCheck));
}
diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h
index 9ff4ae4fc..9f3aa6b07 100644
--- a/src/zenstore/compactcas.h
+++ b/src/zenstore/compactcas.h
@@ -71,7 +71,8 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override;
- virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override;
private:
CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index e28e0dea4..c021e0e21 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1340,15 +1340,22 @@ public:
{
}
- virtual void CompactReferenceStore(GcCtx& Ctx)
+ virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- size_t CompactedCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc file store '{}': removed data for {} unused cids in {}",
- m_FileCasStrategy.m_RootDirectory,
- CompactedCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: filecas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_FileCasStrategy.m_RootDirectory,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> ReferencedCleaned;
ReferencedCleaned.reserve(m_ReferencesToClean.size());
@@ -1382,7 +1389,7 @@ public:
{
continue;
}
- Ctx.RemovedDiskSpace.fetch_add(SizeOnDisk);
+ Stats.RemovedDisk += SizeOnDisk;
}
else
{
@@ -1401,8 +1408,7 @@ public:
ReferencedCleaned.push_back(ChunkHash);
}
}
- CompactedCount = ReferencedCleaned.size();
- Ctx.CompactedReferences.fetch_add(ReferencedCleaned.size());
+ Stats.Compacted += ReferencedCleaned.size();
}
private:
@@ -1415,27 +1421,38 @@ class FileCasReferencePruner : public GcReferencePruner
public:
FileCasReferencePruner(FileCasStrategy& Owner, std::vector<IoHash>&& Cids) : m_FileCasStrategy(Owner), m_Cids(std::move(Cids)) {}
- virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences)
+ virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx,
+ GcReferenceStoreStats& Stats,
+ const GetUnusedReferencesFunc& GetUnusedReferences)
{
- size_t TotalCount = m_Cids.size();
- size_t PruneCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc file store '{}': removed {} unused cid out of {} in {}",
- m_FileCasStrategy.m_RootDirectory,
- PruneCount,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: filecas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_FileCasStrategy.m_RootDirectory,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
- std::vector<IoHash> UnusedReferences = GetUnusedReferences(m_Cids);
- m_Cids.clear();
+ std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
+ if (UnusedCids.empty())
+ {
+ // Nothing to collect
+ return nullptr;
+ }
std::vector<IoHash> PrunedReferences;
- PrunedReferences.reserve(UnusedReferences.size());
+ PrunedReferences.reserve(UnusedCids.size());
{
RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock);
- for (const IoHash& ChunkHash : UnusedReferences)
+ for (const IoHash& ChunkHash : UnusedCids)
{
auto It = m_FileCasStrategy.m_Index.find(ChunkHash);
if (It == m_FileCasStrategy.m_Index.end())
@@ -1454,8 +1471,7 @@ public:
}
}
- PruneCount = PrunedReferences.size();
- Ctx.PrunedReferences.fetch_add(PruneCount);
+ Stats.Pruned += PrunedReferences.size();
return new FileCasStoreCompactor(m_FileCasStrategy, std::move(PrunedReferences));
}
@@ -1464,33 +1480,44 @@ private:
std::vector<IoHash> m_Cids;
};
+std::string
+FileCasStrategy::GetGcName(GcCtx&)
+{
+ return fmt::format("filecas:'{}'", m_RootDirectory.string());
+}
+
GcReferencePruner*
-FileCasStrategy::CreateReferencePruner(GcCtx& Ctx)
+FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- // TODO
- std::size_t TotalCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc file store '{}': found {} cid keys to check in {}",
- m_RootDirectory,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: filecas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_RootDirectory,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> CidsToCheck;
{
RwLock::SharedLockScope __(m_Lock);
+ if (m_Index.empty())
+ {
+ return {};
+ }
CidsToCheck.reserve(m_Index.size());
for (const auto& It : m_Index)
{
CidsToCheck.push_back(It.first);
}
}
- TotalCount = CidsToCheck.size();
- if (TotalCount == 0)
- {
- return {};
- }
- Ctx.References.fetch_add(TotalCount);
+ Stats.Count += CidsToCheck.size();
return new FileCasReferencePruner(*this, std::move(CidsToCheck));
}
diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h
index 2e9a1d5dc..c39a39bb7 100644
--- a/src/zenstore/filecas.h
+++ b/src/zenstore/filecas.h
@@ -44,7 +44,8 @@ struct FileCasStrategy final : public GcStorage, public GcReferenceStore
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override;
- virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override;
private:
void MakeIndexSnapshot();
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index e09f46063..4d146c16c 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -330,6 +330,47 @@ GcManager::~GcManager()
//////// Begin New GC WIP
void
+GcResult::Sum()
+{
+ for (std::pair<std::string, GcReferencerStats>& Referencer : ReferencerStats)
+ {
+ GcReferencerStats& SubStat = Referencer.second;
+ ReferencerStat.Count += SubStat.Count;
+ ReferencerStat.Expired += SubStat.Expired;
+ ReferencerStat.Deleted += SubStat.Deleted;
+ ReferencerStat.RemovedDisk += SubStat.RemovedDisk;
+ ReferencerStat.RemovedMemory += SubStat.RemovedMemory;
+ SubStat.ElapsedMS = SubStat.RemoveExpiredDataMS + SubStat.CreateReferenceCheckersMS + SubStat.LockStateMS;
+
+ ReferencerStat.RemoveExpiredDataMS += SubStat.RemoveExpiredDataMS;
+ ReferencerStat.CreateReferenceCheckersMS += SubStat.CreateReferenceCheckersMS;
+ ReferencerStat.LockStateMS += SubStat.LockStateMS;
+ ReferencerStat.ElapsedMS += SubStat.ElapsedMS;
+
+ RemovedDisk += SubStat.RemovedDisk;
+ RemovedMemory += SubStat.RemovedMemory;
+ }
+ for (std::pair<std::string, GcReferenceStoreStats>& ReferenceStore : ReferenceStoreStats)
+ {
+ GcReferenceStoreStats& SubStat = ReferenceStore.second;
+ ReferenceStoreStat.Count += SubStat.Count;
+ ReferenceStoreStat.Pruned += SubStat.Pruned;
+ ReferenceStoreStat.Compacted += SubStat.Compacted;
+ ReferenceStoreStat.RemovedDisk += SubStat.RemovedDisk;
+ ReferenceStoreStat.RemovedMemory += SubStat.RemovedMemory;
+ SubStat.ElapsedMS = SubStat.CreateReferencePrunerMS + SubStat.RemoveUnreferencedDataMS + SubStat.CompactReferenceStoreMS;
+
+ ReferenceStoreStat.CreateReferencePrunerMS += SubStat.CreateReferencePrunerMS;
+ ReferenceStoreStat.RemoveUnreferencedDataMS += SubStat.RemoveUnreferencedDataMS;
+ ReferenceStoreStat.CompactReferenceStoreMS += SubStat.CompactReferenceStoreMS;
+ ReferenceStoreStat.ElapsedMS += SubStat.ElapsedMS;
+
+ RemovedDisk += SubStat.RemovedDisk;
+ RemovedMemory += SubStat.RemovedMemory;
+ }
+}
+
+void
GcManager::AddGcReferencer(GcReferencer& Referencer)
{
RwLock::ExclusiveLockScope _(m_Lock);
@@ -358,245 +399,270 @@ GcManager::RemoveGcReferenceStore(GcReferenceStore& ReferenceStore)
GcResult
GcManager::CollectGarbage(const GcSettings& Settings)
{
- GcCtx Ctx{.Settings = Settings};
-
- Stopwatch TotalTimer;
- auto __ = MakeGuard([&]() {
- ZEN_INFO(
- "GC: Removed {} items out of {}, deleted {} out of {}. Pruned {} Cid entries out of {}, compacted {} Cid entries out of {}, "
- "freed "
- "{} on disk and {} of memory in {}",
- Ctx.ExpiredItems.load(),
- Ctx.Items.load(),
- Ctx.DeletedItems.load(),
- Ctx.ExpiredItems.load(),
- Ctx.PrunedReferences.load(),
- Ctx.References.load(),
- Ctx.CompactedReferences.load(),
- Ctx.PrunedReferences.load(),
- NiceBytes(Ctx.RemovedDiskSpace.load()),
- NiceBytes(Ctx.RemovedMemory.load()),
- NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
- });
-
- RwLock::SharedLockScope GcLock(m_Lock);
-
- static const bool SingleThread =
-#if ZEN_BUILD_DEBUG
- true
-#else
- false
-#endif
- ;
- WorkerThreadPool ThreadPool(SingleThread ? 0 : 8);
-
- if (!m_GcReferencers.empty())
- {
- Latch WorkLeft(1);
- // First remove any cache keys that may own references
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Removed expired data in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (GcReferencer* Owner : m_GcReferencers)
- {
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, Owner, &WorkLeft]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- Owner->RemoveExpiredData(Ctx);
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
+ GcCtx Ctx{.Settings = Settings};
+ GcResult Result;
- if (Ctx.Settings.SkipCidDelete)
{
- return GcResult{.Items = Ctx.Items.load(),
- .ExpiredItems = Ctx.ExpiredItems.load(),
- .DeletedItems = Ctx.DeletedItems.load(),
- .References = Ctx.References.load(),
- .PrunedReferences = Ctx.PrunedReferences.load(),
- .CompactedReferences = Ctx.CompactedReferences.load(),
- .RemovedDiskSpace = Ctx.RemovedDiskSpace.load(),
- .RemovedMemory = Ctx.RemovedMemory.load()};
- }
+ Stopwatch TotalTimer;
+ auto __ = MakeGuard([&]() { Result.ElapsedMS = std::chrono::milliseconds(TotalTimer.GetElapsedTimeMs()); });
- std::vector<std::unique_ptr<GcReferencePruner>> ReferencePruners;
- if (!m_GcReferenceStores.empty())
- {
- ReferencePruners.reserve(m_GcReferenceStores.size());
- Latch WorkLeft(1);
- RwLock ReferencePrunersLock;
- // Easy to go wide, CreateReferencePruner is usually not very heavy but big data sets change that
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Created Cid pruners in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (GcReferenceStore* CidStore : m_GcReferenceStores)
- {
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, CidStore, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // The CidStore will pick a list of CId entries to check, returning a collector
- std::unique_ptr<GcReferencePruner> ReferencePruner(CidStore->CreateReferencePruner(Ctx));
- if (ReferencePruner)
- {
- RwLock::ExclusiveLockScope __(ReferencePrunersLock);
- ReferencePruners.emplace_back(std::move(ReferencePruner));
- }
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
+ RwLock::SharedLockScope GcLock(m_Lock);
- std::vector<std::unique_ptr<GcReferenceChecker>> ReferenceCheckers;
- if (!m_GcReferencers.empty())
- {
- ReferenceCheckers.reserve(m_GcReferencers.size());
- Latch WorkLeft(1);
- RwLock ReferenceCheckersLock;
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Created Cid checkers in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- // Easy to go wide, CreateReferenceCheckers is potentially heavy
- // Lock all reference owners from changing the reference data and get access to check for referenced data
- for (GcReferencer* Referencer : m_GcReferencers)
- {
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Referencer, &ReferenceCheckersLock, &ReferenceCheckers]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // The Referencer will create a reference checker that guarrantees that the references do not change as long as it lives
- std::vector<GcReferenceChecker*> Checkers = Referencer->CreateReferenceCheckers(Ctx);
- try
- {
- if (!Checkers.empty())
- {
- RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
- for (auto& Checker : Checkers)
- {
- ReferenceCheckers.emplace_back(std::unique_ptr<GcReferenceChecker>(Checker));
- Checker = nullptr;
- }
- }
- }
- catch (std::exception&)
- {
- while (!Checkers.empty())
- {
- delete Checkers.back();
- Checkers.pop_back();
- }
- throw;
- }
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
-
- Stopwatch LockStateTimer;
- if (!ReferenceCheckers.empty())
- {
- // Easy to go wide, locking all references checkers so we hafve a stead state of which references are used
- // From this point we have block all writes to all References (DiskBucket/ProjectStore) until we do delete the ReferenceCheckers
- Latch WorkLeft(1);
-
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Locked Cid checkers in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (std::unique_ptr<GcReferenceChecker>& ReferenceChecker : ReferenceCheckers)
- {
- GcReferenceChecker* Checker = ReferenceChecker.get();
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, Checker, &WorkLeft, &ReferenceCheckers]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- Checker->LockState(Ctx);
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
-
- std::vector<std::unique_ptr<GcReferenceStoreCompactor>> ReferenceStoreCompactors;
- ReferenceStoreCompactors.reserve(ReferencePruners.size());
- if (!ReferencePruners.empty())
- {
- const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
- HashSet UnusedCids(References.begin(), References.end());
- for (const std::unique_ptr<GcReferenceChecker>& ReferenceChecker : ReferenceCheckers)
- {
- ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
- if (UnusedCids.empty())
- {
- return {};
- }
- }
- return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
- };
-
- // Easy to go wide, checking all Cids agains references in cache
- // Ask stores to remove data that the ReferenceCheckers says are not references - this should be a lightweight operation that
- // only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
-
- Latch WorkLeft(1);
- RwLock ReferenceStoreCompactorsLock;
-
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Pruned unreferenced Cid data in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (std::unique_ptr<GcReferencePruner>& ReferencePruner : ReferencePruners)
- {
- GcReferencePruner* Pruner = ReferencePruner.get();
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork(
- [&Ctx, Pruner, &WorkLeft, &GetUnusedReferences, &ReferenceStoreCompactorsLock, &ReferenceStoreCompactors]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
- std::unique_ptr<GcReferenceStoreCompactor> ReferenceCompactor(Pruner->RemoveUnreferencedData(Ctx, GetUnusedReferences));
- if (ReferenceCompactor)
- {
- RwLock::ExclusiveLockScope __(ReferenceStoreCompactorsLock);
- ReferenceStoreCompactors.emplace_back(std::move(ReferenceCompactor));
- }
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
- // Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
- ReferenceCheckers.clear();
- ZEN_INFO("GC: Writes blocked for {}", NiceTimeSpanMs(LockStateTimer.GetElapsedTimeMs()))
-
- // Let go of the pruners
- ReferencePruners.clear();
-
- if (!ReferenceStoreCompactors.empty())
- {
- Latch WorkLeft(1);
-
- // Easy to go wide
- // Remove the stuff we deemed unreferenced from disk - may be heavy operation
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Compacted Cid stores in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (std::unique_ptr<GcReferenceStoreCompactor>& StoreCompactor : ReferenceStoreCompactors)
- {
- GcReferenceStoreCompactor* Compactor = StoreCompactor.get();
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, Compactor, &WorkLeft]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
- Compactor->CompactReferenceStore(Ctx);
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
-
- ReferenceStoreCompactors.clear();
-
- return GcResult{.Items = Ctx.Items.load(),
- .ExpiredItems = Ctx.ExpiredItems.load(),
- .DeletedItems = Ctx.DeletedItems.load(),
- .References = Ctx.References.load(),
- .PrunedReferences = Ctx.PrunedReferences.load(),
- .CompactedReferences = Ctx.CompactedReferences.load(),
- .RemovedDiskSpace = Ctx.RemovedDiskSpace.load(),
- .RemovedMemory = Ctx.RemovedMemory.load()};
+ static const bool SingleThread =
+#if ZEN_BUILD_DEBUG
+ true
+#else
+ false
+#endif
+ ;
+
+#define SCOPED_TIMER(closure) \
+ Stopwatch $Timer##__LINE__; \
+ auto $Guard##__LINE = MakeGuard([&, &Timer = $Timer##__LINE__]() { closure })
+
+ Result.ReferencerStats.resize(m_GcReferencers.size());
+ Result.ReferenceStoreStats.resize(m_GcReferenceStores.size());
+
+ WorkerThreadPool ThreadPool(SingleThread ? 0 : 8);
+
+ ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size());
+ if (!m_GcReferencers.empty())
+ {
+ Latch WorkLeft(1);
+ // First remove any cache keys that may own references
+ SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+ {
+ GcReferencer* Owner = m_GcReferencers[Index];
+ std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, Owner, &Stats, &WorkLeft]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ Stats.first = Owner->GetGcName(Ctx);
+ SCOPED_TIMER(Stats.second.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Owner->RemoveExpiredData(Ctx, Stats.second);
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ if (Ctx.Settings.SkipCidDelete)
+ {
+ Result.Sum();
+ return Result;
+ }
+
+ ZEN_INFO("GCV2: Creating reference pruners from {} reference stores", m_GcReferenceStores.size());
+ std::unordered_map<size_t, std::unique_ptr<GcReferencePruner>> ReferencePruners;
+ if (!m_GcReferenceStores.empty())
+ {
+ ReferencePruners.reserve(m_GcReferenceStores.size());
+ Latch WorkLeft(1);
+ RwLock ReferencePrunersLock;
+ // CreateReferencePruner is usually not very heavy but big data sets change that
+ SCOPED_TIMER(Result.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++)
+ {
+ GcReferenceStore* ReferenceStore = m_GcReferenceStores[Index];
+ std::pair<std::string, GcReferenceStoreStats>& Stats = Result.ReferenceStoreStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ Stats.first = ReferenceStore->GetGcName(Ctx);
+ std::unique_ptr<GcReferencePruner> ReferencePruner;
+ {
+ SCOPED_TIMER(Stats.second.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ // The ReferenceStore will pick a list of CId entries to check, returning a collector
+ ReferencePruner = std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second));
+ }
+ if (ReferencePruner)
+ {
+ RwLock::ExclusiveLockScope __(ReferencePrunersLock);
+ ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
+ }
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ ZEN_INFO("GCV2: Creating reference checkers from {} referencers", m_GcReferencers.size());
+ std::unordered_map<std::unique_ptr<GcReferenceChecker>, size_t> ReferenceCheckers;
+ if (!m_GcReferencers.empty())
+ {
+ ReferenceCheckers.reserve(m_GcReferencers.size());
+ Latch WorkLeft(1);
+ RwLock ReferenceCheckersLock;
+ SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ // Lock all reference owners from changing the reference data and get access to check for referenced data
+ for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+ {
+ GcReferencer* Referencer = m_GcReferencers[Index];
+ std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ // The Referencer will create a reference checker that guarrantees that the references do not change as long as it lives
+ std::vector<GcReferenceChecker*> Checkers;
+ {
+ SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Checkers = Referencer->CreateReferenceCheckers(Ctx);
+ }
+ try
+ {
+ if (!Checkers.empty())
+ {
+ RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
+ for (auto& Checker : Checkers)
+ {
+ ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
+ Checker = nullptr;
+ }
+ }
+ }
+ catch (std::exception&)
+ {
+ while (!Checkers.empty())
+ {
+ delete Checkers.back();
+ Checkers.pop_back();
+ }
+ throw;
+ }
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ std::unordered_map<std::unique_ptr<GcReferenceStoreCompactor>, size_t> ReferenceStoreCompactors;
+ ReferenceStoreCompactors.reserve(ReferencePruners.size());
+
+ ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
+ {
+ SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
+ ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
+ if (!ReferenceCheckers.empty())
+ {
+ // Locking all references checkers so we have a steady state of which references are used
+ // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+ // we delete the ReferenceCheckers
+ Latch WorkLeft(1);
+
+ SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (auto& It : ReferenceCheckers)
+ {
+ GcReferenceChecker* Checker = It.first.get();
+ size_t Index = It.second;
+ std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Checker->LockState(Ctx);
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
+ if (!ReferencePruners.empty())
+ {
+ const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
+ HashSet UnusedCids(References.begin(), References.end());
+ for (const auto& It : ReferenceCheckers)
+ {
+ GcReferenceChecker* ReferenceChecker = It.first.get();
+ ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
+ if (UnusedCids.empty())
+ {
+ return {};
+ }
+ }
+ return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
+ };
+
+ // checking all Cids agains references in cache
+ // Ask stores to remove data that the ReferenceCheckers says are not referenced - this should be a lightweight operation
+ // that only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
+
+ Latch WorkLeft(1);
+ RwLock ReferenceStoreCompactorsLock;
+
+ SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (auto& It : ReferencePruners)
+ {
+ GcReferencePruner* Pruner = It.second.get();
+ size_t Index = It.first;
+ GcReferenceStoreStats& Stats = Result.ReferenceStoreStats[Index].second;
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx,
+ Pruner,
+ &Stats,
+ &WorkLeft,
+ Index,
+ &GetUnusedReferences,
+ &ReferenceStoreCompactorsLock,
+ &ReferenceStoreCompactors]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
+ std::unique_ptr<GcReferenceStoreCompactor> ReferenceCompactor;
+ {
+ SCOPED_TIMER(Stats.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ ReferenceCompactor =
+ std::unique_ptr<GcReferenceStoreCompactor>(Pruner->RemoveUnreferencedData(Ctx, Stats, GetUnusedReferences));
+ }
+ if (ReferenceCompactor)
+ {
+ RwLock::ExclusiveLockScope __(ReferenceStoreCompactorsLock);
+ ReferenceStoreCompactors.insert_or_assign(std::move(ReferenceCompactor), Index);
+ }
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+ // Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
+ ReferenceCheckers.clear();
+ }
+
+ // Let go of the pruners
+ ReferencePruners.clear();
+
+ ZEN_INFO("GCV2: Compacting reference stores for {} reference store compactors", ReferenceStoreCompactors.size());
+ if (!ReferenceStoreCompactors.empty())
+ {
+ Latch WorkLeft(1);
+
+ // Remove the stuff we deemed unreferenced from disk - may be heavy operation
+ SCOPED_TIMER(Result.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (auto& It : ReferenceStoreCompactors)
+ {
+ GcReferenceStoreCompactor* Compactor = It.first.get();
+ size_t Index = It.second;
+ GcReferenceStoreStats& Stats = Result.ReferenceStoreStats[Index].second;
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, Compactor, &Stats, &WorkLeft]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
+ SCOPED_TIMER(Stats.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Compactor->CompactReferenceStore(Ctx, Stats);
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ ReferenceStoreCompactors.clear();
+
+ ZEN_INFO("GCV2: Completed in {}", NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
+ }
+
+ Result.Sum();
+ return Result;
+#undef SCOPED_TIMER
}
//////// End New GC WIP
@@ -998,15 +1064,6 @@ GcScheduler::GetState() const
GcSchedulerState Result{.Status = Status(), .Config = m_Config, .AreDiskWritesBlocked = m_AreDiskWritesBlocked.load()};
- {
- std::unique_lock Lock(m_GcMutex);
- Result.LastFullGcTime = m_LastGcTime;
- Result.LastFullGCDiff = m_LastFullGCDiff;
- Result.LastFullGcDuration = m_LastFullGcDuration;
- Result.LastLightweightGcTime = m_LastLightweightGcTime;
- Result.LastLightweightGCDiff = m_LastLightweightGCDiff;
- Result.LastLightweightGcDuration = m_LastLightweightGcDuration;
- }
std::error_code Ec;
DiskSpace Space = DiskSpaceInfo(Result.Config.RootDirectory, Ec);
if (!Ec)
@@ -1026,30 +1083,40 @@ GcScheduler::GetState() const
Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
}
- GcClock::TimePoint CacheExpireTime =
- Result.Config.MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - Result.Config.MaxCacheDuration;
- GcClock::TimePoint ProjectStoreExpireTime = Result.Config.MaxProjectStoreDuration == GcClock::Duration::max()
- ? GcClock::TimePoint::min()
- : Now - Result.Config.MaxProjectStoreDuration;
+ if (Result.Status != GcSchedulerStatus::kRunning)
+ {
+ {
+ std::unique_lock Lock(m_GcMutex);
+ Result.LastFullGcTime = m_LastGcTime;
+ Result.LastFullGCDiff = m_LastFullGCDiff;
+ Result.LastFullGcDuration = m_LastFullGcDuration;
+ Result.LastLightweightGcTime = m_LastLightweightGcTime;
+ Result.LastLightweightGCDiff = m_LastLightweightGCDiff;
+ Result.LastLightweightGcDuration = m_LastLightweightGcDuration;
+
+ Result.LastLightweightGCV2Result = m_LastLightweightGCV2Result;
+ Result.LastFullGCV2Result = m_LastFullGCV2Result;
+ }
- Result.RemainingTimeUntilFullGc =
- Result.Config.Interval.count() == 0
- ? std::chrono::seconds::max()
- : std::chrono::duration_cast<std::chrono::seconds>(Result.LastFullGcTime + Result.Config.Interval - Now);
+ Result.RemainingTimeUntilFullGc =
+ Result.Config.Interval.count() == 0
+ ? std::chrono::seconds::max()
+ : std::chrono::duration_cast<std::chrono::seconds>(Result.LastFullGcTime + Result.Config.Interval - Now);
- if (Result.RemainingTimeUntilFullGc < std::chrono::seconds::zero())
- {
- Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero();
- }
+ if (Result.RemainingTimeUntilFullGc < std::chrono::seconds::zero())
+ {
+ Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero();
+ }
- Result.RemainingTimeUntilLightweightGc =
- Result.Config.LightweightInterval.count() == 0
- ? std::chrono::seconds::max()
- : std::chrono::duration_cast<std::chrono::seconds>(Result.LastLightweightGcTime + Result.Config.LightweightInterval - Now);
+ Result.RemainingTimeUntilLightweightGc =
+ Result.Config.LightweightInterval.count() == 0
+ ? std::chrono::seconds::max()
+ : std::chrono::duration_cast<std::chrono::seconds>(Result.LastLightweightGcTime + Result.Config.LightweightInterval - Now);
- if (Result.RemainingTimeUntilLightweightGc < std::chrono::seconds::zero())
- {
- Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
+ if (Result.RemainingTimeUntilLightweightGc < std::chrono::seconds::zero())
+ {
+ Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
+ }
}
return Result;
@@ -1474,6 +1541,14 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
{
case GcVersion::kV1:
Diff = m_GcManager.CollectGarbage(GcCtx);
+ if (SkipCid)
+ {
+ m_LastLightweightGCV2Result.reset();
+ }
+ else
+ {
+ m_LastFullGCV2Result.reset();
+ }
break;
case GcVersion::kV2:
{
@@ -1482,7 +1557,33 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
.CollectSmallObjects = CollectSmallObjects,
.IsDeleteMode = Delete,
.SkipCidDelete = SkipCid});
- Diff.DiskSize = Result.RemovedDiskSpace;
+
+ ZEN_INFO(
+ "GCV2: Removed {} items out of {}, deleted {} out of {}. Pruned {} Cid entries out of {}, compacted {} Cid entries "
+ "out of {}, "
+ "freed "
+ "{} on disk and {} of memory in {}",
+ Result.ReferencerStat.Expired,
+ Result.ReferencerStat.Count,
+ Result.ReferencerStat.Deleted,
+ Result.ReferencerStat.Expired,
+ Result.ReferenceStoreStat.Pruned,
+ Result.ReferenceStoreStat.Count,
+ Result.ReferenceStoreStat.Compacted,
+ Result.ReferenceStoreStat.Pruned,
+ NiceBytes(Result.RemovedDisk),
+ NiceBytes(Result.RemovedMemory),
+ NiceTimeSpanMs(Result.ElapsedMS.count()));
+
+ if (SkipCid)
+ {
+ m_LastLightweightGCV2Result = Result;
+ }
+ else
+ {
+ m_LastFullGCV2Result = Result;
+ }
+ Diff.DiskSize = Result.RemovedDisk;
Diff.MemorySize = Result.RemovedMemory;
}
break;
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index fa7dce331..e2e99d5a4 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -61,31 +61,67 @@ struct GcSettings
bool CollectSmallObjects = false;
bool IsDeleteMode = false;
bool SkipCidDelete = false;
+ bool Verbose = false;
+};
+
+struct GcReferencerStats
+{
+ std::uint64_t Count = 0;
+ std::uint64_t Expired = 0;
+ std::uint64_t Deleted = 0;
+ std::uint64_t RemovedDisk = 0;
+ std::uint64_t RemovedMemory = 0;
+
+ std::chrono::milliseconds RemoveExpiredDataMS = {};
+ std::chrono::milliseconds CreateReferenceCheckersMS = {};
+ std::chrono::milliseconds LockStateMS = {};
+ std::chrono::milliseconds ElapsedMS = {};
+};
+
+struct GcReferenceStoreStats
+{
+ std::uint64_t Count = 0;
+ std::uint64_t Pruned = 0;
+ std::uint64_t Compacted = 0;
+ std::uint64_t RemovedDisk = 0;
+ std::uint64_t RemovedMemory = 0;
+
+ std::chrono::milliseconds CreateReferencePrunerMS = {};
+ std::chrono::milliseconds RemoveUnreferencedDataMS = {};
+ std::chrono::milliseconds CompactReferenceStoreMS = {};
+ std::chrono::milliseconds ElapsedMS = {};
};
struct GcResult
{
- uint64_t Items = 0;
- uint64_t ExpiredItems = 0;
- uint64_t DeletedItems = 0;
- uint64_t References = 0;
- uint64_t PrunedReferences = 0;
- uint64_t CompactedReferences = 0;
- uint64_t RemovedDiskSpace = 0;
- uint64_t RemovedMemory = 0;
+ GcReferencerStats ReferencerStat;
+ GcReferenceStoreStats ReferenceStoreStat;
+
+ std::uint64_t RemovedDisk = 0;
+ std::uint64_t RemovedMemory = 0;
+
+ std::vector<std::pair<std::string, GcReferencerStats>> ReferencerStats;
+ std::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats;
+
+ // Wall times, not sum of each
+ std::chrono::milliseconds RemoveExpiredDataMS = {};
+ std::chrono::milliseconds CreateReferenceCheckersMS = {};
+ std::chrono::milliseconds LockStateMS = {};
+
+ std::chrono::milliseconds CreateReferencePrunerMS = {};
+ std::chrono::milliseconds RemoveUnreferencedDataMS = {};
+ std::chrono::milliseconds CompactReferenceStoreMS = {};
+
+ std::chrono::milliseconds WriteBlockMS = {};
+
+ std::chrono::milliseconds ElapsedMS = {};
+
+ void Sum();
};
struct GcCtx
{
- const GcSettings Settings;
- std::atomic_uint64_t Items = 0;
- std::atomic_uint64_t ExpiredItems = 0;
- std::atomic_uint64_t DeletedItems = 0;
- std::atomic_uint64_t References = 0;
- std::atomic_uint64_t PrunedReferences = 0;
- std::atomic_uint64_t CompactedReferences = 0;
- std::atomic_uint64_t RemovedDiskSpace = 0;
- std::atomic_uint64_t RemovedMemory = 0;
+ const GcSettings Settings;
};
typedef tsl::robin_set<IoHash> HashSet;
@@ -106,7 +142,7 @@ public:
virtual ~GcReferenceStoreCompactor() = default;
// Remove data on disk based on results from GcReferencePruner::RemoveUnreferencedData
- virtual void CompactReferenceStore(GcCtx& Ctx) = 0;
+ virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats) = 0;
};
/**
@@ -149,11 +185,13 @@ protected:
virtual ~GcReferencer() = default;
public:
+ virtual std::string GetGcName(GcCtx& Ctx) = 0;
+
// Remove expired data based on either GcCtx::Settings CacheExpireTime/ProjectExpireTime
// TODO: For disk layer we need to first update it with access times from the memory layer
// The implementer of GcReferencer (in our case a disk bucket) does not know about any
// potential memory cache layer :(
- virtual void RemoveExpiredData(GcCtx& Ctx) = 0;
+ virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) = 0;
// Create 0-n GcReferenceChecker for this GcReferencer. Caller will manage lifetime of
// returned instances
@@ -178,7 +216,9 @@ public:
// Caller will manage lifetime of returned instance
// This function should execute as fast as possible, so try to prepare a list of references to check ahead of
// call to this function and make sure the removal of unreferences items is as lightweight as possible.
- virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
+ virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx,
+ GcReferenceStoreStats& Stats,
+ const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
};
/**
@@ -190,9 +230,11 @@ protected:
virtual ~GcReferenceStore() = default;
public:
+ virtual std::string GetGcName(GcCtx& Ctx) = 0;
+
// Create a GcReferencePruner which can check a set of references (decided by implementor) if they are no longer in use
// Caller will manage lifetime of returned instance
- virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx) = 0;
+ virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) = 0;
};
//////// End New GC WIP
@@ -379,6 +421,9 @@ struct GcSchedulerState
GcStorageSize LastFullGCDiff;
std::chrono::milliseconds LastLightweightGcDuration{};
GcStorageSize LastLightweightGCDiff;
+
+ std::optional<GcResult> LastLightweightGCV2Result;
+ std::optional<GcResult> LastFullGCV2Result;
};
class DiskUsageWindow
@@ -461,6 +506,9 @@ private:
std::chrono::milliseconds m_LastLightweightGcDuration{};
GcStorageSize m_LastLightweightGCDiff;
+ std::optional<GcResult> m_LastLightweightGCV2Result;
+ std::optional<GcResult> m_LastFullGCV2Result;
+
std::atomic_uint32_t m_Status{};
std::thread m_GcThread;
mutable std::mutex m_GcMutex;
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-26 10:24:54 +0000