aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2023-10-30 18:29:09 +0100
committerGitHub <[email protected]>2023-10-30 18:29:09 +0100
commitcbdda104ada38108700f9da5b192867d83074119 (patch)
tree98c04b344e041c156fdc1a5c393672bef743be34 /src/zenserver
parentfix changelog (diff)
downloadzen-cbdda104ada38108700f9da5b192867d83074119.tar.xz
zen-cbdda104ada38108700f9da5b192867d83074119.zip
individual gc stats (#506)
- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2 - Feature: New options for zen command `gc-status` - `--details` that enables the detailed output from the last GC operation when using GC V2
Diffstat (limited to 'src/zenserver')
-rw-r--r--src/zenserver/admin/admin.cpp130
-rw-r--r--src/zenserver/cache/cachedisklayer.cpp65
-rw-r--r--src/zenserver/cache/cachedisklayer.h3
-rw-r--r--src/zenserver/cache/structuredcachestore.cpp306
-rw-r--r--src/zenserver/projectstore/projectstore.cpp91
-rw-r--r--src/zenserver/projectstore/projectstore.h3
6 files changed, 402 insertions, 196 deletions
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index fb376f238..83f33d4ea 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -199,6 +199,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
[this](HttpRouterRequest& Req) {
const GcSchedulerState State = m_GcScheduler.GetState();
+ const HttpServerRequest::QueryParams Params = Req.ServerRequest().GetQueryParams();
+
+ bool Details = false;
+ if (auto Param = Params.GetValue("details"); Param == "true")
+ {
+ Details = true;
+ }
+
auto SecondsToString = [](std::chrono::seconds Secs) {
return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count()));
};
@@ -226,6 +234,101 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
Response << "DiskUsed" << NiceBytes(State.DiskUsed);
Response << "DiskFree" << NiceBytes(State.DiskFree);
+ auto WriteReferencerStats = [&](CbObjectWriter& Response, const GcReferencerStats& Stats) {
+ if (Stats.Count == 0)
+ {
+ return;
+ }
+ Response << "Count" << Stats.Count;
+ Response << "Expired" << Stats.Expired;
+ Response << "Deleted" << Stats.Deleted;
+
+ Response << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
+ Response << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
+
+ Response << "RemoveExpiredData" << NiceTimeSpanMs(Stats.RemoveExpiredDataMS.count());
+ Response << "CreateReferenceCheckers" << NiceTimeSpanMs(Stats.CreateReferenceCheckersMS.count());
+ Response << "LockState" << NiceTimeSpanMs(Stats.LockStateMS.count());
+ Response << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
+ };
+
+ auto WriteReferenceStoreStats = [&](CbObjectWriter& Response, const GcReferenceStoreStats& Stats) {
+ if (Stats.Count == 0)
+ {
+ return;
+ }
+ Response << "Count" << Stats.Count;
+ Response << "Pruned" << Stats.Pruned;
+ Response << "Compacted" << Stats.Compacted;
+
+ Response << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
+ Response << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
+
+ Response << "CreateReferencePruner" << NiceTimeSpanMs(Stats.CreateReferencePrunerMS.count());
+ Response << "RemoveUnreferencedData" << NiceTimeSpanMs(Stats.RemoveUnreferencedDataMS.count());
+ Response << "CompactReferenceStore" << NiceTimeSpanMs(Stats.CompactReferenceStoreMS.count());
+ Response << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
+ };
+
+ auto WriteGCResult = [&](CbObjectWriter& Response, const GcResult& Result) {
+ Response << "RemovedDisk" << NiceBytes(Result.RemovedDisk);
+ Response << "RemovedMemory" << NiceBytes(Result.RemovedMemory);
+ Response << "WriteBlock" << NiceTimeSpanMs(Result.WriteBlockMS.count());
+ Response << "Elapsed" << NiceTimeSpanMs(Result.ElapsedMS.count());
+
+ if (!Details)
+ {
+ return;
+ }
+
+ Response << "RemoveExpiredData" << NiceTimeSpanMs(Result.RemoveExpiredDataMS.count());
+ Response << "CreateReferenceCheckers" << NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count());
+ Response << "LockState" << NiceTimeSpanMs(Result.LockStateMS.count());
+
+ Response << "CreateReferencePruner" << NiceTimeSpanMs(Result.CreateReferencePrunerMS.count());
+ Response << "RemoveUnreferencedData" << NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count());
+ Response << "CompactReferenceStore" << NiceTimeSpanMs(Result.CompactReferenceStoreMS.count());
+
+ Response.BeginObject("ReferencerStats");
+ {
+ WriteReferencerStats(Response, Result.ReferencerStat);
+ }
+ Response.EndObject();
+
+ Response.BeginObject("ReferenceStoreStats");
+ {
+ WriteReferenceStoreStats(Response, Result.ReferenceStoreStat);
+ }
+ Response.EndObject();
+
+ if (!Result.ReferencerStats.empty())
+ {
+ Response.BeginArray("Referencers");
+ {
+ for (const std::pair<std::string, GcReferencerStats>& It : Result.ReferencerStats)
+ {
+ Response.BeginObject();
+ Response << "Name" << It.first;
+ WriteReferencerStats(Response, It.second);
+ Response.EndObject();
+ }
+ }
+ Response.EndArray();
+ }
+ if (!Result.ReferenceStoreStats.empty())
+ {
+ Response.BeginArray("ReferenceStores");
+ for (const std::pair<std::string, GcReferenceStoreStats>& It : Result.ReferenceStoreStats)
+ {
+ Response.BeginObject();
+ Response << "Name" << It.first;
+ WriteReferenceStoreStats(Response, It.second);
+ Response.EndObject();
+ }
+ Response.EndArray();
+ }
+ };
+
Response.BeginObject("FullGC");
{
Response << "LastTime" << fmt::format("{}", State.LastFullGcTime);
@@ -234,18 +337,33 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
{
Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC);
}
- Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count());
- Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
- Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
+ if (State.LastFullGCV2Result)
+ {
+ WriteGCResult(Response, State.LastFullGCV2Result.value());
+ }
+ else
+ {
+ Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count());
+ Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
+ Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
+ }
}
Response.EndObject();
Response.BeginObject("LightweightGC");
{
Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime);
Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc);
- Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count());
- Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
- Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
+
+ if (State.LastLightweightGCV2Result)
+ {
+ WriteGCResult(Response, State.LastLightweightGCV2Result.value());
+ }
+ else
+ {
+ Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count());
+ Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
+ Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
+ }
}
Response.EndObject();
Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 38cbf3a93..fa80ed414 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -2213,8 +2213,14 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
});
}
+std::string
+ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
+{
+ return fmt::format("cachebucket:'{}'", m_BucketDir.string());
+}
+
void
-ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
+ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
{
size_t TotalEntries = 0;
tsl::robin_set<IoHash, IoHash::Hasher> ExpiredInlineKeys;
@@ -2222,11 +2228,18 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc cache bucket '{}': removed {} expired keys out of {} in {}",
- m_BucketDir,
- ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size(),
- TotalEntries,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_BucketDir,
+ Stats.Count,
+ Stats.Expired,
+ Stats.Deleted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -2278,7 +2291,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
}
}
- Ctx.ExpiredItems.fetch_add(ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size());
+ Stats.Expired += ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size();
// Get all locations we need to keep for affected blocks
if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2312,7 +2325,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[It->second])
{
size_t PayloadSize = m_CachedPayloads[It->second].GetSize();
- Ctx.RemovedMemory.fetch_add(PayloadSize);
+ Stats.RemovedMemory += PayloadSize;
RemoveMemCacheUsage(PayloadSize);
}
m_Index.erase(It);
@@ -2321,7 +2334,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed);
}
}
- Ctx.Items.fetch_add(TotalEntries);
+ Stats.Count += TotalEntries;
if (ExpiredEntries.empty())
{
@@ -2333,7 +2346,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
return;
}
- Ctx.DeletedItems.fetch_add(ExpiredEntries.size());
+ Stats.Deleted += ExpiredEntries.size();
// Compact standalone items
ExtendablePathBuilder<256> Path;
@@ -2370,7 +2383,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
Ec.message());
continue;
}
- Ctx.RemovedDiskSpace.fetch_add(ExpiredKey.second);
+ Stats.RemovedDisk += ExpiredKey.second;
}
if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2405,7 +2418,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
}
}
m_SlogFile.Append(MovedEntries);
- Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+ Stats.RemovedDisk += FreedDiskSpace;
},
[&]() { return 0; });
}
@@ -2436,14 +2449,18 @@ public:
}
}
- virtual void LockState(GcCtx&) override
+ virtual void LockState(GcCtx& Ctx) override
{
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc cache bucket '{}': found {} references in {}",
- m_CacheBucket.m_BucketDir,
- m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}",
+ m_CacheBucket.m_BucketDir,
+ m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
@@ -2503,11 +2520,19 @@ public:
};
std::vector<GcReferenceChecker*>
-ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx&)
+ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
{
Stopwatch Timer;
- const auto _ = MakeGuard(
- [&] { ZEN_DEBUG("gc cache bucket '{}': refreshed reference cache in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': found {} references in {}",
+ m_BucketDir,
+ m_ReferenceCount,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
std::vector<IoHash> UpdateKeys;
std::vector<IoHash> StandaloneKeys;
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index d8f51c398..ded8f2e70 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -267,7 +267,8 @@ private:
std::atomic_uint64_t m_StandaloneSize{};
std::atomic_uint64_t m_MemCachedSize{};
- virtual void RemoveExpiredData(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 516532528..55ab6bf31 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -1982,18 +1982,20 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() - std::chrono::hours(1),
.ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
.CollectSmallObjects = false,
- .IsDeleteMode = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(0u, Result.ExpiredItems);
- CHECK_EQ(0u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(0u, Result.RemovedDiskSpace);
+ .IsDeleteMode = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(0u, Result.ReferencerStat.Expired);
+ CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(0u, Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2014,18 +2016,20 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = false,
- .IsDeleteMode = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems);
- CHECK_EQ(0u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(0u, Result.RemovedDiskSpace);
+ .IsDeleteMode = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired);
+ CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(0u, Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2046,18 +2050,20 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
- .IsDeleteMode = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(7u, Result.ExpiredItems);
- CHECK_EQ(0u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(0u, Result.RemovedDiskSpace);
+ .IsDeleteMode = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(7u, Result.ReferencerStat.Expired);
+ CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(0u, Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2078,19 +2084,21 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = false,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems);
- CHECK_EQ(1u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired);
+ CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2111,19 +2119,21 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(7u, Result.ExpiredItems);
- CHECK_EQ(7u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GE(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(7u, Result.ReferencerStat.Expired);
+ CHECK_EQ(7u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GE(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
@@ -2144,21 +2154,24 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = false,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems); // Only one cache value is pruned/deleted as that is the only large item in the cache (all other
- // large items as in cas)
- CHECK_EQ(1u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired); // Only one cache value is pruned/deleted as that is the only large item in the cache
+ // (all other large items as in cas)
+ CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
CHECK_EQ(0u,
- Result.PrunedReferences); // We won't remove any references since all referencers are small which retains all references
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+ Result.ReferenceStoreStat
+ .Pruned); // We won't remove any references since all referencers are small which retains all references
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2179,18 +2192,21 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .CollectSmallObjects = true,
- .IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(7u, Result.ExpiredItems);
- CHECK_EQ(7u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(5u, Result.PrunedReferences);
- CHECK_EQ(5u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .CollectSmallObjects = true,
+ .IsDeleteMode = true,
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(7u, Result.ReferencerStat.Expired);
+ CHECK_EQ(7u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
@@ -2212,20 +2228,23 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(6u, Result.ExpiredItems);
- CHECK_EQ(6u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(6u, Result.ReferencerStat.Expired);
+ CHECK_EQ(6u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2247,21 +2266,24 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(5u, Result.ExpiredItems);
- CHECK_EQ(5u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(5u, Result.ReferencerStat.Expired);
+ CHECK_EQ(5u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2283,22 +2305,25 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(4u, Result.ExpiredItems);
- CHECK_EQ(4u, Result.DeletedItems);
- CHECK_EQ(5u, Result.References);
- CHECK_EQ(5u, Result.PrunedReferences);
- CHECK_EQ(5u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(4u, Result.ReferencerStat.Expired);
+ CHECK_EQ(4u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
CHECK_EQ(0u, Result.RemovedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
@@ -2320,6 +2345,7 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
// Prime so we can check GC of memory layer
ZenCacheValue Dummy;
@@ -2331,22 +2357,23 @@ TEST_CASE("z$.newgc.basics")
Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[2], Dummy);
Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[3], Dummy);
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = true});
- CHECK_EQ(7u, Result.Items);
- CHECK_EQ(4u, Result.ExpiredItems);
- CHECK_EQ(4u, Result.DeletedItems);
- CHECK_EQ(0u, Result.References);
- CHECK_EQ(0u, Result.PrunedReferences);
- CHECK_EQ(0u, Result.CompactedReferences);
- CHECK_GT(Result.RemovedDiskSpace, 0);
+ .SkipCidDelete = true,
+ .Verbose = true});
+ CHECK_EQ(7u, Result.ReferencerStat.Count);
+ CHECK_EQ(4u, Result.ReferencerStat.Expired);
+ CHECK_EQ(4u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+ CHECK_GT(Result.RemovedDisk, 0);
uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() +
CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize();
CHECK_EQ(MemoryClean, Result.RemovedMemory);
@@ -2370,35 +2397,36 @@ TEST_CASE("z$.newgc.basics")
*JobQueue,
TempDir.Path() / "cache",
{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+ CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
auto Attachments =
CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::hours(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::hours(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
- Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+ Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
.CollectSmallObjects = true,
.IsDeleteMode = true,
- .SkipCidDelete = false});
- CHECK_EQ(8u, Result.Items);
- CHECK_EQ(1u, Result.ExpiredItems);
- CHECK_EQ(1u, Result.DeletedItems);
- CHECK_EQ(9u, Result.References);
- CHECK_EQ(4u, Result.PrunedReferences);
- CHECK_EQ(4u, Result.CompactedReferences);
- CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(),
- Result.RemovedDiskSpace);
+ .SkipCidDelete = false,
+ .Verbose = true});
+ CHECK_EQ(8u, Result.ReferencerStat.Count);
+ CHECK_EQ(1u, Result.ReferencerStat.Expired);
+ CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+ CHECK_EQ(9u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(4u, Result.ReferenceStoreStat.Pruned);
+ CHECK_EQ(4u, Result.ReferenceStoreStat.Compacted);
+ CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(), Result.RemovedDisk);
uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize();
CHECK_EQ(MemoryClean, Result.RemovedMemory);
}
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 274876123..7da2f7695 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3036,8 +3036,14 @@ ProjectStore::AreDiskWritesAllowed() const
return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed());
}
+std::string
+ProjectStore::GetGcName(GcCtx&)
+{
+ return fmt::format("projectstore:'{}'", m_ProjectBasePath.string());
+}
+
void
-ProjectStore::RemoveExpiredData(GcCtx& Ctx)
+ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
{
size_t ProjectCount = 0;
size_t ExpiredProjectCount = 0;
@@ -3045,13 +3051,18 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
size_t ExpiredOplogCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc project store '{}': removed {} expired projects out of {}, {} expired oplogs out of {} in {}",
- m_ProjectBasePath,
- ExpiredProjectCount,
- ProjectCount,
- ExpiredOplogCount,
- OplogCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_ProjectBasePath,
+ Stats.Count,
+ Stats.Expired,
+ Stats.Deleted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<Ref<Project>> ExpiredProjects;
@@ -3097,14 +3108,14 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
OplogId,
Project->Identifier);
Project->DeleteOplog(OplogId);
- Ctx.RemovedDiskSpace.fetch_add(OplogSize);
+ Stats.RemovedDisk += OplogSize;
}
- Ctx.DeletedItems.fetch_add(ExpiredOplogs.size());
+ Stats.Deleted += ExpiredOplogs.size();
Project->Flush();
}
}
ProjectCount = Projects.size();
- Ctx.Items.fetch_add(ProjectCount + OplogCount);
+ Stats.Count += ProjectCount + OplogCount;
ExpiredProjectCount = ExpiredProjects.size();
if (ExpiredProjects.empty())
@@ -3150,20 +3161,34 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
DeleteDirectories(PathToRemove);
}
- Ctx.DeletedItems.fetch_add(ExpiredProjects.size());
+ Stats.Deleted += ExpiredProjects.size();
}
- Ctx.ExpiredItems.fetch_add(ExpiredOplogCount + ExpiredProjectCount);
+ Stats.Expired += ExpiredOplogCount + ExpiredProjectCount;
}
class ProjectStoreReferenceChecker : public GcReferenceChecker
{
public:
- ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner)
+ ProjectStoreReferenceChecker(GcCtx& Ctx, ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner)
{
if (PreCache)
{
- RwLock::SharedLockScope _(m_Oplog.m_OplogLock);
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': precached {} references in {} from {}/{}",
+ m_Oplog.m_BasePath,
+ m_UncachedReferences.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ m_Oplog.m_OuterProject->Identifier,
+ m_Oplog.OplogId());
+ });
+
+ RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
m_Oplog.IterateOplog([&](CbObjectView Op) {
Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
});
@@ -3173,16 +3198,20 @@ public:
virtual ~ProjectStoreReferenceChecker() {}
- virtual void LockState(GcCtx&) override
+ virtual void LockState(GcCtx& Ctx) override
{
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc project oplog '{}': found {} references in {} from {}/{}",
- m_Oplog.m_BasePath,
- m_UncachedReferences.size(),
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- m_Oplog.m_OuterProject->Identifier,
- m_Oplog.OplogId());
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
+ m_Oplog.m_BasePath,
+ m_UncachedReferences.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ m_Oplog.m_OuterProject->Identifier,
+ m_Oplog.OplogId());
});
m_OplogLock = std::make_unique<RwLock::SharedLockScope>(m_Oplog.m_OplogLock);
@@ -3211,18 +3240,22 @@ public:
};
std::vector<GcReferenceChecker*>
-ProjectStore::CreateReferenceCheckers(GcCtx&)
+ProjectStore::CreateReferenceCheckers(GcCtx& Ctx)
{
size_t ProjectCount = 0;
size_t OplogCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc project store '{}': opened {} projects and {} oplogs in {}",
- m_ProjectBasePath,
- ProjectCount,
- OplogCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [CREATE CHECKERS] '{}': opened {} projects and {} oplogs in {}",
+ m_ProjectBasePath,
+ ProjectCount,
+ OplogCount,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
DiscoverProjects();
@@ -3250,7 +3283,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx&)
ProjectStore::Oplog* Oplog = Project->OpenOplog(OpLogId);
GcClock::TimePoint Now = GcClock::Now();
bool TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5));
- Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache));
+ Checkers.emplace_back(new ProjectStoreReferenceChecker(Ctx, *Oplog, TryPreCache));
}
OplogCount += OpLogs.size();
}
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 94e697278..fbff1444d 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -293,7 +293,8 @@ public:
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override;
- virtual void RemoveExpiredData(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
CbArray GetProjectsList();