individual gc stats (#506)

- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2
- Feature: New options for zen command `gc-status`
  - `--details` that enables the detailed output from the last GC operation when using GC V2

1 files changed, 45 insertions, 20 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 38cbf3a93..fa80ed414 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -2213,8 +2213,14 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 		});
 }
 
+std::string
+ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
+{
+	return fmt::format("cachebucket:'{}'", m_BucketDir.string());
+}
+
 void
-ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
+ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
 {
 	size_t									 TotalEntries = 0;
 	tsl::robin_set<IoHash, IoHash::Hasher>	 ExpiredInlineKeys;
@@ -2222,11 +2228,18 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
-		ZEN_DEBUG("gc cache bucket '{}': removed {} expired keys out of {} in {}",
-				  m_BucketDir,
-				  ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size(),
-				  TotalEntries,
-				  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+				 m_BucketDir,
+				 Stats.Count,
+				 Stats.Expired,
+				 Stats.Deleted,
+				 NiceBytes(Stats.RemovedDisk),
+				 NiceBytes(Stats.RemovedMemory),
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -2278,7 +2291,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 			}
 		}
 
-		Ctx.ExpiredItems.fetch_add(ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size());
+		Stats.Expired += ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size();
 
 		// Get all locations we need to keep for affected blocks
 		if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2312,7 +2325,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 				if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[It->second])
 				{
 					size_t PayloadSize = m_CachedPayloads[It->second].GetSize();
-					Ctx.RemovedMemory.fetch_add(PayloadSize);
+					Stats.RemovedMemory += PayloadSize;
 					RemoveMemCacheUsage(PayloadSize);
 				}
 				m_Index.erase(It);
@@ -2321,7 +2334,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 			m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed);
 		}
 	}
-	Ctx.Items.fetch_add(TotalEntries);
+	Stats.Count += TotalEntries;
 
 	if (ExpiredEntries.empty())
 	{
@@ -2333,7 +2346,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 		return;
 	}
 
-	Ctx.DeletedItems.fetch_add(ExpiredEntries.size());
+	Stats.Deleted += ExpiredEntries.size();
 
 	// Compact standalone items
 	ExtendablePathBuilder<256> Path;
@@ -2370,7 +2383,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 					 Ec.message());
 			continue;
 		}
-		Ctx.RemovedDiskSpace.fetch_add(ExpiredKey.second);
+		Stats.RemovedDisk += ExpiredKey.second;
 	}
 
 	if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2405,7 +2418,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 					}
 				}
 				m_SlogFile.Append(MovedEntries);
-				Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+				Stats.RemovedDisk += FreedDiskSpace;
 			},
 			[&]() { return 0; });
 	}
@@ -2436,14 +2449,18 @@ public:
 		}
 	}
 
-	virtual void LockState(GcCtx&) override
+	virtual void LockState(GcCtx& Ctx) override
 	{
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
-			ZEN_DEBUG("gc cache bucket '{}': found {} references in {}",
-					  m_CacheBucket.m_BucketDir,
-					  m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
-					  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}",
+					 m_CacheBucket.m_BucketDir,
+					 m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
@@ -2503,11 +2520,19 @@ public:
 };
 
 std::vector<GcReferenceChecker*>
-ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx&)
+ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
 {
 	Stopwatch  Timer;
-	const auto _ = MakeGuard(
-		[&] { ZEN_DEBUG("gc cache bucket '{}': refreshed reference cache in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+	const auto _ = MakeGuard([&] {
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': found {} references in {}",
+				 m_BucketDir,
+				 m_ReferenceCount,
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
 
 	std::vector<IoHash> UpdateKeys;
 	std::vector<IoHash> StandaloneKeys;