individual gc stats (#506)

- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2
- Feature: New options for zen command `gc-status`
  - `--details` that enables the detailed output from the last GC operation when using GC V2

3 files changed, 214 insertions, 160 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 38cbf3a93..fa80ed414 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -2213,8 +2213,14 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 		});
 }
 
+std::string
+ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
+{
+	return fmt::format("cachebucket:'{}'", m_BucketDir.string());
+}
+
 void
-ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
+ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
 {
 	size_t									 TotalEntries = 0;
 	tsl::robin_set<IoHash, IoHash::Hasher>	 ExpiredInlineKeys;
@@ -2222,11 +2228,18 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
-		ZEN_DEBUG("gc cache bucket '{}': removed {} expired keys out of {} in {}",
-				  m_BucketDir,
-				  ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size(),
-				  TotalEntries,
-				  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+				 m_BucketDir,
+				 Stats.Count,
+				 Stats.Expired,
+				 Stats.Deleted,
+				 NiceBytes(Stats.RemovedDisk),
+				 NiceBytes(Stats.RemovedMemory),
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -2278,7 +2291,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 			}
 		}
 
-		Ctx.ExpiredItems.fetch_add(ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size());
+		Stats.Expired += ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size();
 
 		// Get all locations we need to keep for affected blocks
 		if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2312,7 +2325,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 				if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[It->second])
 				{
 					size_t PayloadSize = m_CachedPayloads[It->second].GetSize();
-					Ctx.RemovedMemory.fetch_add(PayloadSize);
+					Stats.RemovedMemory += PayloadSize;
 					RemoveMemCacheUsage(PayloadSize);
 				}
 				m_Index.erase(It);
@@ -2321,7 +2334,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 			m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed);
 		}
 	}
-	Ctx.Items.fetch_add(TotalEntries);
+	Stats.Count += TotalEntries;
 
 	if (ExpiredEntries.empty())
 	{
@@ -2333,7 +2346,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 		return;
 	}
 
-	Ctx.DeletedItems.fetch_add(ExpiredEntries.size());
+	Stats.Deleted += ExpiredEntries.size();
 
 	// Compact standalone items
 	ExtendablePathBuilder<256> Path;
@@ -2370,7 +2383,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 					 Ec.message());
 			continue;
 		}
-		Ctx.RemovedDiskSpace.fetch_add(ExpiredKey.second);
+		Stats.RemovedDisk += ExpiredKey.second;
 	}
 
 	if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
@@ -2405,7 +2418,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
 					}
 				}
 				m_SlogFile.Append(MovedEntries);
-				Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+				Stats.RemovedDisk += FreedDiskSpace;
 			},
 			[&]() { return 0; });
 	}
@@ -2436,14 +2449,18 @@ public:
 		}
 	}
 
-	virtual void LockState(GcCtx&) override
+	virtual void LockState(GcCtx& Ctx) override
 	{
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
-			ZEN_DEBUG("gc cache bucket '{}': found {} references in {}",
-					  m_CacheBucket.m_BucketDir,
-					  m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
-					  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}",
+					 m_CacheBucket.m_BucketDir,
+					 m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
@@ -2503,11 +2520,19 @@ public:
 };
 
 std::vector<GcReferenceChecker*>
-ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx&)
+ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
 {
 	Stopwatch  Timer;
-	const auto _ = MakeGuard(
-		[&] { ZEN_DEBUG("gc cache bucket '{}': refreshed reference cache in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+	const auto _ = MakeGuard([&] {
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': found {} references in {}",
+				 m_BucketDir,
+				 m_ReferenceCount,
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
 
 	std::vector<IoHash> UpdateKeys;
 	std::vector<IoHash> StandaloneKeys;
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index d8f51c398..ded8f2e70 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -267,7 +267,8 @@ private:
 		std::atomic_uint64_t	   m_StandaloneSize{};
 		std::atomic_uint64_t	   m_MemCachedSize{};
 
-		virtual void							 RemoveExpiredData(GcCtx& Ctx) override;
+		virtual std::string						 GetGcName(GcCtx& Ctx) override;
+		virtual void							 RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
 		virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
 
 		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 516532528..55ab6bf31 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -1982,18 +1982,20 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(1),
 													   .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
 													   .CollectSmallObjects	   = false,
-													   .IsDeleteMode		   = false});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(0u, Result.ExpiredItems);
-		CHECK_EQ(0u, Result.DeletedItems);
-		CHECK_EQ(5u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_EQ(0u, Result.RemovedDiskSpace);
+													   .IsDeleteMode		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(0u, Result.ReferencerStat.Expired);
+		CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_EQ(0u, Result.RemovedDisk);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2014,18 +2016,20 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = false,
-													   .IsDeleteMode		   = false});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(1u, Result.ExpiredItems);
-		CHECK_EQ(0u, Result.DeletedItems);
-		CHECK_EQ(5u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_EQ(0u, Result.RemovedDiskSpace);
+													   .IsDeleteMode		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(1u, Result.ReferencerStat.Expired);
+		CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_EQ(0u, Result.RemovedDisk);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2046,18 +2050,20 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = true,
-													   .IsDeleteMode		   = false});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(7u, Result.ExpiredItems);
-		CHECK_EQ(0u, Result.DeletedItems);
-		CHECK_EQ(5u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_EQ(0u, Result.RemovedDiskSpace);
+													   .IsDeleteMode		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(7u, Result.ReferencerStat.Expired);
+		CHECK_EQ(0u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_EQ(0u, Result.RemovedDisk);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2078,19 +2084,21 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = false,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = true});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(1u, Result.ExpiredItems);
-		CHECK_EQ(1u, Result.DeletedItems);
-		CHECK_EQ(0u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+													   .SkipCidDelete		   = true,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(1u, Result.ReferencerStat.Expired);
+		CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2111,19 +2119,21 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = true,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = true});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(7u, Result.ExpiredItems);
-		CHECK_EQ(7u, Result.DeletedItems);
-		CHECK_EQ(0u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_GE(Result.RemovedDiskSpace, 0);
+													   .SkipCidDelete		   = true,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(7u, Result.ReferencerStat.Expired);
+		CHECK_EQ(7u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_GE(Result.RemovedDisk, 0);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
@@ -2144,21 +2154,24 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = false,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = false});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(1u, Result.ExpiredItems);	// Only one cache value is pruned/deleted as that is the only large item in the cache (all other
-											// large items as in cas)
-		CHECK_EQ(1u, Result.DeletedItems);
-		CHECK_EQ(5u, Result.References);
+													   .SkipCidDelete		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(1u, Result.ReferencerStat.Expired);  // Only one cache value is pruned/deleted as that is the only large item in the cache
+													  // (all other large items as in cas)
+		CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
 		CHECK_EQ(0u,
-				 Result.PrunedReferences);	// We won't remove any references since all referencers are small which retains all references
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+				 Result.ReferenceStoreStat
+					 .Pruned);	// We won't remove any references since all referencers are small which retains all references
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2179,18 +2192,21 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
-		GcResult		  Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		= GcClock::Now() + std::chrono::minutes(1),
-																.ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
-																.CollectSmallObjects	= true,
-																.IsDeleteMode			= true,
-																.SkipCidDelete			= false});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(7u, Result.ExpiredItems);
-		CHECK_EQ(7u, Result.DeletedItems);
-		CHECK_EQ(5u, Result.References);
-		CHECK_EQ(5u, Result.PrunedReferences);
-		CHECK_EQ(5u, Result.CompactedReferences);
-		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(7u, Result.ReferencerStat.Expired);
+		CHECK_EQ(7u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
+		CHECK_GT(Result.RemovedDisk, 0);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
@@ -2212,20 +2228,23 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
-		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = true,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = true});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(6u, Result.ExpiredItems);
-		CHECK_EQ(6u, Result.DeletedItems);
-		CHECK_EQ(0u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_GT(Result.RemovedDiskSpace, 0);
+													   .SkipCidDelete		   = true,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(6u, Result.ReferencerStat.Expired);
+		CHECK_EQ(6u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_GT(Result.RemovedDisk, 0);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2247,21 +2266,24 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
-		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2));
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = true,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = false});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(5u, Result.ExpiredItems);
-		CHECK_EQ(5u, Result.DeletedItems);
-		CHECK_EQ(5u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_GT(Result.RemovedDiskSpace, 0);
+													   .SkipCidDelete		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(5u, Result.ReferencerStat.Expired);
+		CHECK_EQ(5u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_GT(Result.RemovedDisk, 0);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
@@ -2283,22 +2305,25 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
+
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = true,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = false});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(4u, Result.ExpiredItems);
-		CHECK_EQ(4u, Result.DeletedItems);
-		CHECK_EQ(5u, Result.References);
-		CHECK_EQ(5u, Result.PrunedReferences);
-		CHECK_EQ(5u, Result.CompactedReferences);
-		CHECK_GT(Result.RemovedDiskSpace, 0);
+													   .SkipCidDelete		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(4u, Result.ReferencerStat.Expired);
+		CHECK_EQ(4u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
+		CHECK_GT(Result.RemovedDisk, 0);
 		CHECK_EQ(0u, Result.RemovedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
@@ -2320,6 +2345,7 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		// Prime so we can check GC of memory layer
 		ZenCacheValue Dummy;
@@ -2331,22 +2357,23 @@ TEST_CASE("z$.newgc.basics")
 		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[2], Dummy);
 		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[3], Dummy);
 
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = true,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = true});
-		CHECK_EQ(7u, Result.Items);
-		CHECK_EQ(4u, Result.ExpiredItems);
-		CHECK_EQ(4u, Result.DeletedItems);
-		CHECK_EQ(0u, Result.References);
-		CHECK_EQ(0u, Result.PrunedReferences);
-		CHECK_EQ(0u, Result.CompactedReferences);
-		CHECK_GT(Result.RemovedDiskSpace, 0);
+													   .SkipCidDelete		   = true,
+													   .Verbose				   = true});
+		CHECK_EQ(7u, Result.ReferencerStat.Count);
+		CHECK_EQ(4u, Result.ReferencerStat.Expired);
+		CHECK_EQ(4u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
+		CHECK_GT(Result.RemovedDisk, 0);
 		uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() +
 							   CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize();
 		CHECK_EQ(MemoryClean, Result.RemovedMemory);
@@ -2370,35 +2397,36 @@ TEST_CASE("z$.newgc.basics")
 							  *JobQueue,
 							  TempDir.Path() / "cache",
 							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		auto Attachments =
 			CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
 		IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
 
-		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::hours(2));
 
-		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::hours(2));
 
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
-		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
 													   .CollectSmallObjects	   = true,
 													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = false});
-		CHECK_EQ(8u, Result.Items);
-		CHECK_EQ(1u, Result.ExpiredItems);
-		CHECK_EQ(1u, Result.DeletedItems);
-		CHECK_EQ(9u, Result.References);
-		CHECK_EQ(4u, Result.PrunedReferences);
-		CHECK_EQ(4u, Result.CompactedReferences);
-		CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(),
-				 Result.RemovedDiskSpace);
+													   .SkipCidDelete		   = false,
+													   .Verbose				   = true});
+		CHECK_EQ(8u, Result.ReferencerStat.Count);
+		CHECK_EQ(1u, Result.ReferencerStat.Expired);
+		CHECK_EQ(1u, Result.ReferencerStat.Deleted);
+		CHECK_EQ(9u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(4u, Result.ReferenceStoreStat.Pruned);
+		CHECK_EQ(4u, Result.ReferenceStoreStat.Compacted);
+		CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(), Result.RemovedDisk);
 		uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize();
 		CHECK_EQ(MemoryClean, Result.RemovedMemory);
 	}