individual gc stats (#506)

- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2
- Feature: New options for zen command `gc-status`
  - `--details` that enables the detailed output from the last GC operation when using GC V2

1 files changed, 56 insertions, 33 deletions

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index f93dafa21..655c0558d 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -567,15 +567,22 @@ public:
 	{
 	}
 
-	virtual void CompactReferenceStore(GcCtx& Ctx)
+	virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
 	{
-		size_t	   CompactedCount = 0;
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
-			ZEN_DEBUG("gc block store '{}': compacted {} cids in {}",
-					  m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
-					  CompactedCount,
-					  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: compactcas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+					 Stats.Count,
+					 Stats.Pruned,
+					 Stats.Compacted,
+					 NiceBytes(Stats.RemovedDisk),
+					 NiceBytes(Stats.RemovedMemory),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		if (Ctx.Settings.IsDeleteMode && Ctx.Settings.CollectSmallObjects)
@@ -609,13 +616,12 @@ public:
 						}
 					}
 					m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
-					Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+					Stats.RemovedDisk += FreedDiskSpace;
 				},
 				[&]() { return 0; });
 
-			CompactedCount = m_PrunedKeys.size();
-			Ctx.CompactedReferences.fetch_add(
-				CompactedCount);  // Slightly missleading, it might not be compacted if the block is the currently writing block
+			Stats.Compacted +=
+				m_PrunedKeys.size();  // Slightly missleading, it might not be compacted if the block is the currently writing block
 		}
 	}
 
@@ -634,21 +640,27 @@ public:
 	{
 	}
 
-	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences)
+	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx&						 Ctx,
+															  GcReferenceStoreStats&		 Stats,
+															  const GetUnusedReferencesFunc& GetUnusedReferences)
 	{
-		size_t	   TotalCount = m_Cids.size();
-		size_t	   PruneCount = 0;
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
-			ZEN_DEBUG("gc block store '{}': removed {} unused cid out of {} in {}",
-					  m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
-					  PruneCount,
-					  TotalCount,
-					  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+					 Stats.Count,
+					 Stats.Pruned,
+					 Stats.Compacted,
+					 NiceBytes(Stats.RemovedDisk),
+					 NiceBytes(Stats.RemovedMemory),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
-		m_Cids.clear();
 
 		if (UnusedCids.empty())
 		{
@@ -725,8 +737,7 @@ public:
 			}
 		}
 
-		PruneCount = UnusedKeys.size();
-		Ctx.PrunedReferences.fetch_add(PruneCount);
+		Stats.Pruned += UnusedKeys.size();
 		return new CasContainerStoreCompactor(m_CasContainerStrategy,
 											  std::move(CompactState),
 											  std::move(CompactStateKeys),
@@ -738,33 +749,45 @@ private:
 	std::vector<IoHash>	  m_Cids;
 };
 
+std::string
+CasContainerStrategy::GetGcName(GcCtx&)
+{
+	return fmt::format("compactcas:'{}'", (m_RootDirectory / m_ContainerBaseName).string());
+}
+
 GcReferencePruner*
-CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx)
+CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
 {
-	size_t	   TotalCount = 0;
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
-		ZEN_DEBUG("gc block store '{}': found {} cid keys to check in {}",
-				  m_RootDirectory / m_ContainerBaseName,
-				  TotalCount,
-				  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO("GCV2: compactcas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+				 m_RootDirectory / m_ContainerBaseName,
+				 Stats.Count,
+				 Stats.Pruned,
+				 Stats.Compacted,
+				 NiceBytes(Stats.RemovedDisk),
+				 NiceBytes(Stats.RemovedMemory),
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	std::vector<IoHash> CidsToCheck;
 	{
 		RwLock::SharedLockScope __(m_LocationMapLock);
+		if (m_LocationMap.empty())
+		{
+			return {};
+		}
 		CidsToCheck.reserve(m_LocationMap.size());
 		for (const auto& It : m_LocationMap)
 		{
 			CidsToCheck.push_back(It.first);
 		}
 	}
-	TotalCount = CidsToCheck.size();
-	if (TotalCount == 0)
-	{
-		return {};
-	}
-	Ctx.References.fetch_add(TotalCount);
+	Stats.Count += CidsToCheck.size();
 	return new CasContainerReferencePruner(*this, std::move(CidsToCheck));
 }