individual gc stats (#506)

- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2
- Feature: New options for zen command `gc-status`
  - `--details` that enables the detailed output from the last GC operation when using GC V2

1 files changed, 69 insertions, 21 deletions

diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index fa7dce331..e2e99d5a4 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -61,31 +61,67 @@ struct GcSettings
 	bool			   CollectSmallObjects	  = false;
 	bool			   IsDeleteMode			  = false;
 	bool			   SkipCidDelete		  = false;
+	bool			   Verbose				  = false;
+};
+
+struct GcReferencerStats
+{
+	std::uint64_t Count			= 0;
+	std::uint64_t Expired		= 0;
+	std::uint64_t Deleted		= 0;
+	std::uint64_t RemovedDisk	= 0;
+	std::uint64_t RemovedMemory = 0;
+
+	std::chrono::milliseconds RemoveExpiredDataMS		= {};
+	std::chrono::milliseconds CreateReferenceCheckersMS = {};
+	std::chrono::milliseconds LockStateMS				= {};
+	std::chrono::milliseconds ElapsedMS					= {};
+};
+
+struct GcReferenceStoreStats
+{
+	std::uint64_t Count			= 0;
+	std::uint64_t Pruned		= 0;
+	std::uint64_t Compacted		= 0;
+	std::uint64_t RemovedDisk	= 0;
+	std::uint64_t RemovedMemory = 0;
+
+	std::chrono::milliseconds CreateReferencePrunerMS  = {};
+	std::chrono::milliseconds RemoveUnreferencedDataMS = {};
+	std::chrono::milliseconds CompactReferenceStoreMS  = {};
+	std::chrono::milliseconds ElapsedMS				   = {};
 };
 
 struct GcResult
 {
-	uint64_t Items				 = 0;
-	uint64_t ExpiredItems		 = 0;
-	uint64_t DeletedItems		 = 0;
-	uint64_t References			 = 0;
-	uint64_t PrunedReferences	 = 0;
-	uint64_t CompactedReferences = 0;
-	uint64_t RemovedDiskSpace	 = 0;
-	uint64_t RemovedMemory		 = 0;
+	GcReferencerStats	  ReferencerStat;
+	GcReferenceStoreStats ReferenceStoreStat;
+
+	std::uint64_t RemovedDisk	= 0;
+	std::uint64_t RemovedMemory = 0;
+
+	std::vector<std::pair<std::string, GcReferencerStats>>	   ReferencerStats;
+	std::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats;
+
+	// Wall times, not sum of each
+	std::chrono::milliseconds RemoveExpiredDataMS		= {};
+	std::chrono::milliseconds CreateReferenceCheckersMS = {};
+	std::chrono::milliseconds LockStateMS				= {};
+
+	std::chrono::milliseconds CreateReferencePrunerMS  = {};
+	std::chrono::milliseconds RemoveUnreferencedDataMS = {};
+	std::chrono::milliseconds CompactReferenceStoreMS  = {};
+
+	std::chrono::milliseconds WriteBlockMS = {};
+
+	std::chrono::milliseconds ElapsedMS = {};
+
+	void Sum();
 };
 
 struct GcCtx
 {
-	const GcSettings	 Settings;
-	std::atomic_uint64_t Items				 = 0;
-	std::atomic_uint64_t ExpiredItems		 = 0;
-	std::atomic_uint64_t DeletedItems		 = 0;
-	std::atomic_uint64_t References			 = 0;
-	std::atomic_uint64_t PrunedReferences	 = 0;
-	std::atomic_uint64_t CompactedReferences = 0;
-	std::atomic_uint64_t RemovedDiskSpace	 = 0;
-	std::atomic_uint64_t RemovedMemory		 = 0;
+	const GcSettings Settings;
 };
 
 typedef tsl::robin_set<IoHash> HashSet;
@@ -106,7 +142,7 @@ public:
 	virtual ~GcReferenceStoreCompactor() = default;
 
 	// Remove data on disk based on results from GcReferencePruner::RemoveUnreferencedData
-	virtual void CompactReferenceStore(GcCtx& Ctx) = 0;
+	virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats) = 0;
 };
 
 /**
@@ -149,11 +185,13 @@ protected:
 	virtual ~GcReferencer() = default;
 
 public:
+	virtual std::string GetGcName(GcCtx& Ctx) = 0;
+
 	// Remove expired data based on either GcCtx::Settings CacheExpireTime/ProjectExpireTime
 	// TODO: For disk layer we need to first update it with access times from the memory layer
 	// The implementer of GcReferencer (in our case a disk bucket) does not know about any
 	// potential memory cache layer :(
-	virtual void RemoveExpiredData(GcCtx& Ctx) = 0;
+	virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) = 0;
 
 	// Create 0-n GcReferenceChecker for this GcReferencer. Caller will manage lifetime of
 	// returned instances
@@ -178,7 +216,9 @@ public:
 	// Caller will manage lifetime of returned instance
 	// This function should execute as fast as possible, so try to prepare a list of references to check ahead of
 	// call to this function and make sure the removal of unreferences items is as lightweight as possible.
-	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
+	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx&						 Ctx,
+															  GcReferenceStoreStats&		 Stats,
+															  const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
 };
 
 /**
@@ -190,9 +230,11 @@ protected:
 	virtual ~GcReferenceStore() = default;
 
 public:
+	virtual std::string GetGcName(GcCtx& Ctx) = 0;
+
 	// Create a GcReferencePruner which can check a set of references (decided by implementor) if they are no longer in use
 	// Caller will manage lifetime of returned instance
-	virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx) = 0;
+	virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) = 0;
 };
 
 //////// End New GC WIP
@@ -379,6 +421,9 @@ struct GcSchedulerState
 	GcStorageSize			  LastFullGCDiff;
 	std::chrono::milliseconds LastLightweightGcDuration{};
 	GcStorageSize			  LastLightweightGCDiff;
+
+	std::optional<GcResult> LastLightweightGCV2Result;
+	std::optional<GcResult> LastFullGCV2Result;
 };
 
 class DiskUsageWindow
@@ -461,6 +506,9 @@ private:
 	std::chrono::milliseconds m_LastLightweightGcDuration{};
 	GcStorageSize			  m_LastLightweightGCDiff;
 
+	std::optional<GcResult> m_LastLightweightGCV2Result;
+	std::optional<GcResult> m_LastFullGCV2Result;
+
 	std::atomic_uint32_t			  m_Status{};
 	std::thread						  m_GcThread;
 	mutable std::mutex				  m_GcMutex;