individual gc stats (#506)

- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2
- Feature: New options for zen command `gc-status`
  - `--details` that enables the detailed output from the last GC operation when using GC V2

2 files changed, 64 insertions, 30 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 274876123..7da2f7695 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3036,8 +3036,14 @@ ProjectStore::AreDiskWritesAllowed() const
 	return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed());
 }
 
+std::string
+ProjectStore::GetGcName(GcCtx&)
+{
+	return fmt::format("projectstore:'{}'", m_ProjectBasePath.string());
+}
+
 void
-ProjectStore::RemoveExpiredData(GcCtx& Ctx)
+ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
 {
 	size_t	   ProjectCount		   = 0;
 	size_t	   ExpiredProjectCount = 0;
@@ -3045,13 +3051,18 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
 	size_t	   ExpiredOplogCount   = 0;
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
-		ZEN_DEBUG("gc project store '{}': removed {} expired projects out of {}, {} expired oplogs out of {} in {}",
-				  m_ProjectBasePath,
-				  ExpiredProjectCount,
-				  ProjectCount,
-				  ExpiredOplogCount,
-				  OplogCount,
-				  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+				 m_ProjectBasePath,
+				 Stats.Count,
+				 Stats.Expired,
+				 Stats.Deleted,
+				 NiceBytes(Stats.RemovedDisk),
+				 NiceBytes(Stats.RemovedMemory),
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	std::vector<Ref<Project>> ExpiredProjects;
@@ -3097,14 +3108,14 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
 						  OplogId,
 						  Project->Identifier);
 				Project->DeleteOplog(OplogId);
-				Ctx.RemovedDiskSpace.fetch_add(OplogSize);
+				Stats.RemovedDisk += OplogSize;
 			}
-			Ctx.DeletedItems.fetch_add(ExpiredOplogs.size());
+			Stats.Deleted += ExpiredOplogs.size();
 			Project->Flush();
 		}
 	}
 	ProjectCount = Projects.size();
-	Ctx.Items.fetch_add(ProjectCount + OplogCount);
+	Stats.Count += ProjectCount + OplogCount;
 	ExpiredProjectCount = ExpiredProjects.size();
 
 	if (ExpiredProjects.empty())
@@ -3150,20 +3161,34 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx)
 
 			DeleteDirectories(PathToRemove);
 		}
-		Ctx.DeletedItems.fetch_add(ExpiredProjects.size());
+		Stats.Deleted += ExpiredProjects.size();
 	}
 
-	Ctx.ExpiredItems.fetch_add(ExpiredOplogCount + ExpiredProjectCount);
+	Stats.Expired += ExpiredOplogCount + ExpiredProjectCount;
 }
 
 class ProjectStoreReferenceChecker : public GcReferenceChecker
 {
 public:
-	ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner)
+	ProjectStoreReferenceChecker(GcCtx& Ctx, ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner)
 	{
 		if (PreCache)
 		{
-			RwLock::SharedLockScope _(m_Oplog.m_OplogLock);
+			Stopwatch  Timer;
+			const auto _ = MakeGuard([&] {
+				if (!Ctx.Settings.Verbose)
+				{
+					return;
+				}
+				ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': precached {} references in {} from {}/{}",
+						 m_Oplog.m_BasePath,
+						 m_UncachedReferences.size(),
+						 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+						 m_Oplog.m_OuterProject->Identifier,
+						 m_Oplog.OplogId());
+			});
+
+			RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
 			m_Oplog.IterateOplog([&](CbObjectView Op) {
 				Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
 			});
@@ -3173,16 +3198,20 @@ public:
 
 	virtual ~ProjectStoreReferenceChecker() {}
 
-	virtual void LockState(GcCtx&) override
+	virtual void LockState(GcCtx& Ctx) override
 	{
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
-			ZEN_DEBUG("gc project oplog '{}': found {} references in {} from {}/{}",
-					  m_Oplog.m_BasePath,
-					  m_UncachedReferences.size(),
-					  NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
-					  m_Oplog.m_OuterProject->Identifier,
-					  m_Oplog.OplogId());
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
+					 m_Oplog.m_BasePath,
+					 m_UncachedReferences.size(),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+					 m_Oplog.m_OuterProject->Identifier,
+					 m_Oplog.OplogId());
 		});
 
 		m_OplogLock = std::make_unique<RwLock::SharedLockScope>(m_Oplog.m_OplogLock);
@@ -3211,18 +3240,22 @@ public:
 };
 
 std::vector<GcReferenceChecker*>
-ProjectStore::CreateReferenceCheckers(GcCtx&)
+ProjectStore::CreateReferenceCheckers(GcCtx& Ctx)
 {
 	size_t ProjectCount = 0;
 	size_t OplogCount	= 0;
 
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
-		ZEN_DEBUG("gc project store '{}': opened {} projects and {} oplogs in {}",
-				  m_ProjectBasePath,
-				  ProjectCount,
-				  OplogCount,
-				  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO("GCV2: projectstore [CREATE CHECKERS] '{}': opened {} projects and {} oplogs in {}",
+				 m_ProjectBasePath,
+				 ProjectCount,
+				 OplogCount,
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	DiscoverProjects();
@@ -3250,7 +3283,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx&)
 				ProjectStore::Oplog* Oplog		 = Project->OpenOplog(OpLogId);
 				GcClock::TimePoint	 Now		 = GcClock::Now();
 				bool				 TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5));
-				Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache));
+				Checkers.emplace_back(new ProjectStoreReferenceChecker(Ctx, *Oplog, TryPreCache));
 			}
 			OplogCount += OpLogs.size();
 		}
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 94e697278..fbff1444d 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -293,7 +293,8 @@ public:
 	virtual void		  CollectGarbage(GcContext& GcCtx) override;
 	virtual GcStorageSize StorageSize() const override;
 
-	virtual void							 RemoveExpiredData(GcCtx& Ctx) override;
+	virtual std::string						 GetGcName(GcCtx& Ctx) override;
+	virtual void							 RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
 	virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
 
 	CbArray									 GetProjectsList();