Add GC Cancel/Stop (#568)

- GcScheduler will now cancel any running GC when it shuts down.
  - Old GC is rather limited in *when* it reacts to cancel of GC. GCv2 is more responsive.

1 files changed, 97 insertions, 6 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 64958cdea..2cd1f6aeb 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -424,6 +424,7 @@ WriteGCResult(CbObjectWriter& Writer, const GcResult& Result, bool HumanReadable
 		}
 		Writer << "WriteBlock" << ToTimeSpan(Result.WriteBlockMS);
 		Writer << "Elapsed" << ToTimeSpan(Result.ElapsedMS);
+		Writer << "Cancelled" << Result.WasCancelled;
 		return;
 	}
 
@@ -538,8 +539,8 @@ Add(GcReferenceStoreStats& Sum, const GcReferenceStoreStats& Sub)
 	Sum.ElapsedMS += Sub.ElapsedMS;
 }
 
-void
-Sum(GcResult& Stat)
+GcResult&
+Sum(GcResult& Stat, bool Cancelled = false)
 {
 	for (std::pair<std::string, GcReferencerStats>& Referencer : Stat.ReferencerStats)
 	{
@@ -559,6 +560,10 @@ Sum(GcResult& Stat)
 
 	Add(Stat.CompactStoresStatSum, Stat.ReferencerStatSum.CompactStoreStats);
 	Add(Stat.CompactStoresStatSum, Stat.ReferenceStoreStatSum.CompactStoreStats);
+
+	Stat.WasCancelled = Cancelled;
+
+	return Stat;
 }
 
 void
@@ -594,7 +599,9 @@ GcManager::RemoveGcReferenceStore(GcReferenceStore& ReferenceStore)
 GcResult
 GcManager::CollectGarbage(const GcSettings& Settings)
 {
-	GcCtx	 Ctx{.Settings = Settings};
+	ZEN_TRACE_CPU("Gc::CollectGarbage(v2)");
+
+	GcCtx	 Ctx{.Settings = Settings, .IsCancelledFlag = m_CancelGC};
 	GcResult Result;
 
 	{
@@ -619,6 +626,11 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 		ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size());
 		if (!m_GcReferencers.empty())
 		{
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
+
 			Latch WorkLeft(1);
 			{
 				// First remove any cache keys that may own references
@@ -629,6 +641,12 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 				});
 				for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
 				{
+					if (CheckGCCancel())
+					{
+						WorkLeft.CountDown();
+						WorkLeft.Wait();
+						return Sum(Result, true);
+					}
 					GcReferencer*							   Owner = m_GcReferencers[Index];
 					std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
 					WorkLeft.AddCount(1);
@@ -652,6 +670,11 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 
 		if (!Ctx.Settings.SkipCidDelete)
 		{
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
+
 			Result.ReferenceStoreStats.resize(m_GcReferenceStores.size());
 
 			ZEN_INFO("GCV2: Creating reference pruners from {} reference stores", m_GcReferenceStores.size());
@@ -672,6 +695,13 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 								 });
 					for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++)
 					{
+						if (CheckGCCancel())
+						{
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
+							return Sum(Result, true);
+						}
+
 						GcReferenceStore*							   ReferenceStore = m_GcReferenceStores[Index];
 						std::pair<std::string, GcReferenceStoreStats>& Stats		  = Result.ReferenceStoreStats[Index];
 						WorkLeft.AddCount(1);
@@ -701,6 +731,11 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 
 			if (!ReferencePruners.empty())
 			{
+				if (CheckGCCancel())
+				{
+					return Sum(Result, true);
+				}
+
 				ZEN_INFO("GCV2: Creating reference checkers from {} referencers", m_GcReferencers.size());
 				std::unordered_map<std::unique_ptr<GcReferenceChecker>, size_t> ReferenceCheckers;
 				if (!m_GcReferencers.empty())
@@ -719,6 +754,13 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						// Lock all reference owners from changing the reference data and get access to check for referenced data
 						for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
 						{
+							if (CheckGCCancel())
+							{
+								WorkLeft.CountDown();
+								WorkLeft.Wait();
+								return Sum(Result, true);
+							}
+
 							GcReferencer*							   Referencer = m_GcReferencers[Index];
 							std::pair<std::string, GcReferencerStats>& Stats	  = Result.ReferencerStats[Index];
 							WorkLeft.AddCount(1);
@@ -767,6 +809,11 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 								 ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
 					if (!ReferenceCheckers.empty())
 					{
+						if (CheckGCCancel())
+						{
+							return Sum(Result, true);
+						}
+
 						// Locking all references checkers so we have a steady state of which references are used
 						// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
 						// we delete the ReferenceCheckers
@@ -781,6 +828,13 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										 });
 							for (auto& It : ReferenceCheckers)
 							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
 								GcReferenceChecker*						   Checker = It.first.get();
 								size_t									   Index   = It.second;
 								std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
@@ -827,6 +881,13 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										 });
 							for (auto& It : ReferencePruners)
 							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
 								GcReferencePruner*	   Pruner = It.second.get();
 								size_t				   Index  = It.first;
 								GcReferenceStoreStats& Stats  = Result.ReferenceStoreStats[Index].second;
@@ -866,6 +927,11 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 		ZEN_INFO("GCV2: Compacting using {} store compactors", StoreCompactors.size());
 		if (!StoreCompactors.empty())
 		{
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
+
 			auto ClaimDiskReserve = [&]() -> uint64_t {
 				if (!std::filesystem::is_regular_file(Settings.DiskReservePath))
 				{
@@ -886,6 +952,11 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 				});
 				for (auto& It : StoreCompactors)
 				{
+					if (CheckGCCancel())
+					{
+						return Sum(Result, true);
+					}
+
 					GcStoreCompactor*	 Compactor = It.first.get();
 					GcCompactStoreStats& Stats	   = *It.second;
 					{
@@ -901,8 +972,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 		ZEN_INFO("GCV2: Completed in {}", NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
 	}
 
-	Sum(Result);
-	return Result;
+	return Sum(Result);
 }
 
 #undef SCOPED_TIMER
@@ -910,6 +980,12 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 //////// End GC V2
 
 void
+GcManager::SetCancelGC(bool CancelFlag)
+{
+	m_CancelGC.store(CancelFlag);
+}
+
+void
 GcManager::AddGcContributor(GcContributor* Contributor)
 {
 	RwLock::ExclusiveLockScope _(m_Lock);
@@ -965,6 +1041,10 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 		const auto Guard = MakeGuard([&] { ZEN_INFO("gathered references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
 		for (GcContributor* Contributor : m_GcContribs)
 		{
+			if (CheckGCCancel())
+			{
+				return GCTotalSizeDiff;
+			}
 			Contributor->GatherReferences(GcCtx);
 		}
 	}
@@ -982,6 +1062,11 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 		});
 		for (GcStorage* Storage : m_GcStorage)
 		{
+			if (CheckGCCancel())
+			{
+				break;
+			}
+
 			const auto PreSize = Storage->StorageSize();
 			Storage->CollectGarbage(GcCtx);
 			const auto PostSize = Storage->StorageSize();
@@ -1208,7 +1293,11 @@ GcScheduler::Shutdown()
 	if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)
 	{
 		bool GcIsRunning = m_Status == static_cast<uint32_t>(GcSchedulerStatus::kRunning);
-		m_Status		 = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
+		if (GcIsRunning)
+		{
+			m_GcManager.SetCancelGC(true);
+		}
+		m_Status = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
 		m_GcSignal.notify_one();
 
 		if (m_GcThread.joinable())
@@ -1758,6 +1847,8 @@ GcScheduler::SchedulerThread()
 						   CompactBlockUsageThresholdPercent,
 						   Verbose);
 
+			m_GcManager.SetCancelGC(false);
+
 			uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
 			if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
 			{