safer gcv2 on error (#60)

- Bugfix: Harden GCv2 when errors occur and gracefully abort GC operation on error

6 files changed, 41 insertions, 6 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 27e8b2463..874715818 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3683,7 +3683,7 @@ public:
 	{
 	}
 
-	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&)
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&) override
 	{
 		ZEN_TRACE_CPU("Store::CompactStore");
 
@@ -3747,6 +3747,8 @@ public:
 		m_OplogPathsToRemove.clear();
 	}
 
+	virtual std::string GetGcName(GcCtx&) override { return fmt::format("projectstore: '{}'", m_BasePath.string()); }
+
 private:
 	std::filesystem::path			   m_BasePath;
 	std::vector<std::filesystem::path> m_OplogPathsToRemove;
@@ -3902,10 +3904,17 @@ public:
 
 	virtual ~ProjectStoreReferenceChecker()
 	{
-		m_OplogLock.reset();
-		if (m_OplogCaptureEnabled)
+		try
+		{
+			m_OplogLock.reset();
+			if (m_OplogCaptureEnabled)
+			{
+				m_Oplog.DisableUpdateCapture();
+			}
+		}
+		catch (const std::exception& Ex)
 		{
-			m_Oplog.DisableUpdateCapture();
+			ZEN_ERROR("~ProjectStoreReferenceChecker threw exception: '{}'", Ex.what());
 		}
 	}
 
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index f53ab6f8b..4911ff4f8 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -2886,6 +2886,8 @@ public:
 		}
 	}
 
+	virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); }
+
 private:
 	ZenCacheDiskLayer::CacheBucket&			 m_Bucket;
 	std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys;
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index ec2bfbdec..f3c67eb9a 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -745,6 +745,8 @@ public:
 		}
 	}
 
+	virtual std::string GetGcName(GcCtx& Ctx) override { return m_CasContainerStrategy.GetGcName(Ctx); }
+
 	CasContainerStrategy& m_CasContainerStrategy;
 };
 
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 0f3e2ab5a..7fbc2b9ad 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1437,7 +1437,7 @@ public:
 		m_ReferencesToClean.shrink_to_fit();
 	}
 
-	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&)
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&) override
 	{
 		ZEN_TRACE_CPU("FileCas::CompactStore");
 
@@ -1539,6 +1539,8 @@ public:
 		Reset(m_ReferencesToClean);
 	}
 
+	virtual std::string GetGcName(GcCtx& Ctx) override { return m_FileCasStrategy.GetGcName(Ctx); }
+
 private:
 	FileCasStrategy&	m_FileCasStrategy;
 	std::vector<IoHash> m_ReferencesToClean;
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index d51144a5a..39a747dae 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -666,6 +666,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						catch (const std::exception& Ex)
 						{
 							ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+							SetCancelGC(true);
 						}
 					});
 				}
@@ -738,6 +739,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
 											  ReferenceStore->GetGcName(Ctx),
 											  Ex.what());
+									SetCancelGC(true);
 								}
 							});
 					}
@@ -811,6 +813,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
 												  Referencer->GetGcName(Ctx),
 												  Ex.what());
+										SetCancelGC(true);
 										while (!Checkers.empty())
 										{
 											delete Checkers.back();
@@ -866,6 +869,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+										SetCancelGC(true);
 									}
 								});
 							}
@@ -922,6 +926,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed locking state for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+										SetCancelGC(true);
 									}
 								});
 							}
@@ -932,6 +937,10 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 				}
 				{
 					ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
+					if (CheckGCCancel())
+					{
+						return Sum(Result, true);
+					}
 					{
 						const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
 							HashSet UnusedCids(References.begin(), References.end());
@@ -999,7 +1008,10 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										}
 										catch (const std::exception& Ex)
 										{
-											ZEN_ERROR("GCV2: Failed locking state for {}. Reason: '{}'", Pruner->GetGcName(Ctx), Ex.what());
+											ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													  Pruner->GetGcName(Ctx),
+													  Ex.what());
+											SetCancelGC(true);
 										}
 									});
 							}
@@ -1051,11 +1063,17 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 
 					GcStoreCompactor*	 Compactor = It.first.get();
 					GcCompactStoreStats& Stats	   = *It.second;
+					try
 					{
 						// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve);
 					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 				}
 			}
 			StoreCompactors.clear();
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index b7613883a..5262c6d2e 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -157,6 +157,8 @@ public:
 
 	// Remove data on disk based on results from GcReferencePruner::RemoveUnreferencedData
 	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) = 0;
+
+	virtual std::string GetGcName(GcCtx& Ctx) = 0;
 };
 
 /**