safer gcv2 on error (#60)

- Bugfix: Harden GCv2 when errors occur and gracefully abort GC operation on error

1 files changed, 19 insertions, 1 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index d51144a5a..39a747dae 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -666,6 +666,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						catch (const std::exception& Ex)
 						{
 							ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+							SetCancelGC(true);
 						}
 					});
 				}
@@ -738,6 +739,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
 											  ReferenceStore->GetGcName(Ctx),
 											  Ex.what());
+									SetCancelGC(true);
 								}
 							});
 					}
@@ -811,6 +813,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
 												  Referencer->GetGcName(Ctx),
 												  Ex.what());
+										SetCancelGC(true);
 										while (!Checkers.empty())
 										{
 											delete Checkers.back();
@@ -866,6 +869,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+										SetCancelGC(true);
 									}
 								});
 							}
@@ -922,6 +926,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed locking state for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+										SetCancelGC(true);
 									}
 								});
 							}
@@ -932,6 +937,10 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 				}
 				{
 					ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
+					if (CheckGCCancel())
+					{
+						return Sum(Result, true);
+					}
 					{
 						const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
 							HashSet UnusedCids(References.begin(), References.end());
@@ -999,7 +1008,10 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										}
 										catch (const std::exception& Ex)
 										{
-											ZEN_ERROR("GCV2: Failed locking state for {}. Reason: '{}'", Pruner->GetGcName(Ctx), Ex.what());
+											ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													  Pruner->GetGcName(Ctx),
+													  Ex.what());
+											SetCancelGC(true);
 										}
 									});
 							}
@@ -1051,11 +1063,17 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 
 					GcStoreCompactor*	 Compactor = It.first.get();
 					GcCompactStoreStats& Stats	   = *It.second;
+					try
 					{
 						// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve);
 					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 				}
 			}
 			StoreCompactors.clear();