make OOD and OOM in gc non critical (#381)

* oom and ood exceptions in GC are now treated as warnings instead of errors

1 files changed, 218 insertions, 27 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 56d0e8db6..5bd34fc37 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -757,6 +757,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 								StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats);
 							}
 						}
+						catch (const std::system_error& Ex)
+						{
+							if (IsOOD(Ex) || IsOOM(Ex))
+							{
+								ZEN_WARN("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+							}
+							else
+							{
+								ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+							}
+							SetCancelGC(true);
+						}
+						catch (const std::bad_alloc& Ex)
+						{
+							ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+							SetCancelGC(true);
+						}
 						catch (const std::exception& Ex)
 						{
 							ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
@@ -832,6 +849,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
 									}
 								}
+								catch (const std::system_error& Ex)
+								{
+									if (IsOOD(Ex) || IsOOM(Ex))
+									{
+										ZEN_WARN("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+												 ReferenceStore->GetGcName(Ctx),
+												 Ex.what());
+									}
+									else
+									{
+										ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+												  ReferenceStore->GetGcName(Ctx),
+												  Ex.what());
+									}
+									SetCancelGC(true);
+								}
+								catch (const std::bad_alloc& Ex)
+								{
+									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+											  ReferenceStore->GetGcName(Ctx),
+											  Ex.what());
+									SetCancelGC(true);
+								}
 								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
@@ -890,36 +930,65 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										ZEN_MEMSCOPE(GetGcTag());
 
 										auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-										// The Referencer will create a reference checker that guarantees that the references do not change
-										// as long as it lives
-										std::vector<GcReferenceChecker*> Checkers;
-										try
+										if (!CheckGCCancel())
 										{
+											// The Referencer will create a reference checker that guarantees that the references do not
+											// change as long as it lives
+											std::vector<GcReferenceChecker*> Checkers;
+											auto							 __ = MakeGuard([&Checkers]() {
+												while (!Checkers.empty())
+												{
+													delete Checkers.back();
+													Checkers.pop_back();
+												}
+											});
+											try
 											{
-												SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS =
-																 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-												Checkers = Referencer->CreateReferenceCheckers(Ctx);
+												{
+													SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS =
+																	 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+													Checkers = Referencer->CreateReferenceCheckers(Ctx);
+												}
+												if (!Checkers.empty())
+												{
+													RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
+													for (auto& Checker : Checkers)
+													{
+														ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker),
+																						   Index);
+														Checker = nullptr;
+													}
+												}
 											}
-											if (!Checkers.empty())
+											catch (const std::system_error& Ex)
 											{
-												RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
-												for (auto& Checker : Checkers)
+												if (IsOOD(Ex) || IsOOM(Ex))
+												{
+													ZEN_WARN("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+															 Referencer->GetGcName(Ctx),
+															 Ex.what());
+												}
+												else
 												{
-													ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
-													Checker = nullptr;
+													ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+															  Referencer->GetGcName(Ctx),
+															  Ex.what());
 												}
+												SetCancelGC(true);
 											}
-										}
-										catch (const std::exception& Ex)
-										{
-											ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
-													  Referencer->GetGcName(Ctx),
-													  Ex.what());
-											SetCancelGC(true);
-											while (!Checkers.empty())
+											catch (const std::bad_alloc& Ex)
+											{
+												ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+														  Referencer->GetGcName(Ctx),
+														  Ex.what());
+												SetCancelGC(true);
+											}
+											catch (const std::exception& Ex)
 											{
-												delete Checkers.back();
-												Checkers.pop_back();
+												ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+														  Referencer->GetGcName(Ctx),
+														  Ex.what());
+												SetCancelGC(true);
 											}
 										}
 									});
@@ -975,6 +1044,13 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 
 								auto							   _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
 								std::vector<GcReferenceValidator*> Validators;
+								auto							   __ = MakeGuard([&Validators]() {
+									  while (!Validators.empty())
+									  {
+										  delete Validators.back();
+										  Validators.pop_back();
+									  }
+								  });
 								try
 								{
 									{
@@ -995,17 +1071,35 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										}
 									}
 								}
+								catch (const std::system_error& Ex)
+								{
+									if (IsOOD(Ex) || IsOOM(Ex))
+									{
+										ZEN_WARN("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+												 Referencer->GetGcName(Ctx),
+												 Ex.what());
+									}
+									else
+									{
+										ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+												  Referencer->GetGcName(Ctx),
+												  Ex.what());
+									}
+									SetCancelGC(true);
+								}
+								catch (const std::bad_alloc& Ex)
+								{
+									ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+											  Referencer->GetGcName(Ctx),
+											  Ex.what());
+									SetCancelGC(true);
+								}
 								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
 											  Referencer->GetGcName(Ctx),
 											  Ex.what());
 									SetCancelGC(true);
-									while (!Validators.empty())
-									{
-										delete Validators.back();
-										Validators.pop_back();
-									}
 								}
 							});
 						}
@@ -1054,6 +1148,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 									SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 									Checker->PreCache(Ctx);
 								}
+								catch (const std::system_error& Ex)
+								{
+									if (IsOOD(Ex) || IsOOM(Ex))
+									{
+										ZEN_WARN("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+									}
+									else
+									{
+										ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+									}
+									SetCancelGC(true);
+								}
+								catch (const std::bad_alloc& Ex)
+								{
+									ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+									SetCancelGC(true);
+								}
 								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
@@ -1142,6 +1253,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 															 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 											Checker->UpdateLockedState(Ctx);
 										}
+										catch (const std::system_error& Ex)
+										{
+											if (IsOOD(Ex) || IsOOM(Ex))
+											{
+												ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+														 Checker->GetGcName(Ctx),
+														 Ex.what());
+											}
+											else
+											{
+												ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+														  Checker->GetGcName(Ctx),
+														  Ex.what());
+											}
+											SetCancelGC(true);
+										}
+										catch (const std::bad_alloc& Ex)
+										{
+											ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+													 Checker->GetGcName(Ctx),
+													 Ex.what());
+											SetCancelGC(true);
+										}
 										catch (const std::exception& Ex)
 										{
 											ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'",
@@ -1231,6 +1365,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 											StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->CompactStoreStats);
 										}
 									}
+									catch (const std::system_error& Ex)
+									{
+										if (IsOOD(Ex) || IsOOM(Ex))
+										{
+											ZEN_WARN("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													 Pruner->GetGcName(Ctx),
+													 Ex.what());
+										}
+										else
+										{
+											ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													  Pruner->GetGcName(Ctx),
+													  Ex.what());
+										}
+										SetCancelGC(true);
+									}
+									catch (const std::bad_alloc& Ex)
+									{
+										ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+												  Pruner->GetGcName(Ctx),
+												  Ex.what());
+										SetCancelGC(true);
+									}
 									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
@@ -1294,6 +1451,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve);
 					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						}
+						else
+						{
+							ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						}
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 					catch (const std::exception& Ex)
 					{
 						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
@@ -1335,6 +1509,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						ReferenceValidator->Validate(Ctx, Stats);
 					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						}
+						else
+						{
+							ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						}
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 					catch (const std::exception& Ex)
 					{
 						ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());