Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

1 files changed, 39 insertions, 1 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index f8ef5de82..b63be04bb 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1032,6 +1032,25 @@ GcScheduler::SchedulerThread()
 
 			WaitTime = std::chrono::seconds(0);
 		}
+		catch (std::system_error& SystemError)
+		{
+			if (IsOOM(SystemError.code()))
+			{
+				ZEN_WARN("scheduling garbage collection ran out of memory: '{}'", SystemError.what());
+			}
+			else if (IsOOD(SystemError.code()))
+			{
+				ZEN_WARN("scheduling garbage collection ran out of disk space: '{}'", SystemError.what());
+			}
+			else
+			{
+				ZEN_ERROR("scheduling garbage collection failed with system error exception: '{}'", SystemError.what());
+			}
+		}
+		catch (std::bad_alloc& BadAlloc)
+		{
+			ZEN_WARN("scheduling garbage collection ran out of memory: '{}'", BadAlloc.what());
+		}
 		catch (std::exception& Ex)
 		{
 			ZEN_ERROR("scheduling garbage collection failed with: '{}'", Ex.what());
@@ -1151,9 +1170,28 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 			SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
 			SaveCompactBinaryObject(Path, SchedulerState.Save());
 		}
+		catch (std::system_error& SystemError)
+		{
+			if (IsOOM(SystemError.code()))
+			{
+				ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", SystemError.what());
+			}
+			else if (IsOOD(SystemError.code()))
+			{
+				ZEN_WARN("writing gc scheduler state ran out of disk space: '{}'", SystemError.what());
+			}
+			else
+			{
+				ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}'", SystemError.what());
+			}
+		}
+		catch (std::bad_alloc& BadAlloc)
+		{
+			ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what());
+		}
 		catch (std::exception& Ex)
 		{
-			ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what());
+			ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what());
 		}
 	}
 }