Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

1 files changed, 34 insertions, 8 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 520227474..b5ed17fc6 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -2,6 +2,7 @@
 
 #include <zenstore/blockstore.h>
 
+#include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
@@ -362,7 +363,11 @@ BlockStore::Flush()
 	{
 		uint32_t WriteBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
 		WriteBlockIndex			 = (WriteBlockIndex + 1) & (m_MaxBlockCount - 1);
-		m_WriteBlock			 = nullptr;
+		if (m_WriteBlock)
+		{
+			m_WriteBlock->Flush();
+		}
+		m_WriteBlock = nullptr;
 		m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release);
 		m_CurrentInsertOffset = 0;
 	}
@@ -502,10 +507,18 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 		return;
 	}
 
-	Ref<BlockStoreFile> NewBlockFile;
 	try
 	{
 		ZEN_TRACE_CPU("BlockStore::ReclaimSpace::Compact");
+		Ref<BlockStoreFile> NewBlockFile;
+		auto				NewBlockFileGuard = MakeGuard([&]() {
+			   if (NewBlockFile)
+			   {
+				   ZEN_DEBUG("dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
+				   m_TotalSize.fetch_sub(NewBlockFile->FileSize(), std::memory_order::relaxed);
+				   NewBlockFile->MarkAsDeleteOnClose();
+			   }
+		   });
 
 		uint64_t WriteOffset   = 0;
 		uint32_t NewBlockIndex = 0;
@@ -703,15 +716,28 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 			NewBlockFile = nullptr;
 		}
 	}
-	catch (std::exception& ex)
+	catch (std::system_error& SystemError)
 	{
-		ZEN_ERROR("reclaiming space for '{}' failed with: '{}'", m_BlocksBasePath, ex.what());
-		if (NewBlockFile)
+		if (IsOOM(SystemError.code()))
+		{
+			ZEN_WARN("reclaiming space for '{}' ran out of memory: '{}'", m_BlocksBasePath, SystemError.what());
+		}
+		else if (IsOOD(SystemError.code()))
 		{
-			ZEN_DEBUG("dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
-			m_TotalSize.fetch_sub(NewBlockFile->FileSize(), std::memory_order::relaxed);
-			NewBlockFile->MarkAsDeleteOnClose();
+			ZEN_WARN("reclaiming space for '{}' ran out of disk space: '{}'", m_BlocksBasePath, SystemError.what());
 		}
+		else
+		{
+			ZEN_ERROR("reclaiming space for '{}' failed with system error exception: '{}'", m_BlocksBasePath, SystemError.what());
+		}
+	}
+	catch (std::bad_alloc& BadAlloc)
+	{
+		ZEN_WARN("reclaiming space for '{}' ran out of memory: '{}'", m_BlocksBasePath, BadAlloc.what());
+	}
+	catch (std::exception& ex)
+	{
+		ZEN_ERROR("reclaiming space for '{}' failed with: '{}'", m_BlocksBasePath, ex.what());
 	}
 }