Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

3 files changed, 74 insertions, 13 deletions

diff --git a/src/zencore/except.cpp b/src/zencore/except.cpp
index 65f5ebc62..f98743ea9 100644
--- a/src/zencore/except.cpp
+++ b/src/zencore/except.cpp
@@ -110,4 +110,62 @@ ThrowOutOfMemory(std::string_view Message)
 }
 #endif
 
+#if ZEN_PLATFORM_WINDOWS
+bool
+IsOOM(const std::system_error& SystemError)
+{
+	switch (SystemError.code().value())
+	{
+		case ERROR_NOT_ENOUGH_MEMORY:
+		case ERROR_OUTOFMEMORY:
+		case ERROR_PAGEFILE_QUOTA_EXCEEDED:
+		case ERROR_NONPAGED_SYSTEM_RESOURCES:
+		case ERROR_PAGED_SYSTEM_RESOURCES:
+		case ERROR_PAGEFILE_QUOTA:
+		case ERROR_COMMITMENT_LIMIT:
+			return true;
+		default:
+			return false;
+	}
+}
+bool
+IsOOD(const std::system_error& SystemError)
+{
+	switch (SystemError.code().value())
+	{
+		case ERROR_HANDLE_DISK_FULL:
+		case ERROR_DISK_FULL:
+		case ERROR_DISK_RESOURCES_EXHAUSTED:
+		case ERROR_DISK_QUOTA_EXCEEDED:
+			return true;
+		default:
+			return false;
+	}
+}
+#else
+bool
+IsOOM(const std::system_error& SystemError)
+{
+	switch (SystemError.code().value())
+	{
+		case ENOMEM:
+			return true;
+		default:
+			return false;
+	}
+}
+bool
+IsOOD(const std::system_error& SystemError)
+{
+	switch (SystemError.code().value())
+	{
+		case ENOSPC:
+		case EDQUOT:
+			return true;
+		default:
+			return false;
+	}
+}
+#endif
+
 }  // namespace zen
diff --git a/src/zencore/include/zencore/except.h b/src/zencore/include/zencore/except.h
index 464852f88..6810e6ea9 100644
--- a/src/zencore/include/zencore/except.h
+++ b/src/zencore/include/zencore/except.h
@@ -66,4 +66,7 @@ public:
 	inline explicit OptionParseException(const std::string& Message) : std::runtime_error(Message) {}
 };
 
+bool IsOOM(const std::system_error& SystemError);
+bool IsOOD(const std::system_error& SystemError);
+
 }  // namespace zen
diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index 09cd0a000..74fec4c51 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -339,12 +339,12 @@ IoBufferExtendedCore::Materialize() const
 		if (Error || (BytesRead != m_DataBytes))
 		{
 			std::error_code DummyEc;
-			ZEN_ERROR("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}",
-					  m_FileOffset,
-					  m_DataBytes,
-					  zen::PathFromHandle(m_FileHandle, DummyEc),
-					  zen::FileSizeFromHandle(m_FileHandle),
-					  GetSystemErrorAsString(Error));
+			ZEN_WARN("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}",
+					 m_FileOffset,
+					 m_DataBytes,
+					 zen::PathFromHandle(m_FileHandle, DummyEc),
+					 zen::FileSizeFromHandle(m_FileHandle),
+					 GetSystemErrorAsString(Error));
 			throw std::system_error(std::error_code(Error, std::system_category()),
 									fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
 												m_FileOffset,
@@ -379,7 +379,7 @@ IoBufferExtendedCore::Materialize() const
 	{
 		int32_t			Error = zen::GetLastError();
 		std::error_code DummyEc;
-		ZEN_ERROR("CreateFileMapping failed on file '{}', {}", zen::PathFromHandle(m_FileHandle, DummyEc), GetSystemErrorAsString(Error));
+		ZEN_WARN("CreateFileMapping failed on file '{}', {}", zen::PathFromHandle(m_FileHandle, DummyEc), GetSystemErrorAsString(Error));
 		throw std::system_error(std::error_code(Error, std::system_category()),
 								fmt::format("CreateFileMapping failed on file '{}'", zen::PathFromHandle(m_FileHandle, DummyEc)));
 	}
@@ -412,12 +412,12 @@ IoBufferExtendedCore::Materialize() const
 #endif	// ZEN_PLATFORM_WINDOWS
 
 		std::error_code DummyEc;
-		ZEN_ERROR("MapViewOfFile/mmap failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}",
-				  MapOffset,
-				  MapSize,
-				  zen::PathFromHandle(m_FileHandle, DummyEc),
-				  zen::FileSizeFromHandle(m_FileHandle),
-				  GetSystemErrorAsString(Error));
+		ZEN_WARN("MapViewOfFile/mmap failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}",
+				 MapOffset,
+				 MapSize,
+				 zen::PathFromHandle(m_FileHandle, DummyEc),
+				 zen::FileSizeFromHandle(m_FileHandle),
+				 GetSystemErrorAsString(Error));
 		throw std::system_error(std::error_code(Error, std::system_category()),
 								fmt::format("MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
 											MapOffset,