Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

2 files changed, 39 insertions, 4 deletions

diff --git a/src/zenhttp/httpasio.cpp b/src/zenhttp/httpasio.cpp
index affe328e3..f29b3132e 100644
--- a/src/zenhttp/httpasio.cpp
+++ b/src/zenhttp/httpasio.cpp
@@ -2,6 +2,7 @@
 
 #include "httpasio.h"
 
+#include <zencore/except.h>
 #include <zencore/logging.h>
 #include <zencore/trace.h>
 #include <zenhttp/httpserver.h>
@@ -544,13 +545,34 @@ HttpServerConnection::HandleRequest()
 			{
 				Service->HandleRequest(Request);
 			}
-			catch (std::exception& ex)
+			catch (std::system_error& SystemError)
 			{
-				ZEN_ERROR("Caught exception while handling request: {}", ex.what());
+				// Drop any partially formatted response
+				Request.m_Response.reset();
 
+				if (IsOOM(SystemError.code()) || IsOOD(SystemError.code()))
+				{
+					Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, SystemError.what());
+				}
+				else
+				{
+					ZEN_ERROR("Caught system error exception while handling request: {}", SystemError.what());
+					Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what());
+				}
+			}
+			catch (std::bad_alloc& BadAlloc)
+			{
+				// Drop any partially formatted response
+				Request.m_Response.reset();
+
+				Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what());
+			}
+			catch (std::exception& ex)
+			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
 
+				ZEN_ERROR("Caught exception while handling request: {}", ex.what());
 				Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what());
 			}
 		}
diff --git a/src/zenhttp/httpsys.cpp b/src/zenhttp/httpsys.cpp
index 8c1f68ee8..60358d0b0 100644
--- a/src/zenhttp/httpsys.cpp
+++ b/src/zenhttp/httpsys.cpp
@@ -1812,11 +1812,24 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT
 		// Unable to route
 		return new HttpMessageResponseRequest(Transaction(), 404, "No suitable route found"sv);
 	}
+	catch (std::system_error& SystemError)
+	{
+		if (IsOOM(SystemError.code()) || IsOOD(SystemError.code()))
+		{
+			return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InsufficientStorage, SystemError.what());
+		}
+
+		ZEN_ERROR("Caught system error exception while handling request: {}", SystemError.what());
+		return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InternalServerError, SystemError.what());
+	}
+	catch (std::bad_alloc& BadAlloc)
+	{
+		return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InsufficientStorage, BadAlloc.what());
+	}
 	catch (std::exception& ex)
 	{
 		ZEN_ERROR("Caught exception while handling request: '{}'", ex.what());
-
-		return new HttpMessageResponseRequest(Transaction(), 500, ex.what());
+		return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InternalServerError, ex.what());
 	}
 }