Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

1 files changed, 24 insertions, 2 deletions

diff --git a/src/zenhttp/httpasio.cpp b/src/zenhttp/httpasio.cpp
index affe328e3..f29b3132e 100644
--- a/src/zenhttp/httpasio.cpp
+++ b/src/zenhttp/httpasio.cpp
@@ -2,6 +2,7 @@
 
 #include "httpasio.h"
 
+#include <zencore/except.h>
 #include <zencore/logging.h>
 #include <zencore/trace.h>
 #include <zenhttp/httpserver.h>
@@ -544,13 +545,34 @@ HttpServerConnection::HandleRequest()
 			{
 				Service->HandleRequest(Request);
 			}
-			catch (std::exception& ex)
+			catch (std::system_error& SystemError)
 			{
-				ZEN_ERROR("Caught exception while handling request: {}", ex.what());
+				// Drop any partially formatted response
+				Request.m_Response.reset();
 
+				if (IsOOM(SystemError.code()) || IsOOD(SystemError.code()))
+				{
+					Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, SystemError.what());
+				}
+				else
+				{
+					ZEN_ERROR("Caught system error exception while handling request: {}", SystemError.what());
+					Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what());
+				}
+			}
+			catch (std::bad_alloc& BadAlloc)
+			{
+				// Drop any partially formatted response
+				Request.m_Response.reset();
+
+				Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what());
+			}
+			catch (std::exception& ex)
+			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
 
+				ZEN_ERROR("Caught exception while handling request: {}", ex.what());
 				Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what());
 			}
 		}