Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

1 files changed, 25 insertions, 9 deletions

diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index c3dce2b7b..0d742d7e1 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1031,6 +1031,7 @@ FileCasStrategy::MakeIndexSnapshot()
 	{
 		return;
 	}
+
 	ZEN_DEBUG("write store snapshot for '{}'", m_RootDirectory);
 	uint64_t   EntryCount = 0;
 	Stopwatch  Timer;
@@ -1049,15 +1050,21 @@ FileCasStrategy::MakeIndexSnapshot()
 	// Move index away, we keep it if something goes wrong
 	if (fs::is_regular_file(STmpIndexPath))
 	{
-		fs::remove(STmpIndexPath);
-	}
-	if (fs::is_regular_file(IndexPath))
-	{
-		fs::rename(IndexPath, STmpIndexPath);
+		std::error_code Ec;
+		if (!fs::remove(STmpIndexPath, Ec) || Ec)
+		{
+			ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message());
+			return;
+		}
 	}
 
 	try
 	{
+		if (fs::is_regular_file(IndexPath))
+		{
+			fs::rename(IndexPath, STmpIndexPath);
+		}
+
 		// Write the current state of the location map to a new index state
 		std::vector<FileCasIndexEntry> Entries;
 
@@ -1088,19 +1095,28 @@ FileCasStrategy::MakeIndexSnapshot()
 	}
 	catch (std::exception& Err)
 	{
-		ZEN_ERROR("snapshot FAILED, reason: '{}'", Err.what());
+		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
 
 		// Restore any previous snapshot
 
 		if (fs::is_regular_file(STmpIndexPath))
 		{
-			fs::remove(IndexPath);
-			fs::rename(STmpIndexPath, IndexPath);
+			std::error_code Ec;
+			fs::remove(IndexPath, Ec);	// We don't care if this fails, we try to move the old temp file regardless
+			fs::rename(STmpIndexPath, IndexPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message());
+			}
 		}
 	}
 	if (fs::is_regular_file(STmpIndexPath))
 	{
-		fs::remove(STmpIndexPath);
+		std::error_code Ec;
+		if (!fs::remove(STmpIndexPath, Ec) || Ec)
+		{
+			ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message());
+		}
 	}
 }
 uint64_t