Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

1 files changed, 23 insertions, 8 deletions

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index a138e43e9..1d1797597 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -565,15 +565,21 @@ CasContainerStrategy::MakeIndexSnapshot()
 	// Move index away, we keep it if something goes wrong
 	if (fs::is_regular_file(TempIndexPath))
 	{
-		fs::remove(TempIndexPath);
-	}
-	if (fs::is_regular_file(IndexPath))
-	{
-		fs::rename(IndexPath, TempIndexPath);
+		std::error_code Ec;
+		if (!fs::remove(TempIndexPath, Ec) || Ec)
+		{
+			ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message());
+			return;
+		}
 	}
 
 	try
 	{
+		if (fs::is_regular_file(IndexPath))
+		{
+			fs::rename(IndexPath, TempIndexPath);
+		}
+
 		// Write the current state of the location map to a new index state
 		std::vector<CasDiskIndexEntry> Entries;
 
@@ -613,13 +619,22 @@ CasContainerStrategy::MakeIndexSnapshot()
 
 		if (fs::is_regular_file(TempIndexPath))
 		{
-			fs::remove(IndexPath);
-			fs::rename(TempIndexPath, IndexPath);
+			std::error_code Ec;
+			fs::remove(IndexPath, Ec);	// We don't care if this fails, we try to move the old temp file regardless
+			fs::rename(TempIndexPath, IndexPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message());
+			}
 		}
 	}
 	if (fs::is_regular_file(TempIndexPath))
 	{
-		fs::remove(TempIndexPath);
+		std::error_code Ec;
+		if (!fs::remove(TempIndexPath, Ec) || Ec)
+		{
+			ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message());
+		}
 	}
 }