Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434)

- Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry
- Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry
- Improvement: Make an explicit flush of the active block written to in blockstore flush
- Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit

1 files changed, 25 insertions, 10 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index d53d3f3f4..98a24116f 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -305,15 +305,21 @@ ZenCacheDiskLayer::CacheBucket::MakeIndexSnapshot()
 	// Move index away, we keep it if something goes wrong
 	if (fs::is_regular_file(STmpIndexPath))
 	{
-		fs::remove(STmpIndexPath);
-	}
-	if (fs::is_regular_file(IndexPath))
-	{
-		fs::rename(IndexPath, STmpIndexPath);
+		std::error_code Ec;
+		if (!fs::remove(STmpIndexPath, Ec) || Ec)
+		{
+			ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message());
+			return;
+		}
 	}
 
 	try
 	{
+		if (fs::is_regular_file(IndexPath))
+		{
+			fs::rename(IndexPath, STmpIndexPath);
+		}
+
 		// Write the current state of the location map to a new index state
 		std::vector<DiskIndexEntry> Entries;
 		Entries.resize(m_Index.size());
@@ -351,13 +357,22 @@ ZenCacheDiskLayer::CacheBucket::MakeIndexSnapshot()
 
 		if (fs::is_regular_file(STmpIndexPath))
 		{
-			fs::remove(IndexPath);
-			fs::rename(STmpIndexPath, IndexPath);
+			std::error_code Ec;
+			fs::remove(IndexPath, Ec);	// We don't care if this fails, we try to move the old temp file regardless
+			fs::rename(STmpIndexPath, IndexPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message());
+			}
 		}
 	}
 	if (fs::is_regular_file(STmpIndexPath))
 	{
-		fs::remove(STmpIndexPath);
+		std::error_code Ec;
+		if (!fs::remove(STmpIndexPath, Ec) || Ec)
+		{
+			ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message());
+		}
 	}
 }
 
@@ -1977,8 +1992,8 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 			}
 			catch (const std::exception& Err)
 			{
-				ZEN_ERROR("creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
-				return;
+				ZEN_WARN("creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
+				throw;
 			}
 		}
 	}