From 0abf7994e8913c19360a0f0b8527495c0f99de87 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 2 Oct 2023 12:00:00 +0200 Subject: Handle OOM and OOD more gracefully to not spam Sentry with error reports (#434) - Improvement: Catch Out Of Memory and Out Of Disk exceptions and report back to reqeuster without reporting an error to Sentry - Improvement: If creating bucket fails when storing and item in the structured cache, log a warning and propagate error to requester without reporting an error to Sentry - Improvement: Make an explicit flush of the active block written to in blockstore flush - Improvement: Make sure cache and cas MakeIndexSnapshot does not throw exception on failure which would cause and abnormal termniation at exit --- src/zenstore/compactcas.cpp | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) (limited to 'src/zenstore/compactcas.cpp') diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index a138e43e9..1d1797597 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -565,15 +565,21 @@ CasContainerStrategy::MakeIndexSnapshot() // Move index away, we keep it if something goes wrong if (fs::is_regular_file(TempIndexPath)) { - fs::remove(TempIndexPath); - } - if (fs::is_regular_file(IndexPath)) - { - fs::rename(IndexPath, TempIndexPath); + std::error_code Ec; + if (!fs::remove(TempIndexPath, Ec) || Ec) + { + ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message()); + return; + } } try { + if (fs::is_regular_file(IndexPath)) + { + fs::rename(IndexPath, TempIndexPath); + } + // Write the current state of the location map to a new index state std::vector Entries; @@ -613,13 +619,22 @@ CasContainerStrategy::MakeIndexSnapshot() if (fs::is_regular_file(TempIndexPath)) { - fs::remove(IndexPath); - fs::rename(TempIndexPath, IndexPath); + std::error_code Ec; + fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + fs::rename(TempIndexPath, IndexPath, Ec); + if (Ec) + { + ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message()); + } } } if (fs::is_regular_file(TempIndexPath)) { - fs::remove(TempIndexPath); + std::error_code Ec; + if (!fs::remove(TempIndexPath, Ec) || Ec) + { + ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message()); + } } } -- cgit v1.2.3