hardening of ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue

1 files changed, 36 insertions, 13 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index ee0835fd3..e46f540cf 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -1835,32 +1835,55 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 	// We do a speculative remove of the file instead of probing with a exists call and check the error code instead
 	std::filesystem::remove(FsPath, Ec);
-	if (Ec && Ec.value() != ENOENT)
+	if (Ec)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to replace file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir));
+		if (Ec.value() != ENOENT)
+		{
+			ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message());
+			Sleep(100);
+			Ec.clear();
+			std::filesystem::remove(FsPath, Ec);
+			if (Ec && Ec.value() != ENOENT)
+			{
+				throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir));
+			}
+		}
 	}
 
 	DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
 	if (Ec)
 	{
 		std::filesystem::path ParentPath = FsPath.parent_path();
-		if (std::filesystem::is_directory(ParentPath))
-		{
-			throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir));
-		}
-		Ec.clear();
-		std::filesystem::create_directories(ParentPath, Ec);
-		if (Ec)
+		if (!std::filesystem::is_directory(ParentPath))
 		{
-			throw std::system_error(
-				Ec,
-				fmt::format("Failed to create parent directory '{}' for file '{}' for put in '{}'", ParentPath, FsPath, m_BucketDir));
+			Ec.clear();
+			std::filesystem::create_directories(ParentPath, Ec);
+			if (Ec)
+			{
+				throw std::system_error(
+					Ec,
+					fmt::format("Failed to create parent directory '{}' for file '{}' for put in '{}'", ParentPath, FsPath, m_BucketDir));
+			}
 		}
 
+		// Try again
 		DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
 		if (Ec)
 		{
-			throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir));
+			ZEN_WARN("Failed to finalize file '{}', moving from '{}' for put in '{}', reason: '{}', retrying.",
+					 FsPath,
+					 DataFile.GetPath(),
+					 m_BucketDir,
+					 Ec.message());
+			Sleep(100);
+			Ec.clear();
+			DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
+			if (Ec)
+			{
+				throw std::system_error(
+					Ec,
+					fmt::format("Failed to finalize file '{}', moving from '{}' for put in '{}'", FsPath, DataFile.GetPath(), m_BucketDir));
+			}
 		}
 	}