Increase retry logic (#325)

* Increase timeout and number of retries in CacheBucket::PutStandaloneCacheValue when moving temporary file into place
* changelog

2 files changed, 20 insertions, 13 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f73bb6931..c28d27104 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@
 - Improvement: Added support for streaming decompression
 - Improvement: Added zenserver.exe and zen.exe/zen.pdb to Sentry debug information upload to populate unwind information
 - Improvement: Front-end can now be served from a development directory in release mode as well as debug if there's no zipfs attached
+- Improvement: Increased retry logic in diskcachelayer when we are denied moving a temporary file into place
 - Update: Bump CI VCPKG version to 2023.04.15 and xmake to 2.7.9 (was 2022.08.15 and 2.6.5)
 
 ## 0.2.12
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 54ac2807b..9d3935131 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -1701,30 +1701,36 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 		}
 	}
 
+	// Assume parent directory exists
 	DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
 	if (Ec)
 	{
 		CreateDirectories(FsPath.parent_path());
-		Ec.clear();
 
-		// Try again
+		// Try again after we or someone else created the directory
+		Ec.clear();
 		DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
-		if (Ec)
+
+		// Retry if we still fail to handle contention to file system
+		uint32_t RetriesLeft = 3;
+		while (Ec && RetriesLeft > 0)
 		{
-			ZEN_WARN("Failed to finalize file '{}', moving from '{}' for put in '{}', reason: '{}', retrying.",
+			ZEN_WARN("Failed to finalize file '{}', moving from '{}' for put in '{}', reason: '{}', retries left: {}.",
 					 FsPath,
 					 DataFile.GetPath(),
 					 m_BucketDir,
-					 Ec.message());
-			Sleep(100);
+					 Ec.message(),
+					 RetriesLeft);
+			Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
 			Ec.clear();
 			DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
-			if (Ec)
-			{
-				throw std::system_error(
-					Ec,
-					fmt::format("Failed to finalize file '{}', moving from '{}' for put in '{}'", FsPath, DataFile.GetPath(), m_BucketDir));
-			}
+			RetriesLeft--;
+		}
+		if (Ec)
+		{
+			throw std::system_error(
+				Ec,
+				fmt::format("Failed to finalize file '{}', moving from '{}' for put in '{}'", FsPath, DataFile.GetPath(), m_BucketDir));
 		}
 	}
 
@@ -2124,4 +2130,4 @@ ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const st
 	return Details;
 }
 
-}  // namespace zen
\ No newline at end of file
+}  // namespace zen