From f9652d41e8faa782aa10432470e853b73397b639 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Mon, 5 Jun 2023 13:48:12 +0200
Subject: Increase retry logic (#325)

* Increase timeout and number of retries in CacheBucket::PutStandaloneCacheValue when moving temporary file into place
* changelog
---
 src/zenserver/cache/cachedisklayer.cpp | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

(limited to 'src/zenserver/cache/cachedisklayer.cpp')

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 54ac2807b..9d3935131 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -1701,30 +1701,36 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 		}
 	}
 
+	// Assume parent directory exists
 	DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
 	if (Ec)
 	{
 		CreateDirectories(FsPath.parent_path());
-		Ec.clear();
 
-		// Try again
+		// Try again after we or someone else created the directory
+		Ec.clear();
 		DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
-		if (Ec)
+
+		// Retry if we still fail to handle contention to file system
+		uint32_t RetriesLeft = 3;
+		while (Ec && RetriesLeft > 0)
 		{
-			ZEN_WARN("Failed to finalize file '{}', moving from '{}' for put in '{}', reason: '{}', retrying.",
+			ZEN_WARN("Failed to finalize file '{}', moving from '{}' for put in '{}', reason: '{}', retries left: {}.",
 					 FsPath,
 					 DataFile.GetPath(),
 					 m_BucketDir,
-					 Ec.message());
-			Sleep(100);
+					 Ec.message(),
+					 RetriesLeft);
+			Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
 			Ec.clear();
 			DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
-			if (Ec)
-			{
-				throw std::system_error(
-					Ec,
-					fmt::format("Failed to finalize file '{}', moving from '{}' for put in '{}'", FsPath, DataFile.GetPath(), m_BucketDir));
-			}
+			RetriesLeft--;
+		}
+		if (Ec)
+		{
+			throw std::system_error(
+				Ec,
+				fmt::format("Failed to finalize file '{}', moving from '{}' for put in '{}'", FsPath, DataFile.GetPath(), m_BucketDir));
 		}
 	}
 
@@ -2124,4 +2130,4 @@ ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const st
 	return Details;
 }
 
-}  // namespace zen
\ No newline at end of file
+}  // namespace zen
-- 
cgit v1.2.3