structuredcache: add code to handle conflict which can occur when multiple PUTs of the same value occur close enough to cause issues due to file system races

1 files changed, 27 insertions, 8 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 580446473..ccd06b540 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -710,19 +710,38 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 	// Move file into place (atomically)
 
-	DataFile.MoveTemporaryIntoPlace(DataFilePath.c_str(), Ec);
+	std::filesystem::path FsPath{DataFilePath.c_str()};
+
+	DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
 
 	if (Ec)
 	{
-		std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path();
-		CreateDirectories(ParentPath);
-
-		DataFile.MoveTemporaryIntoPlace(DataFilePath.c_str(), Ec);
+		int RetryCount = 3;
 
-		if (Ec)
+		do
 		{
-			throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath)));
-		}
+			std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path();
+			CreateDirectories(ParentPath);
+
+			DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
+
+			if (Ec)
+			{
+				std::error_code InnerEc;
+				const uint64_t ExistingFileSize = std::filesystem::file_size(FsPath, InnerEc);
+
+				if (!InnerEc && ExistingFileSize == Value.Value.Size())
+				{
+					// Concurrent write of same value?
+					return;
+				}
+			}
+
+			// Semi arbitrary back-off
+			zen::Sleep(1000 * RetryCount);
+		} while (RetryCount--);
+
+		throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath)));
 	}
 
 	// Update index