From a1ef526b0e77b47d9085fbde9ed85b20824588bc Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 13 Oct 2021 19:29:24 +0200 Subject: structuredcache: add code to handle conflict which can occur when multiple PUTs of the same value occur close enough to cause issues due to file system races --- zenserver/cache/structuredcachestore.cpp | 35 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 580446473..ccd06b540 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -710,19 +710,38 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c // Move file into place (atomically) - DataFile.MoveTemporaryIntoPlace(DataFilePath.c_str(), Ec); + std::filesystem::path FsPath{DataFilePath.c_str()}; + + DataFile.MoveTemporaryIntoPlace(FsPath, Ec); if (Ec) { - std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path(); - CreateDirectories(ParentPath); - - DataFile.MoveTemporaryIntoPlace(DataFilePath.c_str(), Ec); + int RetryCount = 3; - if (Ec) + do { - throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); - } + std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path(); + CreateDirectories(ParentPath); + + DataFile.MoveTemporaryIntoPlace(FsPath, Ec); + + if (Ec) + { + std::error_code InnerEc; + const uint64_t ExistingFileSize = std::filesystem::file_size(FsPath, InnerEc); + + if (!InnerEc && ExistingFileSize == Value.Value.Size()) + { + // Concurrent write of same value? + return; + } + } + + // Semi arbitrary back-off + zen::Sleep(1000 * RetryCount); + } while (RetryCount--); + + throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); } // Update index -- cgit v1.2.3