Merge pull request #120 from EpicGames/de/fix-failed-to-finalize-filev0.1.2-pre2v0.1.2

Speculative fix for 'failed to finalize file'

2 files changed, 44 insertions, 13 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d8fae2ed..468f4a1ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,10 @@
 ##
+- Tweak bundle compression settings to streamline build
+- ZenCacheDiskLayer::CacheBucket::GatherReferences: Don't hold index lock while reading standalone values
+- hardening of ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue
+- GitHub Actions: Move release job to in-house linux agent
+
+## v0.1.1
 - BlockStore (small object store) Always block GC of current write block
 - Make it possible to configure GC monitoring interval using `--gc-monitor-interval-seconds`
 - Keep "reason" from upstream response so we can present it even if the request fails without outright error
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index ee0835fd3..bc60a1d26 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -1440,6 +1440,8 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 				 });
 				if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 				{
+					// We don't need to hold the index lock when we read a standalone file
+					__.ReleaseNow();
 					if (!GetStandaloneCacheValue(Loc, Key, CacheValue))
 					{
 						continue;
@@ -1835,32 +1837,55 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 	// We do a speculative remove of the file instead of probing with a exists call and check the error code instead
 	std::filesystem::remove(FsPath, Ec);
-	if (Ec && Ec.value() != ENOENT)
+	if (Ec)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to replace file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir));
+		if (Ec.value() != ENOENT)
+		{
+			ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message());
+			Sleep(100);
+			Ec.clear();
+			std::filesystem::remove(FsPath, Ec);
+			if (Ec && Ec.value() != ENOENT)
+			{
+				throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir));
+			}
+		}
 	}
 
 	DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
 	if (Ec)
 	{
 		std::filesystem::path ParentPath = FsPath.parent_path();
-		if (std::filesystem::is_directory(ParentPath))
-		{
-			throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir));
-		}
-		Ec.clear();
-		std::filesystem::create_directories(ParentPath, Ec);
-		if (Ec)
+		if (!std::filesystem::is_directory(ParentPath))
 		{
-			throw std::system_error(
-				Ec,
-				fmt::format("Failed to create parent directory '{}' for file '{}' for put in '{}'", ParentPath, FsPath, m_BucketDir));
+			Ec.clear();
+			std::filesystem::create_directories(ParentPath, Ec);
+			if (Ec)
+			{
+				throw std::system_error(
+					Ec,
+					fmt::format("Failed to create parent directory '{}' for file '{}' for put in '{}'", ParentPath, FsPath, m_BucketDir));
+			}
 		}
 
+		// Try again
 		DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
 		if (Ec)
 		{
-			throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir));
+			ZEN_WARN("Failed to finalize file '{}', moving from '{}' for put in '{}', reason: '{}', retrying.",
+					 FsPath,
+					 DataFile.GetPath(),
+					 m_BucketDir,
+					 Ec.message());
+			Sleep(100);
+			Ec.clear();
+			DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
+			if (Ec)
+			{
+				throw std::system_error(
+					Ec,
+					fmt::format("Failed to finalize file '{}', moving from '{}' for put in '{}'", FsPath, DataFile.GetPath(), m_BucketDir));
+			}
 		}
 	}