From f43bee9eed90ee5175f9b3e0dd2a7b7d4b3145e4 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Mon, 4 Sep 2023 09:14:46 -0400
Subject: retry file create (#383)

* add retry logic when creating files
* only write disk usage log if disk writes are allowed
* changelog
---
 src/zenstore/blockstore.cpp | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

(limited to 'src/zenstore/blockstore.cpp')

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index d1490dce2..abf77f8a6 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -48,7 +48,17 @@ BlockStoreFile::GetPath() const
 void
 BlockStoreFile::Open()
 {
-	m_File.Open(m_Path, BasicFile::Mode::kDelete);
+	uint32_t RetriesLeft = 3;
+	m_File.Open(m_Path, BasicFile::Mode::kDelete, [&](std::error_code& Ec) {
+		if (RetriesLeft == 0)
+		{
+			return false;
+		}
+		ZEN_WARN("Failed to open cas block '{}', reason: '{}', retries left: {}.", m_Path, Ec.message(), RetriesLeft);
+		Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+		RetriesLeft--;
+		return true;
+	});
 	void* FileHandle = m_File.Handle();
 	m_IoBuffer		 = IoBuffer(IoBuffer::File, FileHandle, 0, m_File.FileSize());
 }
@@ -62,7 +72,18 @@ BlockStoreFile::Create(uint64_t InitialSize)
 		CreateDirectories(ParentPath);
 	}
 
-	m_File.Open(m_Path, BasicFile::Mode::kTruncateDelete);
+	uint32_t RetriesLeft = 3;
+	m_File.Open(m_Path, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) {
+		if (RetriesLeft == 0)
+		{
+			return false;
+		}
+		ZEN_WARN("Failed to create cas block '{}', reason: '{}', retries left: {}.", m_Path, Ec.message(), RetriesLeft);
+		Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+		RetriesLeft--;
+		return true;
+	});
+
 	void* FileHandle = m_File.Handle();
 
 	// We map our m_IoBuffer beyond the file size as we will grow it over time and want
-- 
cgit v1.2.3