retry file create (#383)

* add retry logic when creating files
* only write disk usage log if disk writes are allowed
* changelog

5 files changed, 73 insertions, 5 deletions

diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 2806bc2d1..bbf3a9f32 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -277,7 +277,20 @@ BuildContainer(CidStore&																	ChunkStore,
 								if (!AttachmentBuffer)
 								{
 									BasicFile BlockFile;
-									BlockFile.Open(AttachmentPath, BasicFile::Mode::kTruncateDelete);
+									uint32_t  RetriesLeft = 3;
+									BlockFile.Open(AttachmentPath, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) {
+										if (RetriesLeft == 0)
+										{
+											return false;
+										}
+										ZEN_WARN("Failed to create temp attachment '{}', reason: '{}', retries left: {}.",
+												 AttachmentPath,
+												 Ec.message(),
+												 RetriesLeft);
+										Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+										RetriesLeft--;
+										return true;
+									});
 									uint64_t Offset = 0;
 									for (const SharedBuffer& Buffer : Compressed.GetCompressed().GetSegments())
 									{
@@ -637,7 +650,21 @@ SaveOplog(CidStore&				 ChunkStore,
 			try
 			{
 				BasicFile BlockFile;
-				BlockFile.Open(BlockPath, BasicFile::Mode::kTruncateDelete);
+				uint32_t  RetriesLeft = 3;
+				BlockFile.Open(BlockPath, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) {
+					if (RetriesLeft == 0)
+					{
+						return false;
+					}
+					ZEN_WARN("Failed to create temporary oplog block '{}', reason: '{}', retries left: {}.",
+							 BlockPath,
+							 Ec.message(),
+							 RetriesLeft);
+					Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+					RetriesLeft--;
+					return true;
+				});
+
 				uint64_t Offset = 0;
 				for (const SharedBuffer& Buffer : CompressedBlock.GetCompressed().GetSegments())
 				{
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index d1490dce2..abf77f8a6 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -48,7 +48,17 @@ BlockStoreFile::GetPath() const
 void
 BlockStoreFile::Open()
 {
-	m_File.Open(m_Path, BasicFile::Mode::kDelete);
+	uint32_t RetriesLeft = 3;
+	m_File.Open(m_Path, BasicFile::Mode::kDelete, [&](std::error_code& Ec) {
+		if (RetriesLeft == 0)
+		{
+			return false;
+		}
+		ZEN_WARN("Failed to open cas block '{}', reason: '{}', retries left: {}.", m_Path, Ec.message(), RetriesLeft);
+		Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+		RetriesLeft--;
+		return true;
+	});
 	void* FileHandle = m_File.Handle();
 	m_IoBuffer		 = IoBuffer(IoBuffer::File, FileHandle, 0, m_File.FileSize());
 }
@@ -62,7 +72,18 @@ BlockStoreFile::Create(uint64_t InitialSize)
 		CreateDirectories(ParentPath);
 	}
 
-	m_File.Open(m_Path, BasicFile::Mode::kTruncateDelete);
+	uint32_t RetriesLeft = 3;
+	m_File.Open(m_Path, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) {
+		if (RetriesLeft == 0)
+		{
+			return false;
+		}
+		ZEN_WARN("Failed to create cas block '{}', reason: '{}', retries left: {}.", m_Path, Ec.message(), RetriesLeft);
+		Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+		RetriesLeft--;
+		return true;
+	});
+
 	void* FileHandle = m_File.Handle();
 
 	// We map our m_IoBuffer beyond the file size as we will grow it over time and want
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 245f76c92..79aea2752 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -838,8 +838,11 @@ GcScheduler::SchedulerThread()
 				{
 					const GcClock::Tick EpochTickCount = GcClock::Now().time_since_epoch().count();
 					std::unique_lock	Lock(m_GcMutex);
+					if (AreDiskWritesAllowed())
+					{
+						m_DiskUsageLog.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize});
+					}
 					m_DiskUsageWindow.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize});
-					m_DiskUsageLog.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize});
 					const GcClock::TimePoint LoadGraphStartTime = Now - LoadGraphTime;
 					const GcClock::Tick		 Start				= LoadGraphStartTime.time_since_epoch().count();
 					const GcClock::Tick		 End				= Now.time_since_epoch().count();
diff --git a/src/zenutil/basicfile.cpp b/src/zenutil/basicfile.cpp
index 12d9cf950..84e8bed85 100644
--- a/src/zenutil/basicfile.cpp
+++ b/src/zenutil/basicfile.cpp
@@ -128,6 +128,22 @@ BasicFile::Open(const std::filesystem::path& FileName, Mode InMode, std::error_c
 }
 
 void
+BasicFile::Open(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback)
+{
+	std::error_code Ec;
+	Open(FileName, Mode, Ec);
+	while (Ec && RetryCallback(Ec))
+	{
+		Ec.clear();
+		Open(FileName, Mode, Ec);
+	}
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed to open file '{}', mode: {:x}", FileName, uint32_t(Mode)));
+	}
+}
+
+void
 BasicFile::Close()
 {
 	if (m_FileHandle)
diff --git a/src/zenutil/include/zenutil/basicfile.h b/src/zenutil/include/zenutil/basicfile.h
index 5a27befd9..7cf8b05db 100644
--- a/src/zenutil/include/zenutil/basicfile.h
+++ b/src/zenutil/include/zenutil/basicfile.h
@@ -49,6 +49,7 @@ public:
 
 	void	 Open(const std::filesystem::path& FileName, Mode Mode);
 	void	 Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec);
+	void	 Open(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback);
 	void	 Close();
 	void	 Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	IoBuffer ReadRange(uint64_t FileOffset, uint64_t ByteCount);