From f43bee9eed90ee5175f9b3e0dd2a7b7d4b3145e4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 4 Sep 2023 09:14:46 -0400 Subject: retry file create (#383) * add retry logic when creating files * only write disk usage log if disk writes are allowed * changelog --- src/zenserver/projectstore/remoteprojectstore.cpp | 31 +++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'src/zenserver/projectstore') diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index 2806bc2d1..bbf3a9f32 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -277,7 +277,20 @@ BuildContainer(CidStore& ChunkStore, if (!AttachmentBuffer) { BasicFile BlockFile; - BlockFile.Open(AttachmentPath, BasicFile::Mode::kTruncateDelete); + uint32_t RetriesLeft = 3; + BlockFile.Open(AttachmentPath, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) { + if (RetriesLeft == 0) + { + return false; + } + ZEN_WARN("Failed to create temp attachment '{}', reason: '{}', retries left: {}.", + AttachmentPath, + Ec.message(), + RetriesLeft); + Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + RetriesLeft--; + return true; + }); uint64_t Offset = 0; for (const SharedBuffer& Buffer : Compressed.GetCompressed().GetSegments()) { @@ -637,7 +650,21 @@ SaveOplog(CidStore& ChunkStore, try { BasicFile BlockFile; - BlockFile.Open(BlockPath, BasicFile::Mode::kTruncateDelete); + uint32_t RetriesLeft = 3; + BlockFile.Open(BlockPath, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) { + if (RetriesLeft == 0) + { + return false; + } + ZEN_WARN("Failed to create temporary oplog block '{}', reason: '{}', retries left: {}.", + BlockPath, + Ec.message(), + RetriesLeft); + Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + RetriesLeft--; + return true; + }); + uint64_t Offset = 0; for (const SharedBuffer& Buffer : CompressedBlock.GetCompressed().GetSegments()) { -- cgit v1.2.3