add more forgiving retries with filesystem

1 files changed, 31 insertions, 20 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 2562d8244..f7f9e3abb 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -182,65 +182,74 @@ namespace {
 
 	std::filesystem::path MakeSafeAbsolutePath(const std::string PathString) { return MakeSafeAbsolutePath(StringToPath(PathString)); }
 
-	void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
+	bool IsFileWithRetry(const std::filesystem::path& Path)
 	{
 		std::error_code Ec;
-		RenameFile(SourcePath, TargetPath, Ec);
+		bool			Result = IsFile(Path, Ec);
 		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 		{
 			Sleep(100 + int(Retries * 50));
 			Ec.clear();
-			RenameFile(SourcePath, TargetPath, Ec);
+			Result = IsFile(Path, Ec);
 		}
 		if (Ec)
 		{
-			zen::ThrowSystemError(Ec.value(), Ec.message());
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Check path '{}' is file failed with: {} ({})", Path, Ec.message(), Ec.value()));
 		}
+		return Result;
 	}
 
-	bool IsFileWithRetry(const std::filesystem::path& Path)
+	bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
 	{
 		std::error_code Ec;
-		bool			Result = IsFile(Path, Ec);
+		bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
 		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 		{
 			Sleep(100 + int(Retries * 50));
+			if (!IsFileWithRetry(Path))
+			{
+				return false;
+			}
 			Ec.clear();
-			Result = IsFile(Path, Ec);
+			Result = SetFileReadOnly(Path, ReadOnly, Ec);
 		}
 		if (Ec)
 		{
-			zen::ThrowSystemError(Ec.value(), Ec.message());
+			throw std::system_error(
+				std::error_code(Ec.value(), std::system_category()),
+				fmt::format("Failed {} read only flag for '{}' failed with: {}", ReadOnly ? "setting" : "clearing", Path, Ec.message()));
 		}
 		return Result;
 	}
 
-	bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
+	void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
 	{
 		std::error_code Ec;
-		bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		RenameFile(SourcePath, TargetPath, Ec);
+		for (size_t Retries = 0; Ec && Retries < 10; Retries++)
 		{
-			Sleep(100 + int(Retries * 50));
-			if (!IsFileWithRetry(Path))
+			ZEN_ASSERT_SLOW(IsFile(SourcePath));
+			if (Retries > 5)
 			{
-				return false;
+				ZEN_CONSOLE("Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message());
 			}
+			Sleep(50 + int(Retries * 150));
 			Ec.clear();
-			Result = SetFileReadOnly(Path, ReadOnly, Ec);
+			RenameFile(SourcePath, TargetPath, Ec);
 		}
 		if (Ec)
 		{
-			zen::ThrowSystemError(Ec.value(), Ec.message());
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Rename from '{}' to '{}' failed with: {}", SourcePath, TargetPath, Ec.message()));
 		}
-		return Result;
 	}
 
 	void RemoveFileWithRetry(const std::filesystem::path& Path)
 	{
 		std::error_code Ec;
 		RemoveFile(Path, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		for (size_t Retries = 0; Ec && Retries < 6; Retries++)
 		{
 			Sleep(100 + int(Retries * 50));
 			if (!IsFileWithRetry(Path))
@@ -252,7 +261,8 @@ namespace {
 		}
 		if (Ec)
 		{
-			zen::ThrowSystemError(Ec.value(), Ec.message());
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Removing file '{}' failed with: {}", Path, Ec.message()));
 		}
 	}
 
@@ -272,7 +282,8 @@ namespace {
 		}
 		if (Ec)
 		{
-			zen::ThrowSystemError(Ec.value(), Ec.message());
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Removing directory '{}' failed with: {}", Path, Ec.message()));
 		}
 	}