fix clean directory and make them use effective threading where appropriate (#625)v5.7.8-pre5v5.7.8-pre3v5.7.8-pre2

fix retry logic so it does not immediately sleep if file does not exist
make sure we don't try to delete target folder files if we have already wiped it

1 files changed, 44 insertions, 11 deletions

diff --git a/src/zenremotestore/filesystemutils.cpp b/src/zenremotestore/filesystemutils.cpp
index 20ab3faea..8dff05c6b 100644
--- a/src/zenremotestore/filesystemutils.cpp
+++ b/src/zenremotestore/filesystemutils.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include "filesystemutils.h"
+#include <zenremotestore/filesystemutils.h>
 
 #include <zenremotestore/chunking/chunkedcontent.h>
 
@@ -215,11 +215,11 @@ SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
 	bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
 	for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 	{
-		Sleep(100 + int(Retries * 50));
 		if (!IsFileWithRetry(Path))
 		{
 			return false;
 		}
+		Sleep(100 + int(Retries * 50));
 		Ec.clear();
 		Result = SetFileReadOnly(Path, ReadOnly, Ec);
 	}
@@ -277,11 +277,11 @@ RemoveFileWithRetry(const std::filesystem::path& Path)
 	RemoveFile(Path, Ec);
 	for (size_t Retries = 0; Ec && Retries < 6; Retries++)
 	{
-		Sleep(100 + int(Retries * 50));
 		if (!IsFileWithRetry(Path))
 		{
 			return;
 		}
+		Sleep(100 + int(Retries * 50));
 		Ec.clear();
 		RemoveFile(Path, Ec);
 	}
@@ -426,12 +426,13 @@ CleanDirectory(
 							(void)SetFileReadOnly(FilePath, false, Ec);
 							for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 							{
-								Sleep(100 + int(Retries * 50));
 								if (!IsFileWithRetry(FilePath))
 								{
 									IsRemoved = true;
+									Ec.clear();
 									break;
 								}
+								Sleep(100 + int(Retries * 50));
 								Ec.clear();
 								(void)SetFileReadOnly(FilePath, false, Ec);
 							}
@@ -440,12 +441,13 @@ CleanDirectory(
 								(void)RemoveFile(FilePath, Ec);
 								for (size_t Retries = 0; Ec && Retries < 6; Retries++)
 								{
-									Sleep(100 + int(Retries * 50));
 									if (!IsFileWithRetry(FilePath))
 									{
 										IsRemoved = true;
+										Ec.clear();
 										return;
 									}
+									Sleep(100 + int(Retries * 50));
 									Ec.clear();
 									(void)RemoveFile(FilePath, Ec);
 								}
@@ -542,25 +544,27 @@ CleanDirectory(
 
 			{
 				std::error_code Ec;
-				zen::CleanDirectory(DirectoryToDelete, true, Ec);
+				zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec);
 				if (Ec)
 				{
 					Sleep(200);
-					zen::CleanDirectory(DirectoryToDelete, true, Ec);
+					Ec.clear();
+					zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec);
 				}
 
 				if (!Ec)
 				{
-					RemoveDir(Path, Ec);
+					RemoveDir(DirectoryToDelete, Ec);
 					for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 					{
-						Sleep(100 + int(Retries * 50));
-						if (!IsDir(Path))
+						if (!IsDir(DirectoryToDelete))
 						{
+							Ec.clear();
 							break;
 						}
+						Sleep(100 + int(Retries * 50));
 						Ec.clear();
-						RemoveDir(Path, Ec);
+						RemoveDir(DirectoryToDelete, Ec);
 					}
 				}
 				if (Ec)
@@ -592,4 +596,33 @@ CleanDirectory(
 	return Result;
 }
 
+bool
+CleanAndRemoveDirectory(WorkerThreadPool&			 WorkerPool,
+						std::atomic<bool>&			 AbortFlag,
+						std::atomic<bool>&			 PauseFlag,
+						const std::filesystem::path& Directory)
+{
+	if (!IsDir(Directory))
+	{
+		return true;
+	}
+	if (CleanDirectoryResult Res = CleanDirectory(
+			WorkerPool,
+			AbortFlag,
+			PauseFlag,
+			Directory,
+			{},
+			[](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) {
+				ZEN_UNUSED(Details, TotalCount, RemainingCount, IsPaused, IsAborted);
+			},
+			1000);
+		Res.FailedRemovePaths.empty())
+	{
+		std::error_code Ec;
+		RemoveDir(Directory, Ec);
+		return !Ec;
+	}
+	return false;
+}
+
 }  // namespace zen