move file i/o related files to separate file and remove duplicated code (#576)

6 files changed, 908 insertions, 909 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 6a0a1a0da..a911d1e44 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -32,8 +32,6 @@
 #include <zenremotestore/chunking/chunkedfile.h>
 #include <zenremotestore/chunking/chunkingcontroller.h>
 #include <zenremotestore/jupiter/jupiterhost.h>
-#include <zenutil/bufferedopenfile.h>
-#include <zenutil/bufferedwritefilecache.h>
 #include <zenutil/wildcard.h>
 #include <zenutil/workerpools.h>
 #include <zenutil/zenserverprocess.h>
@@ -385,288 +383,6 @@ namespace {
 		return true;
 	}
 
-	bool IsFileWithRetry(const std::filesystem::path& Path)
-	{
-		std::error_code Ec;
-		bool			Result = IsFile(Path, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			Ec.clear();
-			Result = IsFile(Path, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
-									fmt::format("Check path '{}' is file failed with: {} ({})", Path, Ec.message(), Ec.value()));
-		}
-		return Result;
-	}
-
-	bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
-	{
-		std::error_code Ec;
-		bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			if (!IsFileWithRetry(Path))
-			{
-				return false;
-			}
-			Ec.clear();
-			Result = SetFileReadOnly(Path, ReadOnly, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(
-				std::error_code(Ec.value(), std::system_category()),
-				fmt::format("Failed {} read only flag for '{}' failed with: {}", ReadOnly ? "setting" : "clearing", Path, Ec.message()));
-		}
-		return Result;
-	}
-
-	void RemoveFileWithRetry(const std::filesystem::path& Path)
-	{
-		std::error_code Ec;
-		RemoveFile(Path, Ec);
-		for (size_t Retries = 0; Ec && Retries < 6; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			if (!IsFileWithRetry(Path))
-			{
-				return;
-			}
-			Ec.clear();
-			RemoveFile(Path, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
-									fmt::format("Removing file '{}' failed with: {}", Path, Ec.message()));
-		}
-	}
-
-	void RemoveDirWithRetry(const std::filesystem::path& Path)
-	{
-		std::error_code Ec;
-		RemoveDir(Path, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			if (!IsDir(Path))
-			{
-				return;
-			}
-			Ec.clear();
-			RemoveDir(Path, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
-									fmt::format("Removing directory '{}' failed with: {}", Path, Ec.message()));
-		}
-	}
-
-	bool CleanDirectory(const std::filesystem::path& Path, std::span<const std::string> ExcludeDirectories)
-	{
-		ZEN_TRACE_CPU("CleanDirectory");
-		Stopwatch Timer;
-
-		ProgressBar Progress(ProgressMode, "Clean Folder");
-
-		std::atomic<bool>	  CleanWipe			  = true;
-		std::atomic<uint64_t> DiscoveredItemCount = 0;
-		std::atomic<uint64_t> DeletedItemCount	  = 0;
-		std::atomic<uint64_t> DeletedByteCount	  = 0;
-		ParallelWork		  Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
-		struct AsyncVisitor : public GetDirectoryContentVisitor
-		{
-			AsyncVisitor(const std::filesystem::path& InPath,
-						 std::atomic<bool>&			  InCleanWipe,
-						 std::atomic<uint64_t>&		  InDiscoveredItemCount,
-						 std::atomic<uint64_t>&		  InDeletedItemCount,
-						 std::atomic<uint64_t>&		  InDeletedByteCount,
-						 std::span<const std::string> InExcludeDirectories)
-			: Path(InPath)
-			, CleanWipe(InCleanWipe)
-			, DiscoveredItemCount(InDiscoveredItemCount)
-			, DeletedItemCount(InDeletedItemCount)
-			, DeletedByteCount(InDeletedByteCount)
-			, ExcludeDirectories(InExcludeDirectories)
-			{
-			}
-			virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
-			{
-				ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
-				if (!AbortFlag)
-				{
-					if (!Content.FileNames.empty())
-					{
-						DiscoveredItemCount += Content.FileNames.size();
-
-						const std::string RelativeRootString = RelativeRoot.generic_string();
-						bool			  RemoveContent		 = true;
-						for (const std::string_view ExcludeDirectory : ExcludeDirectories)
-						{
-							if (RelativeRootString.starts_with(ExcludeDirectory))
-							{
-								if (RelativeRootString.length() > ExcludeDirectory.length())
-								{
-									const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
-									if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
-										MaybePathDelimiter == std::filesystem::path::preferred_separator)
-									{
-										RemoveContent = false;
-										break;
-									}
-								}
-								else
-								{
-									RemoveContent = false;
-									break;
-								}
-							}
-						}
-						if (RemoveContent)
-						{
-							ZEN_TRACE_CPU("DeleteFiles");
-							for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
-							{
-								const std::filesystem::path& FileName = Content.FileNames[FileIndex];
-								const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
-								try
-								{
-									SetFileReadOnlyWithRetry(FilePath, false);
-									RemoveFileWithRetry(FilePath);
-									DeletedItemCount++;
-									DeletedByteCount += Content.FileSizes[FileIndex];
-								}
-								catch (const std::exception& Ex)
-								{
-									ZEN_CONSOLE_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what());
-									CleanWipe = false;
-								}
-							}
-						}
-					}
-				}
-			}
-			const std::filesystem::path& Path;
-			std::atomic<bool>&			 CleanWipe;
-			std::atomic<uint64_t>&		 DiscoveredItemCount;
-			std::atomic<uint64_t>&		 DeletedItemCount;
-			std::atomic<uint64_t>&		 DeletedByteCount;
-			std::span<const std::string> ExcludeDirectories;
-		} Visitor(Path, CleanWipe, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories);
-
-		GetDirectoryContent(
-			Path,
-			DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes,
-			Visitor,
-			GetIOWorkerPool(),
-			Work.PendingWork());
-
-		DirectoryContent LocalDirectoryContent;
-		GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
-		DiscoveredItemCount += LocalDirectoryContent.Directories.size();
-		std::vector<std::filesystem::path> DirectoriesToDelete;
-		DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size());
-		for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
-		{
-			bool Leave = false;
-			for (const std::string& ExcludeDirectory : ExcludeDirectories)
-			{
-				if (LocalDirPath == (Path / ExcludeDirectory))
-				{
-					Leave = true;
-					break;
-				}
-			}
-			if (!Leave)
-			{
-				DirectoriesToDelete.emplace_back(std::move(LocalDirPath));
-				DiscoveredItemCount++;
-			}
-		}
-
-		uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
-
-		Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) {
-			ZEN_UNUSED(PendingWork);
-			LastUpdateTimeMs = Timer.GetElapsedTimeMs();
-
-			uint64_t Deleted	  = DeletedItemCount.load();
-			uint64_t DeletedBytes = DeletedByteCount.load();
-			uint64_t Discovered	  = DiscoveredItemCount.load();
-			Progress.UpdateState({.Task			  = "Cleaning folder  ",
-								  .Details		  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
-								  .TotalCount	  = Discovered,
-								  .RemainingCount = Discovered - Deleted,
-								  .Status		  = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
-								 false);
-		});
-
-		for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete)
-		{
-			ZEN_TRACE_CPU("DeleteDirs");
-			try
-			{
-				std::error_code Ec;
-				zen::CleanDirectory(DirectoryToDelete, true, Ec);
-				if (Ec)
-				{
-					Sleep(200);
-					zen::CleanDirectory(DirectoryToDelete, true);
-					Ec.clear();
-				}
-
-				RemoveDirWithRetry(DirectoryToDelete);
-
-				DeletedItemCount++;
-			}
-			catch (const std::exception& Ex)
-			{
-				ZEN_CONSOLE_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
-				CleanWipe = false;
-			}
-
-			uint64_t NowMs = Timer.GetElapsedTimeMs();
-			if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode))
-			{
-				LastUpdateTimeMs = NowMs;
-
-				uint64_t Deleted	  = DeletedItemCount.load();
-				uint64_t DeletedBytes = DeletedByteCount.load();
-				uint64_t Discovered	  = DiscoveredItemCount.load();
-				Progress.UpdateState({.Task		  = "Cleaning folder  ",
-									  .Details	  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
-									  .TotalCount = Discovered,
-									  .RemainingCount = Discovered - Deleted,
-									  .Status		  = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
-									 false);
-			}
-		}
-
-		Progress.Finish();
-		if (AbortFlag)
-		{
-			return false;
-		}
-
-		uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
-		if (ElapsedTimeMs >= 200 && !IsQuiet)
-		{
-			ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}",
-						Path,
-						DiscoveredItemCount.load(),
-						NiceBytes(DeletedByteCount.load()),
-						NiceTimeSpanMs(ElapsedTimeMs));
-		}
-		return CleanWipe;
-	}
-
 	class FilteredRate
 	{
 	public:
@@ -2934,7 +2650,8 @@ namespace {
 
 		if (CleanDirectory(ZenTempFolder, {}))
 		{
-			RemoveDirWithRetry(ZenTempFolder);
+			std::error_code DummyEc;
+			RemoveDir(ZenTempFolder, DummyEc);
 		}
 	}
 
@@ -5132,7 +4849,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 										[SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
 											if (!AbortFlag)
 											{
-												bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false);
+												bool WasReadOnly = SetFileReadOnly(FilePath, false);
 												{
 													BasicFile Source(FilePath, BasicFile::Mode::kWrite);
 													uint64_t  RangeSize = Min(SourceSize / 3, 512u * 1024u);
@@ -5146,7 +4863,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 													Source.Write(TempBuffer2, SourceSize - RangeSize);
 													Source.Write(TempBuffer3, SourceSize - 0);
 												}
-												if (IsReadOnly)
+												if (WasReadOnly)
 												{
 													SetFileReadOnly(FilePath, true);
 												}
@@ -5157,8 +4874,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 							break;
 						case 1:
 							{
-								(void)SetFileReadOnlyWithRetry(FilePath, false);
-								RemoveFileWithRetry(FilePath);
+								(void)SetFileReadOnly(FilePath, false);
+								(void)RemoveFile(FilePath);
 							}
 							break;
 						default:
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 60058c4a4..ca927e643 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -17,7 +17,7 @@
 #include <zencore/string.h>
 #include <zencore/timer.h>
 #include <zencore/trace.h>
-#include <zenutil/bufferedopenfile.h>
+#include "../filesystemutils.h"
 
 #include <numeric>
 
@@ -77,181 +77,10 @@ namespace {
 		return CacheFolderPath / RawHash.ToHexString();
 	}
 
-	uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes)
-	{
-#if ZEN_PLATFORM_WINDOWS
-		if (SourcePlatform == SourcePlatform::Windows)
-		{
-			SetFileAttributesToPath(FilePath, Attributes);
-			return Attributes;
-		}
-		else
-		{
-			uint32_t CurrentAttributes = GetFileAttributesFromPath(FilePath);
-			uint32_t NewAttributes	   = zen::MakeFileAttributeReadOnly(CurrentAttributes, zen::IsFileModeReadOnly(Attributes));
-			if (CurrentAttributes != NewAttributes)
-			{
-				SetFileAttributesToPath(FilePath, NewAttributes);
-			}
-			return NewAttributes;
-		}
-#endif	// ZEN_PLATFORM_WINDOWS
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-		if (SourcePlatform != SourcePlatform::Windows)
-		{
-			zen::SetFileMode(FilePath, Attributes);
-			return Attributes;
-		}
-		else
-		{
-			uint32_t CurrentMode = zen::GetFileMode(FilePath);
-			uint32_t NewMode	 = zen::MakeFileModeReadOnly(CurrentMode, zen::IsFileAttributeReadOnly(Attributes));
-			if (CurrentMode != NewMode)
-			{
-				zen::SetFileMode(FilePath, NewMode);
-			}
-			return NewMode;
-		}
-#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	};
-
-	uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath)
-	{
-#if ZEN_PLATFORM_WINDOWS
-		return GetFileAttributesFromPath(FilePath);
-#endif	// ZEN_PLATFORM_WINDOWS
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-		return GetFileMode(FilePath);
-#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	}
-
-	bool IsFileWithRetry(const std::filesystem::path& Path)
-	{
-		std::error_code Ec;
-		bool			Result = IsFile(Path, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			Ec.clear();
-			Result = IsFile(Path, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
-									fmt::format("Check path '{}' is file failed with: {} ({})", Path, Ec.message(), Ec.value()));
-		}
-		return Result;
-	}
-
-	bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
-	{
-		std::error_code Ec;
-		bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			if (!IsFileWithRetry(Path))
-			{
-				return false;
-			}
-			Ec.clear();
-			Result = SetFileReadOnly(Path, ReadOnly, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(
-				std::error_code(Ec.value(), std::system_category()),
-				fmt::format("Failed {} read only flag for '{}' failed with: {}", ReadOnly ? "setting" : "clearing", Path, Ec.message()));
-		}
-		return Result;
-	}
-
-	void RenameFileWithRetry(BuildOpLogOutput& LogOutput, const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
-	{
-		std::error_code Ec;
-		RenameFile(SourcePath, TargetPath, Ec);
-		for (size_t Retries = 0; Ec && Retries < 10; Retries++)
-		{
-			ZEN_ASSERT_SLOW(IsFile(SourcePath));
-			if (Retries > 5)
-			{
-				LOG_OUTPUT_WARN(LogOutput, "Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message());
-			}
-			Sleep(50 + int(Retries * 150));
-			Ec.clear();
-			RenameFile(SourcePath, TargetPath, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
-									fmt::format("Rename from '{}' to '{}' failed with: {}", SourcePath, TargetPath, Ec.message()));
-		}
-	}
-
-	void TryRemoveFile(BuildOpLogOutput& LogOutput, const std::filesystem::path& Path)
-	{
-		std::error_code Ec;
-		RemoveFile(Path, Ec);
-		if (Ec)
-		{
-			if (IsFile(Path, Ec))
-			{
-				Ec.clear();
-				RemoveFile(Path, Ec);
-				if (Ec)
-				{
-					LOG_OUTPUT_DEBUG(LogOutput, "Failed removing file '{}', reason: {}", Path, Ec.message());
-				}
-			}
-		}
-	}
-
-	void RemoveFileWithRetry(const std::filesystem::path& Path)
-	{
-		std::error_code Ec;
-		RemoveFile(Path, Ec);
-		for (size_t Retries = 0; Ec && Retries < 6; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			if (!IsFileWithRetry(Path))
-			{
-				return;
-			}
-			Ec.clear();
-			RemoveFile(Path, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
-									fmt::format("Removing file '{}' failed with: {}", Path, Ec.message()));
-		}
-	}
-
-	void RemoveDirWithRetry(const std::filesystem::path& Path)
-	{
-		std::error_code Ec;
-		RemoveDir(Path, Ec);
-		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
-		{
-			Sleep(100 + int(Retries * 50));
-			if (!IsDir(Path))
-			{
-				return;
-			}
-			Ec.clear();
-			RemoveDir(Path, Ec);
-		}
-		if (Ec)
-		{
-			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
-									fmt::format("Removing directory '{}' failed with: {}", Path, Ec.message()));
-		}
-	}
-
-	bool CleanDirectory(WorkerThreadPool&			 IOWorkerPool,
+	bool CleanDirectory(BuildOpLogOutput&			 LogOutput,
+						WorkerThreadPool&			 IOWorkerPool,
 						std::atomic<bool>&			 AbortFlag,
 						std::atomic<bool>&			 PauseFlag,
-						BuildOpLogOutput&			 LogOutput,
 						bool						 IsQuiet,
 						const std::filesystem::path& Path,
 						std::span<const std::string> ExcludeDirectories)
@@ -262,241 +91,55 @@ namespace {
 		std::unique_ptr<BuildOpLogOutput::ProgressBar> ProgressBarPtr(LogOutput.CreateProgressBar("Clean Folder"));
 		BuildOpLogOutput::ProgressBar&				   Progress(*ProgressBarPtr);
 
-		std::atomic<bool>	  CleanWipe			  = true;
-		std::atomic<uint64_t> DiscoveredItemCount = 0;
-		std::atomic<uint64_t> DeletedItemCount	  = 0;
-		std::atomic<uint64_t> DeletedByteCount	  = 0;
-		ParallelWork		  Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
-		struct AsyncVisitor : public GetDirectoryContentVisitor
-		{
-			AsyncVisitor(const std::filesystem::path& InPath,
-						 std::atomic<bool>&			  InAbortFlag,
-						 BuildOpLogOutput&			  InLogOutput,
-						 std::atomic<bool>&			  InCleanWipe,
-						 std::atomic<uint64_t>&		  InDiscoveredItemCount,
-						 std::atomic<uint64_t>&		  InDeletedItemCount,
-						 std::atomic<uint64_t>&		  InDeletedByteCount,
-						 std::span<const std::string> InExcludeDirectories)
-			: Path(InPath)
-			, AbortFlag(InAbortFlag)
-			, LogOutput(InLogOutput)
-			, CleanWipe(InCleanWipe)
-			, DiscoveredItemCount(InDiscoveredItemCount)
-			, DeletedItemCount(InDeletedItemCount)
-			, DeletedByteCount(InDeletedByteCount)
-			, ExcludeDirectories(InExcludeDirectories)
-			{
-			}
-			virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
-			{
-				ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
-				if (!AbortFlag)
-				{
-					if (!Content.FileNames.empty())
-					{
-						DiscoveredItemCount += Content.FileNames.size();
-
-						const std::string RelativeRootString = RelativeRoot.generic_string();
-						bool			  RemoveContent		 = true;
-						for (const std::string_view ExcludeDirectory : ExcludeDirectories)
-						{
-							if (RelativeRootString.starts_with(ExcludeDirectory))
-							{
-								if (RelativeRootString.length() > ExcludeDirectory.length())
-								{
-									const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
-									if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
-										MaybePathDelimiter == std::filesystem::path::preferred_separator)
-									{
-										RemoveContent = false;
-										break;
-									}
-								}
-								else
-								{
-									RemoveContent = false;
-									break;
-								}
-							}
-						}
-						if (RemoveContent)
-						{
-							ZEN_TRACE_CPU("DeleteFiles");
-							for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
-							{
-								const std::filesystem::path& FileName = Content.FileNames[FileIndex];
-								const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
-								try
-								{
-									SetFileReadOnlyWithRetry(FilePath, false);
-									RemoveFileWithRetry(FilePath);
-									DeletedItemCount++;
-									DeletedByteCount += Content.FileSizes[FileIndex];
-								}
-								catch (const std::exception& Ex)
-								{
-									LOG_OUTPUT_WARN(LogOutput, "Failed removing file {}. Reason: {}", FilePath, Ex.what());
-									CleanWipe = false;
-								}
-							}
-						}
-					}
-				}
-			}
-			const std::filesystem::path& Path;
-			std::atomic<bool>&			 AbortFlag;
-			BuildOpLogOutput&			 LogOutput;
-			std::atomic<bool>&			 CleanWipe;
-			std::atomic<uint64_t>&		 DiscoveredItemCount;
-			std::atomic<uint64_t>&		 DeletedItemCount;
-			std::atomic<uint64_t>&		 DeletedByteCount;
-			std::span<const std::string> ExcludeDirectories;
-		} Visitor(Path, AbortFlag, LogOutput, CleanWipe, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories);
-
-		GetDirectoryContent(
-			Path,
-			DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes,
-			Visitor,
+		CleanDirectoryResult Result = CleanDirectory(
 			IOWorkerPool,
-			Work.PendingWork());
+			AbortFlag,
+			PauseFlag,
+			Path,
+			ExcludeDirectories,
+			[&](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) {
+				Progress.UpdateState({.Task			  = "Cleaning folder  ",
+									  .Details		  = std::string(Details),
+									  .TotalCount	  = TotalCount,
+									  .RemainingCount = RemainingCount,
+									  .Status		  = BuildOpLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+									 false);
+			},
+			LogOutput.GetProgressUpdateDelayMS());
+
+		Progress.Finish();
 
-		DirectoryContent LocalDirectoryContent;
-		GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
-		DiscoveredItemCount += LocalDirectoryContent.Directories.size();
-		std::vector<std::filesystem::path> DirectoriesToDelete;
-		DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size());
-		for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
+		if (AbortFlag)
 		{
-			bool Leave = false;
-			for (const std::string_view ExcludeDirectory : ExcludeDirectories)
-			{
-				if (LocalDirPath == (Path / ExcludeDirectory))
-				{
-					Leave = true;
-					break;
-				}
-			}
-			if (!Leave)
-			{
-				DirectoriesToDelete.emplace_back(std::move(LocalDirPath));
-				DiscoveredItemCount++;
-			}
+			return false;
 		}
 
-		uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
-
-		Work.Wait(LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) {
-			ZEN_UNUSED(PendingWork);
-			LastUpdateTimeMs = Timer.GetElapsedTimeMs();
-
-			uint64_t Deleted	  = DeletedItemCount.load();
-			uint64_t DeletedBytes = DeletedByteCount.load();
-			uint64_t Discovered	  = DiscoveredItemCount.load();
-			Progress.UpdateState({.Task			  = "Cleaning folder  ",
-								  .Details		  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
-								  .TotalCount	  = Discovered,
-								  .RemainingCount = Discovered - Deleted,
-								  .Status		  = BuildOpLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
-								 false);
-		});
+		uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
 
-		for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete)
+		if (!Result.FailedRemovePaths.empty())
 		{
-			ZEN_TRACE_CPU("DeleteDirs");
-			try
-			{
-				std::error_code Ec;
-				zen::CleanDirectory(DirectoryToDelete, true, Ec);
-				if (Ec)
-				{
-					Sleep(200);
-					zen::CleanDirectory(DirectoryToDelete, true);
-					Ec.clear();
-				}
-
-				RemoveDirWithRetry(DirectoryToDelete);
-
-				DeletedItemCount++;
-			}
-			catch (const std::exception& Ex)
+			ExtendableStringBuilder<512> SB;
+			for (size_t FailedPathIndex = 0; FailedPathIndex < Result.FailedRemovePaths.size(); FailedPathIndex++)
 			{
-				LOG_OUTPUT_WARN(LogOutput, "Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
-				CleanWipe = false;
+				SB << fmt::format("\n    '{}': ({}) {}",
+								  Result.FailedRemovePaths[FailedPathIndex].first,
+								  Result.FailedRemovePaths[FailedPathIndex].second.value(),
+								  Result.FailedRemovePaths[FailedPathIndex].second.message());
 			}
-
-			uint64_t NowMs = Timer.GetElapsedTimeMs();
-			if ((NowMs - LastUpdateTimeMs) >= LogOutput.GetProgressUpdateDelayMS())
-			{
-				LastUpdateTimeMs = NowMs;
-
-				uint64_t Deleted	  = DeletedItemCount.load();
-				uint64_t DeletedBytes = DeletedByteCount.load();
-				uint64_t Discovered	  = DiscoveredItemCount.load();
-				Progress.UpdateState({.Task		  = "Cleaning folder  ",
-									  .Details	  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
-									  .TotalCount = Discovered,
-									  .RemainingCount = Discovered - Deleted,
-									  .Status		  = BuildOpLogOutput::ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
-									 false);
-			}
-		}
-
-		Progress.Finish();
-		if (AbortFlag)
-		{
-			return false;
+			LOG_OUTPUT_WARN(LogOutput, "Clean failed to remove files from '{}': {}", Path, SB.ToView());
 		}
 
-		uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
 		if (ElapsedTimeMs >= 200 && !IsQuiet)
 		{
 			LOG_OUTPUT(LogOutput,
 					   "Wiped folder '{}' {} ({}) in {}",
 					   Path,
-					   DiscoveredItemCount.load(),
-					   NiceBytes(DeletedByteCount.load()),
+					   Result.FoundCount,
+					   NiceBytes(Result.DeletedByteCount),
 					   NiceTimeSpanMs(ElapsedTimeMs));
 		}
-		return CleanWipe;
-	}
 
-	void CopyFile(bool						   AllowFileClone,
-				  bool						   UseSparseFiles,
-				  const std::filesystem::path& SourceFilePath,
-				  const std::filesystem::path& TargetFilePath,
-				  uint64_t					   RawSize,
-				  std::atomic<uint64_t>&	   WriteCount,
-				  std::atomic<uint64_t>&	   WriteByteCount,
-				  std::atomic<uint64_t>&	   CloneCount,
-				  std::atomic<uint64_t>&	   CloneByteCount)
-	{
-		ZEN_TRACE_CPU("CopyFile");
-		if (AllowFileClone && TryCloneFile(SourceFilePath, TargetFilePath))
-		{
-			WriteCount += 1;
-			WriteByteCount += RawSize;
-			CloneCount += 1;
-			CloneByteCount += RawSize;
-		}
-		else
-		{
-			BasicFile TargetFile(TargetFilePath, BasicFile::Mode::kTruncate);
-			if (UseSparseFiles)
-			{
-				PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize);
-			}
-			uint64_t Offset = 0;
-			if (!ScanFile(SourceFilePath, 512u * 1024u, [&](const void* Data, size_t Size) {
-					TargetFile.Write(Data, Size, Offset);
-					Offset += Size;
-					WriteCount++;
-					WriteByteCount += Size;
-				}))
-			{
-				throw std::runtime_error(fmt::format("Failed to copy scavenged file '{}' to '{}'", SourceFilePath, TargetFilePath));
-			}
-		}
+		return Result.FailedRemovePaths.empty();
 	}
 
 	const std::vector<uint32_t> NonCompressableExtensions({HashStringDjb2(".mp4"sv),
@@ -567,7 +210,7 @@ namespace {
 		if (Ec)
 		{
 			throw std::runtime_error(
-				fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value()));
+				fmt::format("Failed opening temporary file '{}', reason: ({}) {}", Workload->TempFile.GetPath(), Ec.message(), Ec.value()));
 		}
 		std::vector<std::function<void()>> WorkItems = Storage.GetLargeBuildBlob(
 			BuildId,
@@ -700,71 +343,6 @@ namespace {
 
 }  // namespace
 
-class ReadFileCache
-{
-public:
-	// A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten
-	ReadFileCache(DiskStatistics&			   DiskStats,
-				  const std::filesystem::path& Path,
-				  const ChunkedFolderContent&  LocalContent,
-				  const ChunkedContentLookup&  LocalLookup,
-				  size_t					   MaxOpenFileCount)
-	: m_Path(Path)
-	, m_LocalContent(LocalContent)
-	, m_LocalLookup(LocalLookup)
-	, m_DiskStats(DiskStats)
-	{
-		m_OpenFiles.reserve(MaxOpenFileCount);
-	}
-	~ReadFileCache() { m_OpenFiles.clear(); }
-
-	CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size)
-	{
-		ZEN_TRACE_CPU("ReadFileCache::GetRange");
-
-		auto CacheIt =
-			std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { return Lhs.first == SequenceIndex; });
-		if (CacheIt != m_OpenFiles.end())
-		{
-			if (CacheIt != m_OpenFiles.begin())
-			{
-				auto CachedFile(std::move(CacheIt->second));
-				m_OpenFiles.erase(CacheIt);
-				m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile)));
-			}
-			CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
-			return Result;
-		}
-		const uint32_t				LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex];
-		const std::filesystem::path LocalFilePath  = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
-		if (Size == m_LocalContent.RawSizes[LocalPathIndex])
-		{
-			IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath);
-			return CompositeBuffer(SharedBuffer(Result));
-		}
-		if (m_OpenFiles.size() == m_OpenFiles.capacity())
-		{
-			m_OpenFiles.pop_back();
-		}
-		m_OpenFiles.insert(m_OpenFiles.begin(),
-						   std::make_pair(SequenceIndex,
-										  std::make_unique<BufferedOpenFile>(LocalFilePath,
-																			 m_DiskStats.OpenReadCount,
-																			 m_DiskStats.CurrentOpenFileCount,
-																			 m_DiskStats.ReadCount,
-																			 m_DiskStats.ReadByteCount)));
-		CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
-		return Result;
-	}
-
-private:
-	const std::filesystem::path											m_Path;
-	const ChunkedFolderContent&											m_LocalContent;
-	const ChunkedContentLookup&											m_LocalLookup;
-	std::vector<std::pair<uint32_t, std::unique_ptr<BufferedOpenFile>>> m_OpenFiles;
-	DiskStatistics&														m_DiskStats;
-};
-
 bool
 ReadStateObject(BuildOpLogOutput&				   LogOutput,
 				CbObjectView					   StateView,
@@ -1210,8 +788,10 @@ StreamDecompress(std::atomic<bool>&			  AbortFlag,
 	DecompressedTemp.CreateTemporary(TempChunkSequenceFileName.parent_path(), Ec);
 	if (Ec)
 	{
-		throw std::runtime_error(
-			fmt::format("Failed creating temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message()));
+		throw std::runtime_error(fmt::format("Failed creating temporary file for decompressing large blob {}, reason: ({}) {}",
+											 SequenceRawHash,
+											 Ec.value(),
+											 Ec.message()));
 	}
 	IoHash			 RawHash;
 	uint64_t		 RawSize;
@@ -1267,8 +847,10 @@ StreamDecompress(std::atomic<bool>&			  AbortFlag,
 	DecompressedTemp.MoveTemporaryIntoPlace(TempChunkSequenceFileName, Ec);
 	if (Ec)
 	{
-		throw std::runtime_error(
-			fmt::format("Failed moving temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message()));
+		throw std::runtime_error(fmt::format("Failed moving temporary file for decompressing large blob {}, reason: ({}) {}",
+											 SequenceRawHash,
+											 Ec.value(),
+											 Ec.message()));
 	}
 	//		WriteChunkStats.ChunkCountWritten++;
 }
@@ -2365,7 +1947,15 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 													FilteredWrittenBytesPerSecond.Stop();
 												}
 
-												TryRemoveFile(m_LogOutput, CompressedChunkPath);
+												std::error_code Ec = TryRemoveFile(CompressedChunkPath);
+												if (Ec)
+												{
+													LOG_OUTPUT_DEBUG(m_LogOutput,
+																	 "Failed removing file '{}', reason: ({}) {}",
+																	 CompressedChunkPath,
+																	 Ec.value(),
+																	 Ec.message());
+												}
 
 												std::vector<uint32_t> CompletedSequences =
 													CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
@@ -2650,7 +2240,15 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 											  throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash));
 										  }
 
-										  TryRemoveFile(m_LogOutput, BlockChunkPath);
+										  std::error_code Ec = TryRemoveFile(BlockChunkPath);
+										  if (Ec)
+										  {
+											  LOG_OUTPUT_DEBUG(m_LogOutput,
+															   "Failed removing file '{}', reason: ({}) {}",
+															   BlockChunkPath,
+															   Ec.value(),
+															   Ec.message());
+										  }
 
 										  WritePartsComplete++;
 
@@ -2829,9 +2427,14 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 													fmt::format("Partial block {} is malformed", BlockDescription.BlockHash));
 											}
 
-											if (!BlockChunkPath.empty())
+											std::error_code Ec = TryRemoveFile(BlockChunkPath);
+											if (Ec)
 											{
-												TryRemoveFile(m_LogOutput, BlockChunkPath);
+												LOG_OUTPUT_DEBUG(m_LogOutput,
+																 "Failed removing file '{}', reason: ({}) {}",
+																 BlockChunkPath,
+																 Ec.value(),
+																 Ec.message());
 											}
 
 											WritePartsComplete++;
@@ -3009,7 +2612,15 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 
 												if (!BlockChunkPath.empty())
 												{
-													TryRemoveFile(m_LogOutput, BlockChunkPath);
+													std::error_code Ec = TryRemoveFile(BlockChunkPath);
+													if (Ec)
+													{
+														LOG_OUTPUT_DEBUG(m_LogOutput,
+																		 "Failed removing file '{}', reason: ({}) {}",
+																		 BlockChunkPath,
+																		 Ec.value(),
+																		 Ec.message());
+													}
 												}
 
 												WritePartsComplete++;
@@ -3267,7 +2878,28 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 						const std::filesystem::path	 CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash);
 						ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath));
 						const std::filesystem::path LocalFilePath = (m_Path / LocalPath).make_preferred();
-						RenameFileWithRetry(m_LogOutput, LocalFilePath, CacheFilePath);
+
+						std::error_code Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath);
+						if (Ec)
+						{
+							LOG_OUTPUT_WARN(m_LogOutput,
+											"Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...",
+											LocalFilePath,
+											CacheFilePath,
+											Ec.value(),
+											Ec.message());
+							Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath);
+							if (Ec)
+							{
+								throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+														fmt::format("Failed to file from '{}' to '{}', reason: ({}) {}",
+																	LocalFilePath,
+																	CacheFilePath,
+																	Ec.value(),
+																	Ec.message()));
+							}
+						}
+
 						CachedCount++;
 						CachedByteCount += m_LocalContent.RawSizes[LocalPathIndex];
 					}
@@ -3317,7 +2949,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 		Stopwatch Timer;
 
 		// Clean target folder
-		if (!CleanDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_LogOutput, m_Options.IsQuiet, m_Path, m_Options.ExcludeFolders))
+		if (!CleanDirectory(m_LogOutput, m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.IsQuiet, m_Path, m_Options.ExcludeFolders))
 		{
 			LOG_OUTPUT_WARN(m_LogOutput, "Some files in {} could not be removed", m_Path);
 		}
@@ -3500,15 +3132,15 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 									std::atomic<uint64_t> WriteByteCount;
 									std::atomic<uint64_t> CloneCount;
 									std::atomic<uint64_t> CloneByteCount;
-									CopyFile(m_Options.AllowFileClone,
-											 m_Options.UseSparseFiles,
-											 SourceFilePath,
-											 FirstTargetFilePath,
-											 RawSize,
-											 WriteCount,
-											 WriteByteCount,
-											 CloneCount,
-											 CloneByteCount);
+									FastCopyFile(m_Options.AllowFileClone,
+												 m_Options.UseSparseFiles,
+												 SourceFilePath,
+												 FirstTargetFilePath,
+												 RawSize,
+												 WriteCount,
+												 WriteByteCount,
+												 CloneCount,
+												 CloneByteCount);
 									m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
 								}
 								else
@@ -3517,7 +3149,26 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 									const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash);
 									ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath));
 
-									RenameFileWithRetry(m_LogOutput, CacheFilePath, FirstTargetFilePath);
+									std::error_code Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
+									if (Ec)
+									{
+										LOG_OUTPUT_WARN(m_LogOutput,
+														"Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...",
+														CacheFilePath,
+														FirstTargetFilePath,
+														Ec.value(),
+														Ec.message());
+										Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
+										if (Ec)
+										{
+											throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+																	fmt::format("Failed to move file from '{}' to '{}', reason: ({}) {}",
+																				CacheFilePath,
+																				FirstTargetFilePath,
+																				Ec.value(),
+																				Ec.message()));
+										}
+									}
 
 									m_RebuildFolderStateStats.FinalizeTreeFilesMovedCount++;
 								}
@@ -3568,15 +3219,15 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 									std::atomic<uint64_t> WriteByteCount;
 									std::atomic<uint64_t> CloneCount;
 									std::atomic<uint64_t> CloneByteCount;
-									CopyFile(m_Options.AllowFileClone,
-											 m_Options.UseSparseFiles,
-											 FirstTargetFilePath,
-											 TargetFilePath,
-											 RawSize,
-											 WriteCount,
-											 WriteByteCount,
-											 CloneCount,
-											 CloneByteCount);
+									FastCopyFile(m_Options.AllowFileClone,
+												 m_Options.UseSparseFiles,
+												 FirstTargetFilePath,
+												 TargetFilePath,
+												 RawSize,
+												 WriteCount,
+												 WriteByteCount,
+												 CloneCount,
+												 CloneByteCount);
 									m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
 								}
 
@@ -3675,7 +3326,15 @@ BuildsOperationUpdateFolder::ScanCacheFolder(tsl::robin_map<IoHash, uint32_t, Io
 				}
 			}
 		}
-		TryRemoveFile(m_LogOutput, CacheDirContent.Files[Index]);
+		std::error_code Ec = TryRemoveFile(CacheDirContent.Files[Index]);
+		if (Ec)
+		{
+			LOG_OUTPUT_DEBUG(m_LogOutput,
+							 "Failed removing file '{}', reason: ({}) {}",
+							 CacheDirContent.Files[Index],
+							 Ec.value(),
+							 Ec.message());
+		}
 	}
 	m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
 }
@@ -3727,7 +3386,15 @@ BuildsOperationUpdateFolder::ScanTempBlocksFolder(tsl::robin_map<IoHash, uint32_
 				}
 			}
 		}
-		TryRemoveFile(m_LogOutput, BlockDirContent.Files[Index]);
+		std::error_code Ec = TryRemoveFile(BlockDirContent.Files[Index]);
+		if (Ec)
+		{
+			LOG_OUTPUT_DEBUG(m_LogOutput,
+							 "Failed removing file '{}', reason: ({}) {}",
+							 BlockDirContent.Files[Index],
+							 Ec.value(),
+							 Ec.message());
+		}
 	}
 
 	m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
@@ -4114,15 +3781,15 @@ BuildsOperationUpdateFolder::WriteScavengedSequenceToCache(const std::filesystem
 	const std::filesystem::path TempFilePath		  = GetTempChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
 
 	const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedPathIndex];
-	CopyFile(m_Options.AllowFileClone,
-			 m_Options.UseSparseFiles,
-			 ScavengedFilePath,
-			 TempFilePath,
-			 RawSize,
-			 m_DiskStats.WriteCount,
-			 m_DiskStats.WriteByteCount,
-			 m_DiskStats.CloneCount,
-			 m_DiskStats.CloneByteCount);
+	FastCopyFile(m_Options.AllowFileClone,
+				 m_Options.UseSparseFiles,
+				 ScavengedFilePath,
+				 TempFilePath,
+				 RawSize,
+				 m_DiskStats.WriteCount,
+				 m_DiskStats.WriteByteCount,
+				 m_DiskStats.CloneCount,
+				 m_DiskStats.CloneByteCount);
 
 	const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
 	RenameFile(TempFilePath, CacheFilePath);
@@ -4817,7 +4484,15 @@ BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(const std::filesystem::pa
 
 					if (!CompressedChunkPath.empty())
 					{
-						TryRemoveFile(m_LogOutput, CompressedChunkPath);
+						std::error_code Ec = TryRemoveFile(CompressedChunkPath);
+						if (Ec)
+						{
+							LOG_OUTPUT_DEBUG(m_LogOutput,
+											 "Failed removing file '{}', reason: ({}) {}",
+											 CompressedChunkPath,
+											 Ec.value(),
+											 Ec.message());
+						}
 					}
 
 					std::vector<uint32_t> CompletedSequences =
@@ -6567,7 +6242,14 @@ BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent&	Content,
 										   ChunkBlockDescription&		OutBlockDescription)
 {
 	ZEN_TRACE_CPU("GenerateBlock");
-	ReadFileCache OpenFileCache(m_DiskStats, m_Path, Content, Lookup, 4);
+	ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount,
+								m_DiskStats.CurrentOpenFileCount,
+								m_DiskStats.ReadCount,
+								m_DiskStats.ReadByteCount,
+								m_Path,
+								Content,
+								Lookup,
+								4);
 
 	std::vector<std::pair<IoHash, FetchChunkFunc>> BlockContent;
 	BlockContent.reserve(ChunksInBlock.size());
@@ -6601,7 +6283,14 @@ BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent&  Content,
 										  const std::vector<uint32_t>& ChunksInBlock)
 {
 	ZEN_TRACE_CPU("RebuildBlock");
-	ReadFileCache OpenFileCache(m_DiskStats, m_Path, Content, Lookup, 4);
+	ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount,
+								m_DiskStats.CurrentOpenFileCount,
+								m_DiskStats.ReadCount,
+								m_DiskStats.ReadByteCount,
+								m_Path,
+								Content,
+								Lookup,
+								4);
 
 	std::vector<SharedBuffer> ResultBuffers;
 	ResultBuffers.reserve(HeaderBuffer.GetSegments().size() + ChunksInBlock.size());
@@ -7126,8 +6815,10 @@ BuildsOperationUploadFolder::CompressChunk(const ChunkedFolderContent& Content,
 		CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec);
 		if (Ec)
 		{
-			throw std::runtime_error(
-				fmt::format("Failed creating temporary file for compressing blob {}. Reason: {}", ChunkHash, Ec.message()));
+			throw std::runtime_error(fmt::format("Failed creating temporary file for compressing blob {}, reason: ({}) {}",
+												 ChunkHash,
+												 Ec.value(),
+												 Ec.message()));
 		}
 
 		uint64_t StreamRawBytes		   = 0;
diff --git a/src/zenremotestore/filesystemutils.cpp b/src/zenremotestore/filesystemutils.cpp
new file mode 100644
index 000000000..20ab3faea
--- /dev/null
+++ b/src/zenremotestore/filesystemutils.cpp
@@ -0,0 +1,595 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "filesystemutils.h"
+
+#include <zenremotestore/chunking/chunkedcontent.h>
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/parallelwork.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+
+namespace zen {
+
+BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path,
+								   std::atomic<uint64_t>&	   OpenReadCount,
+								   std::atomic<uint64_t>&	   CurrentOpenFileCount,
+								   std::atomic<uint64_t>&	   ReadCount,
+								   std::atomic<uint64_t>&	   ReadByteCount)
+: m_Source(Path, BasicFile::Mode::kRead)
+, m_SourceSize(m_Source.FileSize())
+, m_OpenReadCount(OpenReadCount)
+, m_CurrentOpenFileCount(CurrentOpenFileCount)
+, m_ReadCount(ReadCount)
+, m_ReadByteCount(ReadByteCount)
+
+{
+	m_OpenReadCount++;
+	m_CurrentOpenFileCount++;
+}
+
+BufferedOpenFile::~BufferedOpenFile()
+{
+	m_CurrentOpenFileCount--;
+}
+
+CompositeBuffer
+BufferedOpenFile::GetRange(uint64_t Offset, uint64_t Size)
+{
+	ZEN_TRACE_CPU("BufferedOpenFile::GetRange");
+
+	ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache);
+	auto _ = MakeGuard([&]() { ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); });
+
+	ZEN_ASSERT((Offset + Size) <= m_SourceSize);
+	const uint64_t BlockIndexStart = Offset / BlockSize;
+	const uint64_t BlockIndexEnd   = (Offset + Size - 1) / BlockSize;
+
+	std::vector<SharedBuffer> BufferRanges;
+	BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1);
+
+	uint64_t ReadOffset = Offset;
+	for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++)
+	{
+		const uint64_t BlockStartOffset = BlockIndex * BlockSize;
+		if (m_CacheBlockIndex != BlockIndex)
+		{
+			uint64_t CacheSize = Min(BlockSize, m_SourceSize - BlockStartOffset);
+			ZEN_ASSERT(CacheSize > 0);
+			m_Cache = IoBuffer(CacheSize);
+			m_Source.Read(m_Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset);
+			m_ReadCount++;
+			m_ReadByteCount += CacheSize;
+			m_CacheBlockIndex = BlockIndex;
+		}
+
+		const uint64_t BytesRead = ReadOffset - Offset;
+		ZEN_ASSERT(BlockStartOffset <= ReadOffset);
+		const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset;
+		ZEN_ASSERT(OffsetIntoBlock < m_Cache.GetSize());
+		const uint64_t BlockBytes = Min(m_Cache.GetSize() - OffsetIntoBlock, Size - BytesRead);
+		BufferRanges.emplace_back(SharedBuffer(IoBuffer(m_Cache, OffsetIntoBlock, BlockBytes)));
+		ReadOffset += BlockBytes;
+	}
+	CompositeBuffer Result(std::move(BufferRanges));
+	ZEN_ASSERT(Result.GetSize() == Size);
+	return Result;
+}
+
+ReadFileCache::ReadFileCache(std::atomic<uint64_t>&		  OpenReadCount,
+							 std::atomic<uint64_t>&		  CurrentOpenFileCount,
+							 std::atomic<uint64_t>&		  ReadCount,
+							 std::atomic<uint64_t>&		  ReadByteCount,
+							 const std::filesystem::path& Path,
+							 const ChunkedFolderContent&  LocalContent,
+							 const ChunkedContentLookup&  LocalLookup,
+							 size_t						  MaxOpenFileCount)
+: m_Path(Path)
+, m_LocalContent(LocalContent)
+, m_LocalLookup(LocalLookup)
+, m_OpenReadCount(OpenReadCount)
+, m_CurrentOpenFileCount(CurrentOpenFileCount)
+, m_ReadCount(ReadCount)
+, m_ReadByteCount(ReadByteCount)
+{
+	m_OpenFiles.reserve(MaxOpenFileCount);
+}
+ReadFileCache::~ReadFileCache()
+{
+	m_OpenFiles.clear();
+}
+
+CompositeBuffer
+ReadFileCache::GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size)
+{
+	ZEN_TRACE_CPU("ReadFileCache::GetRange");
+
+	auto CacheIt =
+		std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { return Lhs.first == SequenceIndex; });
+	if (CacheIt != m_OpenFiles.end())
+	{
+		if (CacheIt != m_OpenFiles.begin())
+		{
+			auto CachedFile(std::move(CacheIt->second));
+			m_OpenFiles.erase(CacheIt);
+			m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile)));
+		}
+		CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
+		return Result;
+	}
+	const uint32_t				LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+	const std::filesystem::path LocalFilePath  = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
+	if (Size == m_LocalContent.RawSizes[LocalPathIndex])
+	{
+		IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath);
+		return CompositeBuffer(SharedBuffer(Result));
+	}
+	if (m_OpenFiles.size() == m_OpenFiles.capacity())
+	{
+		m_OpenFiles.pop_back();
+	}
+	m_OpenFiles.insert(
+		m_OpenFiles.begin(),
+		std::make_pair(
+			SequenceIndex,
+			std::make_unique<BufferedOpenFile>(LocalFilePath, m_OpenReadCount, m_CurrentOpenFileCount, m_ReadCount, m_ReadByteCount)));
+	CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
+	return Result;
+}
+
+uint32_t
+SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes)
+{
+#if ZEN_PLATFORM_WINDOWS
+	if (SourcePlatform == SourcePlatform::Windows)
+	{
+		SetFileAttributesToPath(FilePath, Attributes);
+		return Attributes;
+	}
+	else
+	{
+		uint32_t CurrentAttributes = GetFileAttributesFromPath(FilePath);
+		uint32_t NewAttributes	   = zen::MakeFileAttributeReadOnly(CurrentAttributes, zen::IsFileModeReadOnly(Attributes));
+		if (CurrentAttributes != NewAttributes)
+		{
+			SetFileAttributesToPath(FilePath, NewAttributes);
+		}
+		return NewAttributes;
+	}
+#endif	// ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	if (SourcePlatform != SourcePlatform::Windows)
+	{
+		zen::SetFileMode(FilePath, Attributes);
+		return Attributes;
+	}
+	else
+	{
+		uint32_t CurrentMode = zen::GetFileMode(FilePath);
+		uint32_t NewMode	 = zen::MakeFileModeReadOnly(CurrentMode, zen::IsFileAttributeReadOnly(Attributes));
+		if (CurrentMode != NewMode)
+		{
+			zen::SetFileMode(FilePath, NewMode);
+		}
+		return NewMode;
+	}
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+};
+
+uint32_t
+GetNativeFileAttributes(const std::filesystem::path FilePath)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return GetFileAttributesFromPath(FilePath);
+#endif	// ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	return GetFileMode(FilePath);
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+}
+
+bool
+IsFileWithRetry(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	bool			Result = IsFile(Path, Ec);
+	for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+	{
+		Sleep(100 + int(Retries * 50));
+		Ec.clear();
+		Result = IsFile(Path, Ec);
+	}
+	if (Ec)
+	{
+		throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+								fmt::format("Failed to check path '{}' is file, reason: ({}) {}", Path, Ec.value(), Ec.message()));
+	}
+	return Result;
+}
+
+bool
+SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
+{
+	std::error_code Ec;
+	bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
+	for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+	{
+		Sleep(100 + int(Retries * 50));
+		if (!IsFileWithRetry(Path))
+		{
+			return false;
+		}
+		Ec.clear();
+		Result = SetFileReadOnly(Path, ReadOnly, Ec);
+	}
+	if (Ec)
+	{
+		throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+								fmt::format("Failed {} read only flag for file '{}', reason: ({}) {}",
+											ReadOnly ? "setting" : "clearing",
+											Path,
+											Ec.value(),
+											Ec.message()));
+	}
+	return Result;
+}
+
+std::error_code
+RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
+{
+	std::error_code Ec;
+	RenameFile(SourcePath, TargetPath, Ec);
+	for (size_t Retries = 0; Ec && Retries < 5; Retries++)
+	{
+		ZEN_ASSERT_SLOW(IsFile(SourcePath));
+		Sleep(50 + int(Retries * 150));
+		Ec.clear();
+		RenameFile(SourcePath, TargetPath, Ec);
+	}
+	return Ec;
+}
+
+std::error_code
+TryRemoveFile(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	RemoveFile(Path, Ec);
+	if (Ec)
+	{
+		if (IsFile(Path, Ec))
+		{
+			Ec.clear();
+			RemoveFile(Path, Ec);
+			if (Ec)
+			{
+				return Ec;
+			}
+		}
+	}
+	return {};
+}
+
+void
+RemoveFileWithRetry(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	RemoveFile(Path, Ec);
+	for (size_t Retries = 0; Ec && Retries < 6; Retries++)
+	{
+		Sleep(100 + int(Retries * 50));
+		if (!IsFileWithRetry(Path))
+		{
+			return;
+		}
+		Ec.clear();
+		RemoveFile(Path, Ec);
+	}
+	if (Ec)
+	{
+		throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+								fmt::format("Failed removing file '{}', reason: ({}) {}", Path, Ec.value(), Ec.message()));
+	}
+}
+
+void
+FastCopyFile(bool						  AllowFileClone,
+			 bool						  UseSparseFiles,
+			 const std::filesystem::path& SourceFilePath,
+			 const std::filesystem::path& TargetFilePath,
+			 uint64_t					  RawSize,
+			 std::atomic<uint64_t>&		  WriteCount,
+			 std::atomic<uint64_t>&		  WriteByteCount,
+			 std::atomic<uint64_t>&		  CloneCount,
+			 std::atomic<uint64_t>&		  CloneByteCount)
+{
+	ZEN_TRACE_CPU("CopyFile");
+	if (AllowFileClone && TryCloneFile(SourceFilePath, TargetFilePath))
+	{
+		WriteCount += 1;
+		WriteByteCount += RawSize;
+		CloneCount += 1;
+		CloneByteCount += RawSize;
+	}
+	else
+	{
+		BasicFile TargetFile(TargetFilePath, BasicFile::Mode::kTruncate);
+		if (UseSparseFiles)
+		{
+			PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize);
+		}
+		uint64_t Offset = 0;
+		if (!ScanFile(SourceFilePath, 512u * 1024u, [&](const void* Data, size_t Size) {
+				TargetFile.Write(Data, Size, Offset);
+				Offset += Size;
+				WriteCount++;
+				WriteByteCount += Size;
+			}))
+		{
+			throw std::runtime_error(fmt::format("Failed to copy file '{}' to '{}'", SourceFilePath, TargetFilePath));
+		}
+	}
+}
+
+CleanDirectoryResult
+CleanDirectory(
+	WorkerThreadPool&			 IOWorkerPool,
+	std::atomic<bool>&			 AbortFlag,
+	std::atomic<bool>&			 PauseFlag,
+	const std::filesystem::path& Path,
+	std::span<const std::string> ExcludeDirectories,
+	std::function<void(const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted)>&&
+			 ProgressFunc,
+	uint32_t ProgressUpdateDelayMS)
+{
+	ZEN_TRACE_CPU("CleanDirectory");
+	Stopwatch Timer;
+
+	std::atomic<uint64_t> DiscoveredItemCount = 0;
+	std::atomic<uint64_t> DeletedItemCount	  = 0;
+	std::atomic<uint64_t> DeletedByteCount	  = 0;
+
+	CleanDirectoryResult Result;
+	RwLock				 ResultLock;
+	auto				 _ = MakeGuard([&]() {
+		Result.DeletedCount		= DeletedItemCount.load();
+		Result.DeletedByteCount = DeletedByteCount.load();
+		Result.FoundCount		= DiscoveredItemCount.load();
+	});
+
+	ParallelWork Work(AbortFlag,
+					  PauseFlag,
+					  ProgressFunc ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog);
+
+	struct AsyncVisitor : public GetDirectoryContentVisitor
+	{
+		AsyncVisitor(const std::filesystem::path& InPath,
+					 std::atomic<bool>&			  InAbortFlag,
+					 std::atomic<uint64_t>&		  InDiscoveredItemCount,
+					 std::atomic<uint64_t>&		  InDeletedItemCount,
+					 std::atomic<uint64_t>&		  InDeletedByteCount,
+					 std::span<const std::string> InExcludeDirectories,
+					 CleanDirectoryResult&		  InResult,
+					 RwLock&					  InResultLock)
+		: Path(InPath)
+		, AbortFlag(InAbortFlag)
+		, DiscoveredItemCount(InDiscoveredItemCount)
+		, DeletedItemCount(InDeletedItemCount)
+		, DeletedByteCount(InDeletedByteCount)
+		, ExcludeDirectories(InExcludeDirectories)
+		, Result(InResult)
+		, ResultLock(InResultLock)
+		{
+		}
+		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+		{
+			ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
+			if (!AbortFlag)
+			{
+				if (!Content.FileNames.empty())
+				{
+					DiscoveredItemCount += Content.FileNames.size();
+
+					const std::string RelativeRootString = RelativeRoot.generic_string();
+					bool			  RemoveContent		 = true;
+					for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+					{
+						if (RelativeRootString.starts_with(ExcludeDirectory))
+						{
+							if (RelativeRootString.length() > ExcludeDirectory.length())
+							{
+								const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
+								if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
+									MaybePathDelimiter == std::filesystem::path::preferred_separator)
+								{
+									RemoveContent = false;
+									break;
+								}
+							}
+							else
+							{
+								RemoveContent = false;
+								break;
+							}
+						}
+					}
+					if (RemoveContent)
+					{
+						ZEN_TRACE_CPU("DeleteFiles");
+						for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+						{
+							const std::filesystem::path& FileName = Content.FileNames[FileIndex];
+							const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
+
+							bool			IsRemoved = false;
+							std::error_code Ec;
+							(void)SetFileReadOnly(FilePath, false, Ec);
+							for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+							{
+								Sleep(100 + int(Retries * 50));
+								if (!IsFileWithRetry(FilePath))
+								{
+									IsRemoved = true;
+									break;
+								}
+								Ec.clear();
+								(void)SetFileReadOnly(FilePath, false, Ec);
+							}
+							if (!IsRemoved && !Ec)
+							{
+								(void)RemoveFile(FilePath, Ec);
+								for (size_t Retries = 0; Ec && Retries < 6; Retries++)
+								{
+									Sleep(100 + int(Retries * 50));
+									if (!IsFileWithRetry(FilePath))
+									{
+										IsRemoved = true;
+										return;
+									}
+									Ec.clear();
+									(void)RemoveFile(FilePath, Ec);
+								}
+							}
+							if (!IsRemoved && Ec)
+							{
+								RwLock::ExclusiveLockScope _(ResultLock);
+								Result.FailedRemovePaths.push_back(std::make_pair(FilePath, Ec));
+							}
+							else
+							{
+								DeletedItemCount++;
+								DeletedByteCount += Content.FileSizes[FileIndex];
+							}
+						}
+					}
+				}
+			}
+		}
+		const std::filesystem::path& Path;
+		std::atomic<bool>&			 AbortFlag;
+		std::atomic<uint64_t>&		 DiscoveredItemCount;
+		std::atomic<uint64_t>&		 DeletedItemCount;
+		std::atomic<uint64_t>&		 DeletedByteCount;
+		std::span<const std::string> ExcludeDirectories;
+		CleanDirectoryResult&		 Result;
+		RwLock&						 ResultLock;
+	} Visitor(Path, AbortFlag, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories, Result, ResultLock);
+
+	GetDirectoryContent(Path,
+						DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes,
+						Visitor,
+						IOWorkerPool,
+						Work.PendingWork());
+
+	DirectoryContent LocalDirectoryContent;
+	GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
+	DiscoveredItemCount += LocalDirectoryContent.Directories.size();
+	std::vector<std::filesystem::path> DirectoriesToDelete;
+	DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size());
+	for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
+	{
+		bool Leave = false;
+		for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+		{
+			if (LocalDirPath == (Path / ExcludeDirectory))
+			{
+				Leave = true;
+				break;
+			}
+		}
+		if (!Leave)
+		{
+			DirectoriesToDelete.emplace_back(std::move(LocalDirPath));
+			DiscoveredItemCount++;
+		}
+	}
+
+	uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+	if (ProgressFunc && ProgressUpdateDelayMS != 0)
+	{
+		Work.Wait(ProgressUpdateDelayMS, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) {
+			ZEN_UNUSED(PendingWork);
+			LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+			uint64_t	Deleted		 = DeletedItemCount.load();
+			uint64_t	DeletedBytes = DeletedByteCount.load();
+			uint64_t	Discovered	 = DiscoveredItemCount.load();
+			std::string Details		 = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes));
+			ProgressFunc(Details, Discovered, Discovered - Deleted, IsPaused, IsAborted);
+		});
+	}
+	else
+	{
+		Work.Wait();
+	}
+
+	{
+		ZEN_TRACE_CPU("DeleteDirs");
+		for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete)
+		{
+			if (AbortFlag)
+			{
+				break;
+			}
+			else
+			{
+				while (PauseFlag && !AbortFlag)
+				{
+					Sleep(2000);
+				}
+			}
+
+			{
+				std::error_code Ec;
+				zen::CleanDirectory(DirectoryToDelete, true, Ec);
+				if (Ec)
+				{
+					Sleep(200);
+					zen::CleanDirectory(DirectoryToDelete, true, Ec);
+				}
+
+				if (!Ec)
+				{
+					RemoveDir(Path, Ec);
+					for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+					{
+						Sleep(100 + int(Retries * 50));
+						if (!IsDir(Path))
+						{
+							break;
+						}
+						Ec.clear();
+						RemoveDir(Path, Ec);
+					}
+				}
+				if (Ec)
+				{
+					RwLock::ExclusiveLockScope __(ResultLock);
+					Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec));
+				}
+				else
+				{
+					DeletedItemCount++;
+				}
+			}
+
+			uint64_t NowMs = Timer.GetElapsedTimeMs();
+
+			if ((NowMs - LastUpdateTimeMs) >= ProgressUpdateDelayMS)
+			{
+				LastUpdateTimeMs = NowMs;
+
+				uint64_t	Deleted		 = DeletedItemCount.load();
+				uint64_t	DeletedBytes = DeletedByteCount.load();
+				uint64_t	Discovered	 = DiscoveredItemCount.load();
+				std::string Details		 = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes));
+				ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag);
+			}
+		}
+	}
+
+	return Result;
+}
+
+}  // namespace zen
diff --git a/src/zenremotestore/filesystemutils.h b/src/zenremotestore/filesystemutils.h
new file mode 100644
index 000000000..cfe6adc6c
--- /dev/null
+++ b/src/zenremotestore/filesystemutils.h
@@ -0,0 +1,114 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/basicfile.h>
+#include <zenremotestore/chunking/chunkedcontent.h>
+
+namespace zen {
+
+class CompositeBuffer;
+
+class BufferedOpenFile
+{
+public:
+	BufferedOpenFile(const std::filesystem::path Path,
+					 std::atomic<uint64_t>&		 OpenReadCount,
+					 std::atomic<uint64_t>&		 CurrentOpenFileCount,
+					 std::atomic<uint64_t>&		 ReadCount,
+					 std::atomic<uint64_t>&		 ReadByteCount);
+	~BufferedOpenFile();
+	BufferedOpenFile()						  = delete;
+	BufferedOpenFile(const BufferedOpenFile&) = delete;
+	BufferedOpenFile(BufferedOpenFile&&)	  = delete;
+	BufferedOpenFile& operator=(BufferedOpenFile&&) = delete;
+	BufferedOpenFile& operator=(const BufferedOpenFile&) = delete;
+
+	CompositeBuffer GetRange(uint64_t Offset, uint64_t Size);
+
+public:
+	void* Handle() { return m_Source.Handle(); }
+
+private:
+	const uint64_t BlockSize = 256u * 1024u;
+
+	BasicFile			   m_Source;
+	const uint64_t		   m_SourceSize;
+	std::atomic<uint64_t>& m_OpenReadCount;
+	std::atomic<uint64_t>& m_CurrentOpenFileCount;
+	std::atomic<uint64_t>& m_ReadCount;
+	std::atomic<uint64_t>& m_ReadByteCount;
+	uint64_t			   m_CacheBlockIndex = (uint64_t)-1;
+	IoBuffer			   m_Cache;
+};
+
+class ReadFileCache
+{
+public:
+	// A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten
+	ReadFileCache(std::atomic<uint64_t>&	   OpenReadCount,
+				  std::atomic<uint64_t>&	   CurrentOpenFileCount,
+				  std::atomic<uint64_t>&	   ReadCount,
+				  std::atomic<uint64_t>&	   ReadByteCount,
+				  const std::filesystem::path& Path,
+				  const ChunkedFolderContent&  LocalContent,
+				  const ChunkedContentLookup&  LocalLookup,
+				  size_t					   MaxOpenFileCount);
+	~ReadFileCache();
+
+	CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size);
+
+private:
+	const std::filesystem::path											m_Path;
+	const ChunkedFolderContent&											m_LocalContent;
+	const ChunkedContentLookup&											m_LocalLookup;
+	std::vector<std::pair<uint32_t, std::unique_ptr<BufferedOpenFile>>> m_OpenFiles;
+	std::atomic<uint64_t>&												m_OpenReadCount;
+	std::atomic<uint64_t>&												m_CurrentOpenFileCount;
+	std::atomic<uint64_t>&												m_ReadCount;
+	std::atomic<uint64_t>&												m_ReadByteCount;
+};
+
+uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes);
+
+uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath);
+
+bool IsFileWithRetry(const std::filesystem::path& Path);
+
+bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly);
+
+std::error_code RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath);
+
+std::error_code TryRemoveFile(const std::filesystem::path& Path);
+
+void RemoveFileWithRetry(const std::filesystem::path& Path);
+
+void FastCopyFile(bool						   AllowFileClone,
+				  bool						   UseSparseFiles,
+				  const std::filesystem::path& SourceFilePath,
+				  const std::filesystem::path& TargetFilePath,
+				  uint64_t					   RawSize,
+				  std::atomic<uint64_t>&	   WriteCount,
+				  std::atomic<uint64_t>&	   WriteByteCount,
+				  std::atomic<uint64_t>&	   CloneCount,
+				  std::atomic<uint64_t>&	   CloneByteCount);
+
+struct CleanDirectoryResult
+{
+	uint64_t													   FoundCount		= 0;
+	uint64_t													   DeletedCount		= 0;
+	uint64_t													   DeletedByteCount = 0;
+	std::vector<std::pair<std::filesystem::path, std::error_code>> FailedRemovePaths;
+};
+
+CleanDirectoryResult CleanDirectory(
+	WorkerThreadPool&			 IOWorkerPool,
+	std::atomic<bool>&			 AbortFlag,
+	std::atomic<bool>&			 PauseFlag,
+	const std::filesystem::path& Path,
+	std::span<const std::string> ExcludeDirectories,
+	std::function<void(const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted)>&&
+			 ProgressFunc,
+	uint32_t ProgressUpdateDelayMS);
+
+}  // namespace zen
diff --git a/src/zenutil/bufferedopenfile.cpp b/src/zenutil/bufferedopenfile.cpp
deleted file mode 100644
index 360011302..000000000
--- a/src/zenutil/bufferedopenfile.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include <zenutil/bufferedopenfile.h>
-
-#include <zencore/scopeguard.h>
-#include <zencore/trace.h>
-
-namespace zen {
-
-BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path,
-								   std::atomic<uint64_t>&	   OpenReadCount,
-								   std::atomic<uint64_t>&	   CurrentOpenFileCount,
-								   std::atomic<uint64_t>&	   ReadCount,
-								   std::atomic<uint64_t>&	   ReadByteCount)
-: m_Source(Path, BasicFile::Mode::kRead)
-, m_SourceSize(m_Source.FileSize())
-, m_OpenReadCount(OpenReadCount)
-, m_CurrentOpenFileCount(CurrentOpenFileCount)
-, m_ReadCount(ReadCount)
-, m_ReadByteCount(ReadByteCount)
-
-{
-	m_OpenReadCount++;
-	m_CurrentOpenFileCount++;
-}
-
-BufferedOpenFile::~BufferedOpenFile()
-{
-	m_CurrentOpenFileCount--;
-}
-
-CompositeBuffer
-BufferedOpenFile::GetRange(uint64_t Offset, uint64_t Size)
-{
-	ZEN_TRACE_CPU("BufferedOpenFile::GetRange");
-
-	ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache);
-	auto _ = MakeGuard([&]() { ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); });
-
-	ZEN_ASSERT((Offset + Size) <= m_SourceSize);
-	const uint64_t BlockIndexStart = Offset / BlockSize;
-	const uint64_t BlockIndexEnd   = (Offset + Size - 1) / BlockSize;
-
-	std::vector<SharedBuffer> BufferRanges;
-	BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1);
-
-	uint64_t ReadOffset = Offset;
-	for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++)
-	{
-		const uint64_t BlockStartOffset = BlockIndex * BlockSize;
-		if (m_CacheBlockIndex != BlockIndex)
-		{
-			uint64_t CacheSize = Min(BlockSize, m_SourceSize - BlockStartOffset);
-			ZEN_ASSERT(CacheSize > 0);
-			m_Cache = IoBuffer(CacheSize);
-			m_Source.Read(m_Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset);
-			m_ReadCount++;
-			m_ReadByteCount += CacheSize;
-			m_CacheBlockIndex = BlockIndex;
-		}
-
-		const uint64_t BytesRead = ReadOffset - Offset;
-		ZEN_ASSERT(BlockStartOffset <= ReadOffset);
-		const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset;
-		ZEN_ASSERT(OffsetIntoBlock < m_Cache.GetSize());
-		const uint64_t BlockBytes = Min(m_Cache.GetSize() - OffsetIntoBlock, Size - BytesRead);
-		BufferRanges.emplace_back(SharedBuffer(IoBuffer(m_Cache, OffsetIntoBlock, BlockBytes)));
-		ReadOffset += BlockBytes;
-	}
-	CompositeBuffer Result(std::move(BufferRanges));
-	ZEN_ASSERT(Result.GetSize() == Size);
-	return Result;
-}
-
-}  // namespace zen
diff --git a/src/zenutil/include/zenutil/bufferedopenfile.h b/src/zenutil/include/zenutil/bufferedopenfile.h
deleted file mode 100644
index 3af7c9991..000000000
--- a/src/zenutil/include/zenutil/bufferedopenfile.h
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/basicfile.h>
-
-namespace zen {
-
-class CompositeBuffer;
-
-class BufferedOpenFile
-{
-public:
-	BufferedOpenFile(const std::filesystem::path Path,
-					 std::atomic<uint64_t>&		 OpenReadCount,
-					 std::atomic<uint64_t>&		 CurrentOpenFileCount,
-					 std::atomic<uint64_t>&		 ReadCount,
-					 std::atomic<uint64_t>&		 ReadByteCount);
-	~BufferedOpenFile();
-	BufferedOpenFile()						  = delete;
-	BufferedOpenFile(const BufferedOpenFile&) = delete;
-	BufferedOpenFile(BufferedOpenFile&&)	  = delete;
-	BufferedOpenFile& operator=(BufferedOpenFile&&) = delete;
-	BufferedOpenFile& operator=(const BufferedOpenFile&) = delete;
-
-	const uint64_t	BlockSize = 256u * 1024u;
-	CompositeBuffer GetRange(uint64_t Offset, uint64_t Size);
-
-public:
-	void* Handle() { return m_Source.Handle(); }
-
-private:
-	BasicFile			   m_Source;
-	const uint64_t		   m_SourceSize;
-	std::atomic<uint64_t>& m_OpenReadCount;
-	std::atomic<uint64_t>& m_CurrentOpenFileCount;
-	std::atomic<uint64_t>& m_ReadCount;
-	std::atomic<uint64_t>& m_ReadByteCount;
-	uint64_t			   m_CacheBlockIndex = (uint64_t)-1;
-	IoBuffer			   m_Cache;
-};
-
-}  // namespace zen