fix clean directory and make them use effective threading where appropriate (#625)v5.7.8-pre5v5.7.8-pre3v5.7.8-pre2

fix retry logic so it does not immediately sleep if file does not exist
make sure we don't try to delete target folder files if we have already wiped it

1 files changed, 119 insertions, 0 deletions

diff --git a/src/zenremotestore/include/zenremotestore/filesystemutils.h b/src/zenremotestore/include/zenremotestore/filesystemutils.h
new file mode 100644
index 000000000..a6c88e5cb
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/filesystemutils.h
@@ -0,0 +1,119 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/basicfile.h>
+#include <zenremotestore/chunking/chunkedcontent.h>
+
+namespace zen {
+
+class CompositeBuffer;
+
+class BufferedOpenFile
+{
+public:
+	BufferedOpenFile(const std::filesystem::path Path,
+					 std::atomic<uint64_t>&		 OpenReadCount,
+					 std::atomic<uint64_t>&		 CurrentOpenFileCount,
+					 std::atomic<uint64_t>&		 ReadCount,
+					 std::atomic<uint64_t>&		 ReadByteCount);
+	~BufferedOpenFile();
+	BufferedOpenFile()						  = delete;
+	BufferedOpenFile(const BufferedOpenFile&) = delete;
+	BufferedOpenFile(BufferedOpenFile&&)	  = delete;
+	BufferedOpenFile& operator=(BufferedOpenFile&&) = delete;
+	BufferedOpenFile& operator=(const BufferedOpenFile&) = delete;
+
+	CompositeBuffer GetRange(uint64_t Offset, uint64_t Size);
+
+public:
+	void* Handle() { return m_Source.Handle(); }
+
+private:
+	const uint64_t BlockSize = 256u * 1024u;
+
+	BasicFile			   m_Source;
+	const uint64_t		   m_SourceSize;
+	std::atomic<uint64_t>& m_OpenReadCount;
+	std::atomic<uint64_t>& m_CurrentOpenFileCount;
+	std::atomic<uint64_t>& m_ReadCount;
+	std::atomic<uint64_t>& m_ReadByteCount;
+	uint64_t			   m_CacheBlockIndex = (uint64_t)-1;
+	IoBuffer			   m_Cache;
+};
+
+class ReadFileCache
+{
+public:
+	// A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten
+	ReadFileCache(std::atomic<uint64_t>&	   OpenReadCount,
+				  std::atomic<uint64_t>&	   CurrentOpenFileCount,
+				  std::atomic<uint64_t>&	   ReadCount,
+				  std::atomic<uint64_t>&	   ReadByteCount,
+				  const std::filesystem::path& Path,
+				  const ChunkedFolderContent&  LocalContent,
+				  const ChunkedContentLookup&  LocalLookup,
+				  size_t					   MaxOpenFileCount);
+	~ReadFileCache();
+
+	CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size);
+
+private:
+	const std::filesystem::path											m_Path;
+	const ChunkedFolderContent&											m_LocalContent;
+	const ChunkedContentLookup&											m_LocalLookup;
+	std::vector<std::pair<uint32_t, std::unique_ptr<BufferedOpenFile>>> m_OpenFiles;
+	std::atomic<uint64_t>&												m_OpenReadCount;
+	std::atomic<uint64_t>&												m_CurrentOpenFileCount;
+	std::atomic<uint64_t>&												m_ReadCount;
+	std::atomic<uint64_t>&												m_ReadByteCount;
+};
+
+uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes);
+
+uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath);
+
+bool IsFileWithRetry(const std::filesystem::path& Path);
+
+bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly);
+
+std::error_code RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath);
+
+std::error_code TryRemoveFile(const std::filesystem::path& Path);
+
+void RemoveFileWithRetry(const std::filesystem::path& Path);
+
+void FastCopyFile(bool						   AllowFileClone,
+				  bool						   UseSparseFiles,
+				  const std::filesystem::path& SourceFilePath,
+				  const std::filesystem::path& TargetFilePath,
+				  uint64_t					   RawSize,
+				  std::atomic<uint64_t>&	   WriteCount,
+				  std::atomic<uint64_t>&	   WriteByteCount,
+				  std::atomic<uint64_t>&	   CloneCount,
+				  std::atomic<uint64_t>&	   CloneByteCount);
+
+struct CleanDirectoryResult
+{
+	uint64_t													   FoundCount		= 0;
+	uint64_t													   DeletedCount		= 0;
+	uint64_t													   DeletedByteCount = 0;
+	std::vector<std::pair<std::filesystem::path, std::error_code>> FailedRemovePaths;
+};
+
+CleanDirectoryResult CleanDirectory(
+	WorkerThreadPool&			 IOWorkerPool,
+	std::atomic<bool>&			 AbortFlag,
+	std::atomic<bool>&			 PauseFlag,
+	const std::filesystem::path& Path,
+	std::span<const std::string> ExcludeDirectories,
+	std::function<void(const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted)>&&
+			 ProgressFunc,
+	uint32_t ProgressUpdateDelayMS);
+
+bool CleanAndRemoveDirectory(WorkerThreadPool&			  WorkerPool,
+							 std::atomic<bool>&			  AbortFlag,
+							 std::atomic<bool>&			  PauseFlag,
+							 const std::filesystem::path& Directory);
+
+}  // namespace zen