added options to configure exclude folders and extensions to zen build commands (#706)

* added `--exclude-folders` to `zen upload`, `zen download` and `zen diff`
added `--exclude-extensions` to `zen upload` and `zen diff`
excluded folder names are now matched by folder name in subfolders in addition to root level folders
* allow multiple token separators

1 files changed, 218 insertions, 149 deletions

diff --git a/src/zenremotestore/filesystemutils.cpp b/src/zenremotestore/filesystemutils.cpp
index 8dff05c6b..fa1ce6f78 100644
--- a/src/zenremotestore/filesystemutils.cpp
+++ b/src/zenremotestore/filesystemutils.cpp
@@ -11,6 +11,11 @@
 #include <zencore/timer.h>
 #include <zencore/trace.h>
 
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#endif	// ZEN_WITH_TESTS
+
 namespace zen {
 
 BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path,
@@ -349,13 +354,14 @@ CleanDirectory(
 	std::atomic<uint64_t> DeletedItemCount	  = 0;
 	std::atomic<uint64_t> DeletedByteCount	  = 0;
 
-	CleanDirectoryResult Result;
-	RwLock				 ResultLock;
-	auto				 _ = MakeGuard([&]() {
-		Result.DeletedCount		= DeletedItemCount.load();
-		Result.DeletedByteCount = DeletedByteCount.load();
-		Result.FoundCount		= DiscoveredItemCount.load();
-	});
+	std::vector<std::filesystem::path> DirectoriesToDelete;
+	CleanDirectoryResult			   Result;
+	RwLock							   ResultLock;
+	auto							   _ = MakeGuard([&]() {
+		  Result.DeletedCount	  = DeletedItemCount.load();
+		  Result.DeletedByteCount = DeletedByteCount.load();
+		  Result.FoundCount		  = DiscoveredItemCount.load();
+	  });
 
 	ParallelWork Work(AbortFlag,
 					  PauseFlag,
@@ -363,119 +369,133 @@ CleanDirectory(
 
 	struct AsyncVisitor : public GetDirectoryContentVisitor
 	{
-		AsyncVisitor(const std::filesystem::path& InPath,
-					 std::atomic<bool>&			  InAbortFlag,
-					 std::atomic<uint64_t>&		  InDiscoveredItemCount,
-					 std::atomic<uint64_t>&		  InDeletedItemCount,
-					 std::atomic<uint64_t>&		  InDeletedByteCount,
-					 std::span<const std::string> InExcludeDirectories,
-					 CleanDirectoryResult&		  InResult,
-					 RwLock&					  InResultLock)
+		AsyncVisitor(const std::filesystem::path&		 InPath,
+					 std::atomic<bool>&					 InAbortFlag,
+					 std::atomic<uint64_t>&				 InDiscoveredItemCount,
+					 std::atomic<uint64_t>&				 InDeletedItemCount,
+					 std::atomic<uint64_t>&				 InDeletedByteCount,
+					 std::span<const std::string>		 InExcludeDirectories,
+					 std::vector<std::filesystem::path>& OutDirectoriesToDelete,
+					 CleanDirectoryResult&				 InResult,
+					 RwLock&							 InResultLock)
 		: Path(InPath)
 		, AbortFlag(InAbortFlag)
 		, DiscoveredItemCount(InDiscoveredItemCount)
 		, DeletedItemCount(InDeletedItemCount)
 		, DeletedByteCount(InDeletedByteCount)
 		, ExcludeDirectories(InExcludeDirectories)
+		, DirectoriesToDelete(OutDirectoriesToDelete)
 		, Result(InResult)
 		, ResultLock(InResultLock)
 		{
 		}
+
+		virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const override
+		{
+			ZEN_UNUSED(Parent);
+
+			if (AbortFlag)
+			{
+				return false;
+			}
+			const std::string DirectoryString = DirectoryName.string();
+			for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+			{
+				if (DirectoryString == ExcludeDirectory)
+				{
+					return false;
+				}
+			}
+			return true;
+		}
+
 		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
 		{
 			ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
 			if (!AbortFlag)
 			{
-				if (!Content.FileNames.empty())
+				DiscoveredItemCount += Content.FileNames.size();
+
+				ZEN_TRACE_CPU("DeleteFiles");
+				std::vector<std::pair<std::filesystem::path, std::error_code>> FailedRemovePaths;
+				for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
 				{
-					DiscoveredItemCount += Content.FileNames.size();
+					const std::filesystem::path& FileName = Content.FileNames[FileIndex];
+					const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
 
-					const std::string RelativeRootString = RelativeRoot.generic_string();
-					bool			  RemoveContent		 = true;
-					for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+					bool			IsRemoved = false;
+					std::error_code Ec;
+					(void)SetFileReadOnly(FilePath, false, Ec);
+					for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 					{
-						if (RelativeRootString.starts_with(ExcludeDirectory))
+						if (!IsFileWithRetry(FilePath))
 						{
-							if (RelativeRootString.length() > ExcludeDirectory.length())
-							{
-								const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
-								if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
-									MaybePathDelimiter == std::filesystem::path::preferred_separator)
-								{
-									RemoveContent = false;
-									break;
-								}
-							}
-							else
-							{
-								RemoveContent = false;
-								break;
-							}
+							IsRemoved = true;
+							Ec.clear();
+							break;
 						}
+						Sleep(100 + int(Retries * 50));
+						Ec.clear();
+						(void)SetFileReadOnly(FilePath, false, Ec);
 					}
-					if (RemoveContent)
+					if (!IsRemoved && !Ec)
 					{
-						ZEN_TRACE_CPU("DeleteFiles");
-						for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+						(void)RemoveFile(FilePath, Ec);
+						for (size_t Retries = 0; Ec && Retries < 6; Retries++)
 						{
-							const std::filesystem::path& FileName = Content.FileNames[FileIndex];
-							const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
-
-							bool			IsRemoved = false;
-							std::error_code Ec;
-							(void)SetFileReadOnly(FilePath, false, Ec);
-							for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+							if (!IsFileWithRetry(FilePath))
 							{
-								if (!IsFileWithRetry(FilePath))
-								{
-									IsRemoved = true;
-									Ec.clear();
-									break;
-								}
-								Sleep(100 + int(Retries * 50));
+								IsRemoved = true;
 								Ec.clear();
-								(void)SetFileReadOnly(FilePath, false, Ec);
-							}
-							if (!IsRemoved && !Ec)
-							{
-								(void)RemoveFile(FilePath, Ec);
-								for (size_t Retries = 0; Ec && Retries < 6; Retries++)
-								{
-									if (!IsFileWithRetry(FilePath))
-									{
-										IsRemoved = true;
-										Ec.clear();
-										return;
-									}
-									Sleep(100 + int(Retries * 50));
-									Ec.clear();
-									(void)RemoveFile(FilePath, Ec);
-								}
-							}
-							if (!IsRemoved && Ec)
-							{
-								RwLock::ExclusiveLockScope _(ResultLock);
-								Result.FailedRemovePaths.push_back(std::make_pair(FilePath, Ec));
-							}
-							else
-							{
-								DeletedItemCount++;
-								DeletedByteCount += Content.FileSizes[FileIndex];
+								break;
 							}
+							Sleep(100 + int(Retries * 50));
+							Ec.clear();
+							(void)RemoveFile(FilePath, Ec);
 						}
 					}
+					if (!IsRemoved && Ec)
+					{
+						FailedRemovePaths.push_back(std::make_pair(FilePath, Ec));
+					}
+					else
+					{
+						DeletedItemCount++;
+						DeletedByteCount += Content.FileSizes[FileIndex];
+					}
+				}
+
+				if (!FailedRemovePaths.empty())
+				{
+					RwLock::ExclusiveLockScope _(ResultLock);
+					FailedRemovePaths.insert(FailedRemovePaths.end(), FailedRemovePaths.begin(), FailedRemovePaths.end());
+				}
+				else if (!RelativeRoot.empty())
+				{
+					DiscoveredItemCount++;
+					RwLock::ExclusiveLockScope _(ResultLock);
+					DirectoriesToDelete.push_back(RelativeRoot);
 				}
 			}
 		}
-		const std::filesystem::path& Path;
-		std::atomic<bool>&			 AbortFlag;
-		std::atomic<uint64_t>&		 DiscoveredItemCount;
-		std::atomic<uint64_t>&		 DeletedItemCount;
-		std::atomic<uint64_t>&		 DeletedByteCount;
-		std::span<const std::string> ExcludeDirectories;
-		CleanDirectoryResult&		 Result;
-		RwLock&						 ResultLock;
-	} Visitor(Path, AbortFlag, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories, Result, ResultLock);
+		const std::filesystem::path&		Path;
+		std::atomic<bool>&					AbortFlag;
+		std::atomic<uint64_t>&				DiscoveredItemCount;
+		std::atomic<uint64_t>&				DeletedItemCount;
+		std::atomic<uint64_t>&				DeletedByteCount;
+		std::span<const std::string>		ExcludeDirectories;
+		std::vector<std::filesystem::path>& DirectoriesToDelete;
+		CleanDirectoryResult&				Result;
+		RwLock&								ResultLock;
+	} Visitor(Path,
+			  AbortFlag,
+			  DiscoveredItemCount,
+			  DeletedItemCount,
+			  DeletedByteCount,
+			  ExcludeDirectories,
+			  DirectoriesToDelete,
+			  Result,
+			  ResultLock);
 
 	GetDirectoryContent(Path,
 						DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes,
@@ -483,29 +503,6 @@ CleanDirectory(
 						IOWorkerPool,
 						Work.PendingWork());
 
-	DirectoryContent LocalDirectoryContent;
-	GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
-	DiscoveredItemCount += LocalDirectoryContent.Directories.size();
-	std::vector<std::filesystem::path> DirectoriesToDelete;
-	DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size());
-	for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
-	{
-		bool Leave = false;
-		for (const std::string_view ExcludeDirectory : ExcludeDirectories)
-		{
-			if (LocalDirPath == (Path / ExcludeDirectory))
-			{
-				Leave = true;
-				break;
-			}
-		}
-		if (!Leave)
-		{
-			DirectoriesToDelete.emplace_back(std::move(LocalDirPath));
-			DiscoveredItemCount++;
-		}
-	}
-
 	uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
 
 	if (ProgressFunc && ProgressUpdateDelayMS != 0)
@@ -528,6 +525,15 @@ CleanDirectory(
 
 	{
 		ZEN_TRACE_CPU("DeleteDirs");
+
+		std::sort(DirectoriesToDelete.begin(),
+				  DirectoriesToDelete.end(),
+				  [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) {
+					  auto DistanceLhs = std::distance(Lhs.begin(), Lhs.end());
+					  auto DistanceRhs = std::distance(Rhs.begin(), Rhs.end());
+					  return DistanceLhs > DistanceRhs;
+				  });
+
 		for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete)
 		{
 			if (AbortFlag)
@@ -542,53 +548,48 @@ CleanDirectory(
 				}
 			}
 
-			{
-				std::error_code Ec;
-				zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec);
-				if (Ec)
-				{
-					Sleep(200);
-					Ec.clear();
-					zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec);
-				}
+			const std::filesystem::path FullPath = Path / DirectoryToDelete;
 
-				if (!Ec)
+			std::error_code Ec;
+			RemoveDir(FullPath, Ec);
+			if (Ec)
+			{
+				for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 				{
-					RemoveDir(DirectoryToDelete, Ec);
-					for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+					if (!IsDir(FullPath))
 					{
-						if (!IsDir(DirectoryToDelete))
-						{
-							Ec.clear();
-							break;
-						}
-						Sleep(100 + int(Retries * 50));
 						Ec.clear();
-						RemoveDir(DirectoryToDelete, Ec);
+						break;
 					}
-				}
-				if (Ec)
-				{
-					RwLock::ExclusiveLockScope __(ResultLock);
-					Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec));
-				}
-				else
-				{
-					DeletedItemCount++;
+					Sleep(100 + int(Retries * 50));
+					Ec.clear();
+					RemoveDir(FullPath, Ec);
 				}
 			}
+			if (Ec)
+			{
+				RwLock::ExclusiveLockScope __(ResultLock);
+				Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec));
+			}
+			else
+			{
+				DeletedItemCount++;
+			}
 
-			uint64_t NowMs = Timer.GetElapsedTimeMs();
-
-			if ((NowMs - LastUpdateTimeMs) >= ProgressUpdateDelayMS)
+			if (ProgressFunc)
 			{
-				LastUpdateTimeMs = NowMs;
+				uint64_t NowMs = Timer.GetElapsedTimeMs();
+
+				if ((NowMs - LastUpdateTimeMs) > 0)
+				{
+					LastUpdateTimeMs = NowMs;
 
-				uint64_t	Deleted		 = DeletedItemCount.load();
-				uint64_t	DeletedBytes = DeletedByteCount.load();
-				uint64_t	Discovered	 = DiscoveredItemCount.load();
-				std::string Details		 = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes));
-				ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag);
+					uint64_t	Deleted		 = DeletedItemCount.load();
+					uint64_t	DeletedBytes = DeletedByteCount.load();
+					uint64_t	Discovered	 = DiscoveredItemCount.load();
+					std::string Details		 = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes));
+					ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag);
+				}
 			}
 		}
 	}
@@ -625,4 +626,72 @@ CleanAndRemoveDirectory(WorkerThreadPool&			 WorkerPool,
 	return false;
 }
 
+#if ZEN_WITH_TESTS
+
+void
+filesystemutils_forcelink()
+{
+}
+
+namespace {
+	void GenerateFile(const std::filesystem::path& Path) { BasicFile _(Path, BasicFile::Mode::kTruncate); }
+}  // namespace
+
+TEST_CASE("filesystemutils.CleanDirectory")
+{
+	ScopedTemporaryDirectory TmpDir;
+
+	CreateDirectories(TmpDir.Path() / ".keepme");
+	GenerateFile(TmpDir.Path() / ".keepme" / "keep");
+	GenerateFile(TmpDir.Path() / "deleteme1");
+	GenerateFile(TmpDir.Path() / "deleteme2");
+	GenerateFile(TmpDir.Path() / "deleteme3");
+	CreateDirectories(TmpDir.Path() / ".keepmenot");
+	CreateDirectories(TmpDir.Path() / "no.keepme");
+
+	CreateDirectories(TmpDir.Path() / "DeleteMe");
+	GenerateFile(TmpDir.Path() / "DeleteMe" / "delete1");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete1");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete3");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe" / ".keepme");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete3");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme");
+
+	WorkerThreadPool  Pool(4);
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+
+	CleanDirectory(Pool, AbortFlag, PauseFlag, TmpDir.Path(), std::vector<std::string>{".keepme"}, {}, 0);
+
+	CHECK(IsDir(TmpDir.Path() / ".keepme"));
+	CHECK(IsFile(TmpDir.Path() / ".keepme" / "keep"));
+	CHECK(!IsFile(TmpDir.Path() / "deleteme1"));
+	CHECK(!IsFile(TmpDir.Path() / "deleteme2"));
+	CHECK(!IsFile(TmpDir.Path() / "deleteme3"));
+	CHECK(!IsFile(TmpDir.Path() / ".keepmenot"));
+	CHECK(!IsFile(TmpDir.Path() / "no.keepme"));
+
+	CHECK(!IsDir(TmpDir.Path() / "DeleteMe"));
+	CHECK(!IsDir(TmpDir.Path() / "DeleteMe2"));
+
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe" / ".keepme"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept"));
+	CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1"));
+	CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2"));
+	CHECK(!IsFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme"));
+}
+
+#endif
+
 }  // namespace zen