added options to configure exclude folders and extensions to zen build commands (#706)

* added `--exclude-folders` to `zen upload`, `zen download` and `zen diff`
added `--exclude-extensions` to `zen upload` and `zen diff`
excluded folder names are now matched by folder name in subfolders in addition to root level folders
* allow multiple token separators

10 files changed, 468 insertions, 223 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 25f66e0ee..99f29376e 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -427,22 +427,24 @@ namespace {
 					NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000));
 	}
 
-	void UploadFolder(OperationLogOutput&		   Output,
-					  TransferThreadWorkers&	   Workers,
-					  StorageInstance&			   Storage,
-					  const Oid&				   BuildId,
-					  const Oid&				   BuildPartId,
-					  const std::string_view	   BuildPartName,
-					  const std::filesystem::path& Path,
-					  const std::filesystem::path& TempDir,
-					  const std::filesystem::path& ManifestPath,
-					  const uint64_t			   FindBlockMaxCount,
-					  const uint8_t				   BlockReuseMinPercentLimit,
-					  bool						   AllowMultiparts,
-					  const CbObject&			   MetaData,
-					  bool						   CreateBuild,
-					  bool						   IgnoreExistingBlocks,
-					  bool						   UploadToZenCache)
+	void UploadFolder(OperationLogOutput&			  Output,
+					  TransferThreadWorkers&		  Workers,
+					  StorageInstance&				  Storage,
+					  const Oid&					  BuildId,
+					  const Oid&					  BuildPartId,
+					  const std::string_view		  BuildPartName,
+					  const std::filesystem::path&	  Path,
+					  const std::filesystem::path&	  TempDir,
+					  const std::filesystem::path&	  ManifestPath,
+					  const uint64_t				  FindBlockMaxCount,
+					  const uint8_t					  BlockReuseMinPercentLimit,
+					  bool							  AllowMultiparts,
+					  const CbObject&				  MetaData,
+					  bool							  CreateBuild,
+					  bool							  IgnoreExistingBlocks,
+					  bool							  UploadToZenCache,
+					  const std::vector<std::string>& ExcludeFolders,
+					  const std::vector<std::string>& ExcludeExtensions)
 	{
 		ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder");
 		{
@@ -470,8 +472,8 @@ namespace {
 													 .AllowMultiparts			= AllowMultiparts,
 													 .IgnoreExistingBlocks		= IgnoreExistingBlocks,
 													 .TempDir					= TempDir,
-													 .ExcludeFolders			= DefaultExcludeFolders,
-													 .ExcludeExtensions			= DefaultExcludeExtensions,
+													 .ExcludeFolders			= ExcludeFolders,
+													 .ExcludeExtensions			= ExcludeExtensions,
 													 .ZenExcludeManifestName	= ZenExcludeManifestName,
 													 .NonCompressableExtensions = DefaultSplitOnlyExtensions,
 													 .PopulateCache				= UploadToZenCache});
@@ -682,12 +684,13 @@ namespace {
 		uint64_t			  VerifyElapsedWallTimeUs = 0;
 	};
 
-	void VerifyFolder(TransferThreadWorkers&	   Workers,
-					  const ChunkedFolderContent&  Content,
-					  const ChunkedContentLookup&  Lookup,
-					  const std::filesystem::path& Path,
-					  bool						   VerifyFileHash,
-					  VerifyFolderStatistics&	   VerifyFolderStats)
+	void VerifyFolder(TransferThreadWorkers&		  Workers,
+					  const ChunkedFolderContent&	  Content,
+					  const ChunkedContentLookup&	  Lookup,
+					  const std::filesystem::path&	  Path,
+					  const std::vector<std::string>& ExcludeFolders,
+					  bool							  VerifyFileHash,
+					  VerifyFolderStatistics&		  VerifyFolderStats)
 	{
 		ZEN_TRACE_CPU("VerifyFolder");
 
@@ -704,7 +707,7 @@ namespace {
 		RwLock					 ErrorLock;
 		std::vector<std::string> Errors;
 
-		auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool {
+		auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool {
 			for (const std::string& ExcludeFolder : ExcludeFolders)
 			{
 				if (RelativePath.starts_with(ExcludeFolder))
@@ -1593,6 +1596,7 @@ namespace {
 		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
 		bool					 PopulateCache				= true;
 		bool					 AppendNewContent			= false;
+		std::vector<std::string> ExcludeFolders;
 	};
 
 	void DownloadFolder(OperationLogOutput&					 Output,
@@ -1834,8 +1838,7 @@ namespace {
 					.EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
 					.EnableTargetFolderScavenging	= Options.EnableTargetFolderScavenging || Options.AppendNewContent,
 					.ValidateCompletedSequences		= Options.PostDownloadVerify,
-					.ExcludeFolders					= DefaultExcludeFolders,
-					.ExcludeExtensions				= DefaultExcludeExtensions,
+					.ExcludeFolders					= Options.ExcludeFolders,
 					.MaximumInMemoryPayloadSize		= Options.MaximumInMemoryPayloadSize,
 					.PopulateCache					= Options.PopulateCache});
 			{
@@ -1861,7 +1864,13 @@ namespace {
 
 					ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount);
 
-					VerifyFolder(Workers, RemoteContent, RemoteLookup, Path, Options.PostDownloadVerify, VerifyFolderStats);
+					VerifyFolder(Workers,
+								 RemoteContent,
+								 RemoteLookup,
+								 Path,
+								 Options.ExcludeFolders,
+								 Options.PostDownloadVerify,
+								 VerifyFolderStats);
 
 					Stopwatch WriteStateTimer;
 					CbObject  StateObject = CreateBuildSaveStateObject(LocalState);
@@ -2077,10 +2086,12 @@ namespace {
 		}
 	}
 
-	void DiffFolders(TransferThreadWorkers&		  Workers,
-					 const std::filesystem::path& BasePath,
-					 const std::filesystem::path& ComparePath,
-					 bool						  OnlyChunked)
+	void DiffFolders(TransferThreadWorkers&			 Workers,
+					 const std::filesystem::path&	 BasePath,
+					 const std::filesystem::path&	 ComparePath,
+					 bool							 OnlyChunked,
+					 const std::vector<std::string>& InExcludeFolders,
+					 const std::vector<std::string>& InExcludeExtensions)
 	{
 		ZEN_TRACE_CPU("DiffFolders");
 
@@ -2104,7 +2115,7 @@ namespace {
 		{
 			StandardChunkingControllerSettings	ChunkingSettings;
 			std::unique_ptr<ChunkingController> ChunkController	  = CreateStandardChunkingController(ChunkingSettings);
-			std::vector<std::string>			ExcludeExtensions = DefaultExcludeExtensions;
+			std::vector<std::string>			ExcludeExtensions = InExcludeExtensions;
 			if (OnlyChunked)
 			{
 				ExcludeExtensions.insert(ExcludeExtensions.end(),
@@ -2115,7 +2126,7 @@ namespace {
 										 ChunkingSettings.SplitAndCompressExtensions.end());
 			}
 
-			auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool {
+			auto IsAcceptedFolder = [ExcludeFolders = InExcludeFolders](const std::string_view& RelativePath) -> bool {
 				for (const std::string& ExcludeFolder : ExcludeFolders)
 				{
 					if (RelativePath.starts_with(ExcludeFolder))
@@ -2404,6 +2415,25 @@ BuildsCommand::BuildsCommand()
 					   "<excludewildcard>");
 	};
 
+	auto AddExcludeFolderOption = [this](cxxopts::Options& Ops) {
+		Ops.add_option("",
+					   "",
+					   "exclude-folders",
+					   "Names of folders to exclude, separated by ;",
+					   cxxopts::value(m_ExcludeFolders),
+					   "<excludefolders>");
+	};
+
+	auto AddExcludeExtensionsOption = [this](cxxopts::Options& Ops) {
+		Ops.add_option("",
+					   "",
+					   "exclude-extensions",
+					   "Extensions to exclude, separated by ;"
+					   "include filter",
+					   cxxopts::value(m_ExcludeExtensions),
+					   "<excludeextensions>");
+	};
+
 	auto AddMultipartOptions = [this](cxxopts::Options& Ops) {
 		Ops.add_option("",
 					   "",
@@ -2521,6 +2551,8 @@ BuildsCommand::BuildsCommand()
 	AddCacheOptions(m_UploadOptions);
 	AddWorkerOptions(m_UploadOptions);
 	AddZenFolderOptions(m_UploadOptions);
+	AddExcludeFolderOption(m_UploadOptions);
+	AddExcludeExtensionsOption(m_UploadOptions);
 	m_UploadOptions.add_options()("h,help", "Print help");
 	m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
 	m_UploadOptions.add_option("",
@@ -2599,6 +2631,7 @@ BuildsCommand::BuildsCommand()
 	AddWorkerOptions(m_DownloadOptions);
 	AddWildcardOptions(m_DownloadOptions);
 	AddAppendNewContentOptions(m_DownloadOptions);
+	AddExcludeFolderOption(m_DownloadOptions);
 
 	m_DownloadOptions.add_option("cache",
 								 "",
@@ -2697,6 +2730,8 @@ BuildsCommand::BuildsCommand()
 	// diff
 	AddOutputOptions(m_DiffOptions);
 	AddWorkerOptions(m_DiffOptions);
+	AddExcludeFolderOption(m_DiffOptions);
+	AddExcludeExtensionsOption(m_DiffOptions);
 	m_DiffOptions.add_options()("h,help", "Print help");
 	m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
 	m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>");
@@ -3188,6 +3223,28 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		OutExcludeWildcards = SplitWildcard(m_ExcludeWildcard);
 	};
 
+	auto ParseExcludeFolderAndExtension = [&](std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) {
+		auto SplitExclusion = [](const std::string_view Input) -> std::vector<std::string> {
+			std::vector<std::string> Exclusions;
+			ForEachStrTok(Input, ";,", [&Exclusions](std::string_view Exclusion) {
+				if (!Exclusion.empty())
+				{
+					std::string CleanExclusion(ToLower(Exclusion));
+					if (CleanExclusion.length() > 2 && CleanExclusion.front() == '"' && CleanExclusion.back() == '"')
+					{
+						CleanExclusion = CleanExclusion.substr(1, CleanExclusion.length() - 2);
+					}
+					Exclusions.emplace_back(std::move(CleanExclusion));
+				}
+				return true;
+			});
+			return Exclusions;
+		};
+
+		OutExcludeFolders	 = SplitExclusion(m_ExcludeFolders);
+		OutExcludeExtensions = SplitExclusion(m_ExcludeExtensions);
+	};
+
 	auto ParseDiffPath = [&]() {
 		if (m_DiffPath.empty())
 		{
@@ -3600,6 +3657,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 		const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath);
 
+		std::vector<std::string> ExcludeFolders	   = DefaultExcludeFolders;
+		std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
+		ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+
 		UploadFolder(*Output,
 					 Workers,
 					 Storage,
@@ -3615,7 +3676,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					 MetaData,
 					 m_CreateBuild,
 					 m_Clean,
-					 m_UploadToZenCache);
+					 m_UploadToZenCache,
+					 ExcludeFolders,
+					 ExcludeExtensions);
 
 		if (!AbortFlag)
 		{
@@ -3737,6 +3800,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			throw OptionParseException("'--append' conflicts with '--clean'", SubOption->help());
 		}
 
+		std::vector<std::string> ExcludeFolders	   = DefaultExcludeFolders;
+		std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
+		ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+
 		DownloadFolder(*Output,
 					   Workers,
 					   Storage,
@@ -3759,7 +3826,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .ExcludeWildcards			   = ExcludeWildcards,
 									   .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory),
 									   .PopulateCache			   = m_UploadToZenCache,
-									   .AppendNewContent		   = m_AppendNewContent});
+									   .AppendNewContent		   = m_AppendNewContent,
+									   .ExcludeFolders			   = ExcludeFolders});
 
 		if (AbortFlag)
 		{
@@ -3825,7 +3893,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		ParsePath();
 		ParseDiffPath();
 
-		DiffFolders(Workers, m_Path, m_DiffPath, m_OnlyChunked);
+		std::vector<std::string> ExcludeFolders	   = DefaultExcludeFolders;
+		std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
+		ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+
+		DiffFolders(Workers, m_Path, m_DiffPath, m_OnlyChunked, ExcludeFolders, ExcludeExtensions);
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Diff folders aborted");
@@ -4245,7 +4317,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					 MetaData,
 					 true,
 					 false,
-					 m_UploadToZenCache);
+					 m_UploadToZenCache,
+					 DefaultExcludeFolders,
+					 DefaultExcludeExtensions);
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Upload build)");
@@ -4542,7 +4616,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					 MetaData2,
 					 true,
 					 false,
-					 m_UploadToZenCache);
+					 m_UploadToZenCache,
+					 DefaultExcludeFolders,
+					 DefaultExcludeExtensions);
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Upload scrambled)");
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 081a3a460..4f6209f93 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -90,6 +90,12 @@ private:
 
 	std::filesystem::path m_Path;
 
+	std::string m_IncludeWildcard;
+	std::string m_ExcludeWildcard;
+
+	std::string m_ExcludeFolders;
+	std::string m_ExcludeExtensions;
+
 	cxxopts::Options m_UploadOptions{"upload", "Upload a folder"};
 	uint64_t		 m_FindBlockMaxCount = 10000;
 	bool			 m_PostUploadVerify	 = false;
@@ -101,8 +107,6 @@ private:
 	bool					 m_EnableScavenging	  = true;
 
 	cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"};
-	std::string		 m_IncludeWildcard;
-	std::string		 m_ExcludeWildcard;
 
 	cxxopts::Options	  m_DiffOptions{"diff", "Compare two local folders"};
 	std::filesystem::path m_DiffPath;
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 7f341818b..4da412c17 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -2669,36 +2669,41 @@ GetDirectoryContent(const std::filesystem::path& RootDir,
 			}
 			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive))
 			{
-				PendingWorkCount.AddCount(1);
-				try
-				{
-					WorkerPool.ScheduleWork(
-						[WorkerPool		  = &WorkerPool,
-						 PendingWorkCount = &PendingWorkCount,
-						 Visitor		  = Visitor,
-						 Flags			  = Flags,
-						 Path			  = std::move(Path),
-						 RelativeRoot	  = RelativeRoot / DirectoryName]() {
-							ZEN_ASSERT(Visitor);
-							auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
-							try
-							{
-								MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
-								FileSystemTraversal	 Traversal;
-								Traversal.TraverseFileSystem(Path, SubVisitor);
-								Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
-							}
-							catch (const std::exception& Ex)
-							{
-								ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'", Path / RelativeRoot, Ex.what());
-							}
-						},
-						WorkerThreadPool::EMode::DisableBacklog);
-				}
-				catch (const std::exception& Ex)
+				if (Visitor->AsyncAllowDirectory(Parent, DirectoryName))
 				{
-					ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what());
-					PendingWorkCount.CountDown();
+					PendingWorkCount.AddCount(1);
+					try
+					{
+						WorkerPool.ScheduleWork(
+							[WorkerPool		  = &WorkerPool,
+							 PendingWorkCount = &PendingWorkCount,
+							 Visitor		  = Visitor,
+							 Flags			  = Flags,
+							 Path			  = std::move(Path),
+							 RelativeRoot	  = RelativeRoot / DirectoryName]() {
+								ZEN_ASSERT(Visitor);
+								auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
+								try
+								{
+									MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
+									FileSystemTraversal	 Traversal;
+									Traversal.TraverseFileSystem(Path, SubVisitor);
+									Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
+								}
+								catch (const std::exception& Ex)
+								{
+									ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'",
+											  Path / RelativeRoot,
+											  Ex.what());
+								}
+							},
+							WorkerThreadPool::EMode::DisableBacklog);
+					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what());
+						PendingWorkCount.CountDown();
+					}
 				}
 			}
 			return false;
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index b4906aebf..938a05b59 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -368,6 +368,11 @@ public:
 		std::vector<std::filesystem::path> DirectoryNames;
 		std::vector<uint32_t>			   DirectoryAttributes;
 	};
+	virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const
+	{
+		ZEN_UNUSED(Parent, DirectoryName);
+		return true;
+	}
 	virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) = 0;
 };
 
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
index 93f8add0a..4379f2f80 100644
--- a/src/zencore/include/zencore/string.h
+++ b/src/zencore/include/zencore/string.h
@@ -821,19 +821,66 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
 
 	while (It != End)
 	{
-		if (*It == Delim)
+		std::string_view Remaining{It, size_t(ptrdiff_t(End - It))};
+		size_t			 Idx = Remaining.find(Delim, 0);
+		if (Idx == 0)
 		{
 			It++;
 			continue;
 		}
 
+		size_t DelimSize = 0;
+
+		if (Idx == std::string_view::npos)
+		{
+			Idx = Remaining.size();
+		}
+		else
+		{
+			DelimSize = 1;
+		}
+
+		Count++;
+		std::string_view Token{It, Idx};
+		if (!Func(Token))
+		{
+			break;
+		}
+
+		It = It + (Idx + DelimSize);
+	}
+
+	return Count;
+}
+
+template<typename Fn>
+uint32_t
+ForEachStrTok(const std::string_view& Str, const char* Delims, Fn&& Func)
+{
+	const char* It	  = Str.data();
+	const char* End	  = It + Str.length();
+	uint32_t	Count = 0;
+
+	while (It != End)
+	{
 		std::string_view Remaining{It, size_t(ptrdiff_t(End - It))};
-		size_t			 Idx = Remaining.find(Delim, 0);
+		size_t			 Idx = Remaining.find_first_of(Delims, 0);
+		if (Idx == 0)
+		{
+			It++;
+			continue;
+		}
+
+		size_t DelimSize = 0;
 
 		if (Idx == std::string_view::npos)
 		{
 			Idx = Remaining.size();
 		}
+		else
+		{
+			DelimSize = 1;
+		}
 
 		Count++;
 		std::string_view Token{It, Idx};
@@ -842,7 +889,7 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
 			break;
 		}
 
-		It = It + Idx;
+		It = It + (Idx + DelimSize);
 	}
 
 	return Count;
diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp
index c8c7c2cde..0ee863b74 100644
--- a/src/zencore/string.cpp
+++ b/src/zencore/string.cpp
@@ -1067,6 +1067,42 @@ TEST_CASE("string")
 		TokenCount		   = ForEachStrTok(""sv, ',', [](const std::string_view&) { return true; });
 		CHECK(TokenCount == ExpectedTokenCount);
 	}
+
+	SUBCASE("ForEachStrTok2")
+	{
+		const auto		   Tokens			  = "here,is;my,different tokens"sv;
+		const auto		   ExpectedTokens	  = "here,is,my,different,tokens"sv;
+		int32_t			   ExpectedTokenCount = 5;
+		int32_t			   TokenCount		  = 0;
+		StringBuilder<512> Sb;
+
+		TokenCount = ForEachStrTok(Tokens, " ,;", [&Sb](const std::string_view& Token) {
+			if (Sb.Size())
+			{
+				Sb << ",";
+			}
+			Sb << Token;
+			return true;
+		});
+
+		CHECK(TokenCount == ExpectedTokenCount);
+		CHECK(Sb.ToString() == ExpectedTokens);
+
+		ExpectedTokenCount = 1;
+		const auto Str	   = "mosdef"sv;
+
+		Sb.Reset();
+		TokenCount = ForEachStrTok(Str, " ,;", [&Sb](const std::string_view& Token) {
+			Sb << Token;
+			return true;
+		});
+		CHECK(Sb.ToString() == Str);
+		CHECK(TokenCount == ExpectedTokenCount);
+
+		ExpectedTokenCount = 0;
+		TokenCount		   = ForEachStrTok(""sv, " ,;", [](const std::string_view&) { return true; });
+		CHECK(TokenCount == ExpectedTokenCount);
+	}
 }
 
 #endif
diff --git a/src/zenremotestore/filesystemutils.cpp b/src/zenremotestore/filesystemutils.cpp
index 8dff05c6b..fa1ce6f78 100644
--- a/src/zenremotestore/filesystemutils.cpp
+++ b/src/zenremotestore/filesystemutils.cpp
@@ -11,6 +11,11 @@
 #include <zencore/timer.h>
 #include <zencore/trace.h>
 
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#endif	// ZEN_WITH_TESTS
+
 namespace zen {
 
 BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path,
@@ -349,13 +354,14 @@ CleanDirectory(
 	std::atomic<uint64_t> DeletedItemCount	  = 0;
 	std::atomic<uint64_t> DeletedByteCount	  = 0;
 
-	CleanDirectoryResult Result;
-	RwLock				 ResultLock;
-	auto				 _ = MakeGuard([&]() {
-		Result.DeletedCount		= DeletedItemCount.load();
-		Result.DeletedByteCount = DeletedByteCount.load();
-		Result.FoundCount		= DiscoveredItemCount.load();
-	});
+	std::vector<std::filesystem::path> DirectoriesToDelete;
+	CleanDirectoryResult			   Result;
+	RwLock							   ResultLock;
+	auto							   _ = MakeGuard([&]() {
+		  Result.DeletedCount	  = DeletedItemCount.load();
+		  Result.DeletedByteCount = DeletedByteCount.load();
+		  Result.FoundCount		  = DiscoveredItemCount.load();
+	  });
 
 	ParallelWork Work(AbortFlag,
 					  PauseFlag,
@@ -363,119 +369,133 @@ CleanDirectory(
 
 	struct AsyncVisitor : public GetDirectoryContentVisitor
 	{
-		AsyncVisitor(const std::filesystem::path& InPath,
-					 std::atomic<bool>&			  InAbortFlag,
-					 std::atomic<uint64_t>&		  InDiscoveredItemCount,
-					 std::atomic<uint64_t>&		  InDeletedItemCount,
-					 std::atomic<uint64_t>&		  InDeletedByteCount,
-					 std::span<const std::string> InExcludeDirectories,
-					 CleanDirectoryResult&		  InResult,
-					 RwLock&					  InResultLock)
+		AsyncVisitor(const std::filesystem::path&		 InPath,
+					 std::atomic<bool>&					 InAbortFlag,
+					 std::atomic<uint64_t>&				 InDiscoveredItemCount,
+					 std::atomic<uint64_t>&				 InDeletedItemCount,
+					 std::atomic<uint64_t>&				 InDeletedByteCount,
+					 std::span<const std::string>		 InExcludeDirectories,
+					 std::vector<std::filesystem::path>& OutDirectoriesToDelete,
+					 CleanDirectoryResult&				 InResult,
+					 RwLock&							 InResultLock)
 		: Path(InPath)
 		, AbortFlag(InAbortFlag)
 		, DiscoveredItemCount(InDiscoveredItemCount)
 		, DeletedItemCount(InDeletedItemCount)
 		, DeletedByteCount(InDeletedByteCount)
 		, ExcludeDirectories(InExcludeDirectories)
+		, DirectoriesToDelete(OutDirectoriesToDelete)
 		, Result(InResult)
 		, ResultLock(InResultLock)
 		{
 		}
+
+		virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const override
+		{
+			ZEN_UNUSED(Parent);
+
+			if (AbortFlag)
+			{
+				return false;
+			}
+			const std::string DirectoryString = DirectoryName.string();
+			for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+			{
+				if (DirectoryString == ExcludeDirectory)
+				{
+					return false;
+				}
+			}
+			return true;
+		}
+
 		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
 		{
 			ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
 			if (!AbortFlag)
 			{
-				if (!Content.FileNames.empty())
+				DiscoveredItemCount += Content.FileNames.size();
+
+				ZEN_TRACE_CPU("DeleteFiles");
+				std::vector<std::pair<std::filesystem::path, std::error_code>> FailedRemovePaths;
+				for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
 				{
-					DiscoveredItemCount += Content.FileNames.size();
+					const std::filesystem::path& FileName = Content.FileNames[FileIndex];
+					const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
 
-					const std::string RelativeRootString = RelativeRoot.generic_string();
-					bool			  RemoveContent		 = true;
-					for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+					bool			IsRemoved = false;
+					std::error_code Ec;
+					(void)SetFileReadOnly(FilePath, false, Ec);
+					for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 					{
-						if (RelativeRootString.starts_with(ExcludeDirectory))
+						if (!IsFileWithRetry(FilePath))
 						{
-							if (RelativeRootString.length() > ExcludeDirectory.length())
-							{
-								const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
-								if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
-									MaybePathDelimiter == std::filesystem::path::preferred_separator)
-								{
-									RemoveContent = false;
-									break;
-								}
-							}
-							else
-							{
-								RemoveContent = false;
-								break;
-							}
+							IsRemoved = true;
+							Ec.clear();
+							break;
 						}
+						Sleep(100 + int(Retries * 50));
+						Ec.clear();
+						(void)SetFileReadOnly(FilePath, false, Ec);
 					}
-					if (RemoveContent)
+					if (!IsRemoved && !Ec)
 					{
-						ZEN_TRACE_CPU("DeleteFiles");
-						for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+						(void)RemoveFile(FilePath, Ec);
+						for (size_t Retries = 0; Ec && Retries < 6; Retries++)
 						{
-							const std::filesystem::path& FileName = Content.FileNames[FileIndex];
-							const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
-
-							bool			IsRemoved = false;
-							std::error_code Ec;
-							(void)SetFileReadOnly(FilePath, false, Ec);
-							for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+							if (!IsFileWithRetry(FilePath))
 							{
-								if (!IsFileWithRetry(FilePath))
-								{
-									IsRemoved = true;
-									Ec.clear();
-									break;
-								}
-								Sleep(100 + int(Retries * 50));
+								IsRemoved = true;
 								Ec.clear();
-								(void)SetFileReadOnly(FilePath, false, Ec);
-							}
-							if (!IsRemoved && !Ec)
-							{
-								(void)RemoveFile(FilePath, Ec);
-								for (size_t Retries = 0; Ec && Retries < 6; Retries++)
-								{
-									if (!IsFileWithRetry(FilePath))
-									{
-										IsRemoved = true;
-										Ec.clear();
-										return;
-									}
-									Sleep(100 + int(Retries * 50));
-									Ec.clear();
-									(void)RemoveFile(FilePath, Ec);
-								}
-							}
-							if (!IsRemoved && Ec)
-							{
-								RwLock::ExclusiveLockScope _(ResultLock);
-								Result.FailedRemovePaths.push_back(std::make_pair(FilePath, Ec));
-							}
-							else
-							{
-								DeletedItemCount++;
-								DeletedByteCount += Content.FileSizes[FileIndex];
+								break;
 							}
+							Sleep(100 + int(Retries * 50));
+							Ec.clear();
+							(void)RemoveFile(FilePath, Ec);
 						}
 					}
+					if (!IsRemoved && Ec)
+					{
+						FailedRemovePaths.push_back(std::make_pair(FilePath, Ec));
+					}
+					else
+					{
+						DeletedItemCount++;
+						DeletedByteCount += Content.FileSizes[FileIndex];
+					}
+				}
+
+				if (!FailedRemovePaths.empty())
+				{
+					RwLock::ExclusiveLockScope _(ResultLock);
+					FailedRemovePaths.insert(FailedRemovePaths.end(), FailedRemovePaths.begin(), FailedRemovePaths.end());
+				}
+				else if (!RelativeRoot.empty())
+				{
+					DiscoveredItemCount++;
+					RwLock::ExclusiveLockScope _(ResultLock);
+					DirectoriesToDelete.push_back(RelativeRoot);
 				}
 			}
 		}
-		const std::filesystem::path& Path;
-		std::atomic<bool>&			 AbortFlag;
-		std::atomic<uint64_t>&		 DiscoveredItemCount;
-		std::atomic<uint64_t>&		 DeletedItemCount;
-		std::atomic<uint64_t>&		 DeletedByteCount;
-		std::span<const std::string> ExcludeDirectories;
-		CleanDirectoryResult&		 Result;
-		RwLock&						 ResultLock;
-	} Visitor(Path, AbortFlag, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories, Result, ResultLock);
+		const std::filesystem::path&		Path;
+		std::atomic<bool>&					AbortFlag;
+		std::atomic<uint64_t>&				DiscoveredItemCount;
+		std::atomic<uint64_t>&				DeletedItemCount;
+		std::atomic<uint64_t>&				DeletedByteCount;
+		std::span<const std::string>		ExcludeDirectories;
+		std::vector<std::filesystem::path>& DirectoriesToDelete;
+		CleanDirectoryResult&				Result;
+		RwLock&								ResultLock;
+	} Visitor(Path,
+			  AbortFlag,
+			  DiscoveredItemCount,
+			  DeletedItemCount,
+			  DeletedByteCount,
+			  ExcludeDirectories,
+			  DirectoriesToDelete,
+			  Result,
+			  ResultLock);
 
 	GetDirectoryContent(Path,
 						DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes,
@@ -483,29 +503,6 @@ CleanDirectory(
 						IOWorkerPool,
 						Work.PendingWork());
 
-	DirectoryContent LocalDirectoryContent;
-	GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
-	DiscoveredItemCount += LocalDirectoryContent.Directories.size();
-	std::vector<std::filesystem::path> DirectoriesToDelete;
-	DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size());
-	for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
-	{
-		bool Leave = false;
-		for (const std::string_view ExcludeDirectory : ExcludeDirectories)
-		{
-			if (LocalDirPath == (Path / ExcludeDirectory))
-			{
-				Leave = true;
-				break;
-			}
-		}
-		if (!Leave)
-		{
-			DirectoriesToDelete.emplace_back(std::move(LocalDirPath));
-			DiscoveredItemCount++;
-		}
-	}
-
 	uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
 
 	if (ProgressFunc && ProgressUpdateDelayMS != 0)
@@ -528,6 +525,15 @@ CleanDirectory(
 
 	{
 		ZEN_TRACE_CPU("DeleteDirs");
+
+		std::sort(DirectoriesToDelete.begin(),
+				  DirectoriesToDelete.end(),
+				  [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) {
+					  auto DistanceLhs = std::distance(Lhs.begin(), Lhs.end());
+					  auto DistanceRhs = std::distance(Rhs.begin(), Rhs.end());
+					  return DistanceLhs > DistanceRhs;
+				  });
+
 		for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete)
 		{
 			if (AbortFlag)
@@ -542,53 +548,48 @@ CleanDirectory(
 				}
 			}
 
-			{
-				std::error_code Ec;
-				zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec);
-				if (Ec)
-				{
-					Sleep(200);
-					Ec.clear();
-					zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec);
-				}
+			const std::filesystem::path FullPath = Path / DirectoryToDelete;
 
-				if (!Ec)
+			std::error_code Ec;
+			RemoveDir(FullPath, Ec);
+			if (Ec)
+			{
+				for (size_t Retries = 0; Ec && Retries < 3; Retries++)
 				{
-					RemoveDir(DirectoryToDelete, Ec);
-					for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+					if (!IsDir(FullPath))
 					{
-						if (!IsDir(DirectoryToDelete))
-						{
-							Ec.clear();
-							break;
-						}
-						Sleep(100 + int(Retries * 50));
 						Ec.clear();
-						RemoveDir(DirectoryToDelete, Ec);
+						break;
 					}
-				}
-				if (Ec)
-				{
-					RwLock::ExclusiveLockScope __(ResultLock);
-					Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec));
-				}
-				else
-				{
-					DeletedItemCount++;
+					Sleep(100 + int(Retries * 50));
+					Ec.clear();
+					RemoveDir(FullPath, Ec);
 				}
 			}
+			if (Ec)
+			{
+				RwLock::ExclusiveLockScope __(ResultLock);
+				Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec));
+			}
+			else
+			{
+				DeletedItemCount++;
+			}
 
-			uint64_t NowMs = Timer.GetElapsedTimeMs();
-
-			if ((NowMs - LastUpdateTimeMs) >= ProgressUpdateDelayMS)
+			if (ProgressFunc)
 			{
-				LastUpdateTimeMs = NowMs;
+				uint64_t NowMs = Timer.GetElapsedTimeMs();
+
+				if ((NowMs - LastUpdateTimeMs) > 0)
+				{
+					LastUpdateTimeMs = NowMs;
 
-				uint64_t	Deleted		 = DeletedItemCount.load();
-				uint64_t	DeletedBytes = DeletedByteCount.load();
-				uint64_t	Discovered	 = DiscoveredItemCount.load();
-				std::string Details		 = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes));
-				ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag);
+					uint64_t	Deleted		 = DeletedItemCount.load();
+					uint64_t	DeletedBytes = DeletedByteCount.load();
+					uint64_t	Discovered	 = DiscoveredItemCount.load();
+					std::string Details		 = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes));
+					ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag);
+				}
 			}
 		}
 	}
@@ -625,4 +626,72 @@ CleanAndRemoveDirectory(WorkerThreadPool&			 WorkerPool,
 	return false;
 }
 
+#if ZEN_WITH_TESTS
+
+void
+filesystemutils_forcelink()
+{
+}
+
+namespace {
+	void GenerateFile(const std::filesystem::path& Path) { BasicFile _(Path, BasicFile::Mode::kTruncate); }
+}  // namespace
+
+TEST_CASE("filesystemutils.CleanDirectory")
+{
+	ScopedTemporaryDirectory TmpDir;
+
+	CreateDirectories(TmpDir.Path() / ".keepme");
+	GenerateFile(TmpDir.Path() / ".keepme" / "keep");
+	GenerateFile(TmpDir.Path() / "deleteme1");
+	GenerateFile(TmpDir.Path() / "deleteme2");
+	GenerateFile(TmpDir.Path() / "deleteme3");
+	CreateDirectories(TmpDir.Path() / ".keepmenot");
+	CreateDirectories(TmpDir.Path() / "no.keepme");
+
+	CreateDirectories(TmpDir.Path() / "DeleteMe");
+	GenerateFile(TmpDir.Path() / "DeleteMe" / "delete1");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete1");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete3");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe" / ".keepme");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete3");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme");
+	CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2");
+	GenerateFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme");
+
+	WorkerThreadPool  Pool(4);
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+
+	CleanDirectory(Pool, AbortFlag, PauseFlag, TmpDir.Path(), std::vector<std::string>{".keepme"}, {}, 0);
+
+	CHECK(IsDir(TmpDir.Path() / ".keepme"));
+	CHECK(IsFile(TmpDir.Path() / ".keepme" / "keep"));
+	CHECK(!IsFile(TmpDir.Path() / "deleteme1"));
+	CHECK(!IsFile(TmpDir.Path() / "deleteme2"));
+	CHECK(!IsFile(TmpDir.Path() / "deleteme3"));
+	CHECK(!IsFile(TmpDir.Path() / ".keepmenot"));
+	CHECK(!IsFile(TmpDir.Path() / "no.keepme"));
+
+	CHECK(!IsDir(TmpDir.Path() / "DeleteMe"));
+	CHECK(!IsDir(TmpDir.Path() / "DeleteMe2"));
+
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe" / ".keepme"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme"));
+	CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept"));
+	CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1"));
+	CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2"));
+	CHECK(!IsFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme"));
+}
+
+#endif
+
 }  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 51cd7f841..32c8bda01 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -141,7 +141,6 @@ public:
 		bool					 EnableTargetFolderScavenging	= true;
 		bool					 ValidateCompletedSequences		= true;
 		std::vector<std::string> ExcludeFolders;
-		std::vector<std::string> ExcludeExtensions;
 		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
 		bool					 PopulateCache				= true;
 	};
diff --git a/src/zenremotestore/include/zenremotestore/filesystemutils.h b/src/zenremotestore/include/zenremotestore/filesystemutils.h
index a6c88e5cb..cfd6f02e1 100644
--- a/src/zenremotestore/include/zenremotestore/filesystemutils.h
+++ b/src/zenremotestore/include/zenremotestore/filesystemutils.h
@@ -116,4 +116,6 @@ bool CleanAndRemoveDirectory(WorkerThreadPool&			  WorkerPool,
 							 std::atomic<bool>&			  PauseFlag,
 							 const std::filesystem::path& Directory);
 
+void filesystemutils_forcelink();  // internal
+
 }  // namespace zen
diff --git a/src/zenremotestore/zenremotestore.cpp b/src/zenremotestore/zenremotestore.cpp
index e074455b3..7f785599f 100644
--- a/src/zenremotestore/zenremotestore.cpp
+++ b/src/zenremotestore/zenremotestore.cpp
@@ -5,6 +5,7 @@
 #include <zenremotestore/builds/buildsavedstate.h>
 #include <zenremotestore/chunking/chunkedcontent.h>
 #include <zenremotestore/chunking/chunkedfile.h>
+#include <zenremotestore/filesystemutils.h>
 #include <zenremotestore/projectstore/remoteprojectstore.h>
 
 #if ZEN_WITH_TESTS
@@ -19,6 +20,7 @@ zenremotestore_forcelinktests()
 	chunkedcontent_forcelink();
 	chunkedfile_forcelink();
 	chunkedcontent_forcelink();
+	filesystemutils_forcelink();
 	remoteprojectstore_forcelink();
 }