added options to configure exclude folders and extensions to zen build commands (#706)

* added `--exclude-folders` to `zen upload`, `zen download` and `zen diff`
added `--exclude-extensions` to `zen upload` and `zen diff`
excluded folder names are now matched by folder name in subfolders in addition to root level folders
* allow multiple token separators

4 files changed, 125 insertions, 32 deletions

diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 7f341818b..4da412c17 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -2669,36 +2669,41 @@ GetDirectoryContent(const std::filesystem::path& RootDir,
 			}
 			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive))
 			{
-				PendingWorkCount.AddCount(1);
-				try
-				{
-					WorkerPool.ScheduleWork(
-						[WorkerPool		  = &WorkerPool,
-						 PendingWorkCount = &PendingWorkCount,
-						 Visitor		  = Visitor,
-						 Flags			  = Flags,
-						 Path			  = std::move(Path),
-						 RelativeRoot	  = RelativeRoot / DirectoryName]() {
-							ZEN_ASSERT(Visitor);
-							auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
-							try
-							{
-								MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
-								FileSystemTraversal	 Traversal;
-								Traversal.TraverseFileSystem(Path, SubVisitor);
-								Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
-							}
-							catch (const std::exception& Ex)
-							{
-								ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'", Path / RelativeRoot, Ex.what());
-							}
-						},
-						WorkerThreadPool::EMode::DisableBacklog);
-				}
-				catch (const std::exception& Ex)
+				if (Visitor->AsyncAllowDirectory(Parent, DirectoryName))
 				{
-					ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what());
-					PendingWorkCount.CountDown();
+					PendingWorkCount.AddCount(1);
+					try
+					{
+						WorkerPool.ScheduleWork(
+							[WorkerPool		  = &WorkerPool,
+							 PendingWorkCount = &PendingWorkCount,
+							 Visitor		  = Visitor,
+							 Flags			  = Flags,
+							 Path			  = std::move(Path),
+							 RelativeRoot	  = RelativeRoot / DirectoryName]() {
+								ZEN_ASSERT(Visitor);
+								auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
+								try
+								{
+									MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
+									FileSystemTraversal	 Traversal;
+									Traversal.TraverseFileSystem(Path, SubVisitor);
+									Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
+								}
+								catch (const std::exception& Ex)
+								{
+									ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'",
+											  Path / RelativeRoot,
+											  Ex.what());
+								}
+							},
+							WorkerThreadPool::EMode::DisableBacklog);
+					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what());
+						PendingWorkCount.CountDown();
+					}
 				}
 			}
 			return false;
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index b4906aebf..938a05b59 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -368,6 +368,11 @@ public:
 		std::vector<std::filesystem::path> DirectoryNames;
 		std::vector<uint32_t>			   DirectoryAttributes;
 	};
+	virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const
+	{
+		ZEN_UNUSED(Parent, DirectoryName);
+		return true;
+	}
 	virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) = 0;
 };
 
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
index 93f8add0a..4379f2f80 100644
--- a/src/zencore/include/zencore/string.h
+++ b/src/zencore/include/zencore/string.h
@@ -821,19 +821,66 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
 
 	while (It != End)
 	{
-		if (*It == Delim)
+		std::string_view Remaining{It, size_t(ptrdiff_t(End - It))};
+		size_t			 Idx = Remaining.find(Delim, 0);
+		if (Idx == 0)
 		{
 			It++;
 			continue;
 		}
 
+		size_t DelimSize = 0;
+
+		if (Idx == std::string_view::npos)
+		{
+			Idx = Remaining.size();
+		}
+		else
+		{
+			DelimSize = 1;
+		}
+
+		Count++;
+		std::string_view Token{It, Idx};
+		if (!Func(Token))
+		{
+			break;
+		}
+
+		It = It + (Idx + DelimSize);
+	}
+
+	return Count;
+}
+
+template<typename Fn>
+uint32_t
+ForEachStrTok(const std::string_view& Str, const char* Delims, Fn&& Func)
+{
+	const char* It	  = Str.data();
+	const char* End	  = It + Str.length();
+	uint32_t	Count = 0;
+
+	while (It != End)
+	{
 		std::string_view Remaining{It, size_t(ptrdiff_t(End - It))};
-		size_t			 Idx = Remaining.find(Delim, 0);
+		size_t			 Idx = Remaining.find_first_of(Delims, 0);
+		if (Idx == 0)
+		{
+			It++;
+			continue;
+		}
+
+		size_t DelimSize = 0;
 
 		if (Idx == std::string_view::npos)
 		{
 			Idx = Remaining.size();
 		}
+		else
+		{
+			DelimSize = 1;
+		}
 
 		Count++;
 		std::string_view Token{It, Idx};
@@ -842,7 +889,7 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
 			break;
 		}
 
-		It = It + Idx;
+		It = It + (Idx + DelimSize);
 	}
 
 	return Count;
diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp
index c8c7c2cde..0ee863b74 100644
--- a/src/zencore/string.cpp
+++ b/src/zencore/string.cpp
@@ -1067,6 +1067,42 @@ TEST_CASE("string")
 		TokenCount		   = ForEachStrTok(""sv, ',', [](const std::string_view&) { return true; });
 		CHECK(TokenCount == ExpectedTokenCount);
 	}
+
+	SUBCASE("ForEachStrTok2")
+	{
+		const auto		   Tokens			  = "here,is;my,different tokens"sv;
+		const auto		   ExpectedTokens	  = "here,is,my,different,tokens"sv;
+		int32_t			   ExpectedTokenCount = 5;
+		int32_t			   TokenCount		  = 0;
+		StringBuilder<512> Sb;
+
+		TokenCount = ForEachStrTok(Tokens, " ,;", [&Sb](const std::string_view& Token) {
+			if (Sb.Size())
+			{
+				Sb << ",";
+			}
+			Sb << Token;
+			return true;
+		});
+
+		CHECK(TokenCount == ExpectedTokenCount);
+		CHECK(Sb.ToString() == ExpectedTokens);
+
+		ExpectedTokenCount = 1;
+		const auto Str	   = "mosdef"sv;
+
+		Sb.Reset();
+		TokenCount = ForEachStrTok(Str, " ,;", [&Sb](const std::string_view& Token) {
+			Sb << Token;
+			return true;
+		});
+		CHECK(Sb.ToString() == Str);
+		CHECK(TokenCount == ExpectedTokenCount);
+
+		ExpectedTokenCount = 0;
+		TokenCount		   = ForEachStrTok(""sv, " ,;", [](const std::string_view&) { return true; });
+		CHECK(TokenCount == ExpectedTokenCount);
+	}
 }
 
 #endif