builds multi wildcard (#524)

* allow multiple include/exclude wildcards in zen builds command

1 files changed, 83 insertions, 50 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 483763f94..c585c50c9 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -409,16 +409,34 @@ namespace {
 
 	);
 
-	bool IncludePath(const std::string_view IncludeWildcard, const std::string_view ExcludeWildcard, const std::filesystem::path& Path)
+	bool IncludePath(std::span<const std::string> IncludeWildcards,
+					 std::span<const std::string> ExcludeWildcards,
+					 const std::filesystem::path& Path)
 	{
-		const std::string PathString = Path.generic_string();
-		if (!IncludeWildcard.empty() && !MatchWildcard(IncludeWildcard, PathString, /*CaseSensitive*/ false))
+		const std::string PathString  = Path.generic_string();
+		bool			  IncludePath = true;
+		if (!IncludeWildcards.empty())
 		{
-			return false;
+			IncludePath = false;
+			for (const std::string& IncludeWildcard : IncludeWildcards)
+			{
+				if (MatchWildcard(IncludeWildcard, PathString, /*CaseSensitive*/ false))
+				{
+					IncludePath = true;
+					break;
+				}
+			}
+			if (!IncludePath)
+			{
+				return false;
+			}
 		}
-		if (!ExcludeWildcard.empty() && MatchWildcard(ExcludeWildcard, PathString, /*CaseSensitive*/ false))
+		for (const std::string& ExcludeWildcard : ExcludeWildcards)
 		{
-			return false;
+			if (MatchWildcard(ExcludeWildcard, PathString, /*CaseSensitive*/ false))
+			{
+				return false;
+			}
 		}
 		return true;
 	}
@@ -8887,8 +8905,8 @@ namespace {
 	ChunkedFolderContent GetRemoteContent(StorageInstance&								  Storage,
 										  const Oid&									  BuildId,
 										  const std::vector<std::pair<Oid, std::string>>& BuildParts,
-										  std::string_view								  IncludeWildcard,
-										  std::string_view								  ExcludeWildcard,
+										  std::span<const std::string>					  IncludeWildcards,
+										  std::span<const std::string>					  ExcludeWildcards,
 										  std::unique_ptr<ChunkingController>&			  OutChunkController,
 										  std::vector<ChunkedFolderContent>&			  OutPartContents,
 										  std::vector<ChunkBlockDescription>&			  OutBlockDescriptions,
@@ -8921,8 +8939,8 @@ namespace {
 										 const Oid&							 BuildId,
 										 const Oid&							 BuildPartId,
 										 CbObject							 BuildPartManifest,
-										 std::string_view					 IncludeWildcard,
-										 std::string_view					 ExcludeWildcard,
+										 std::span<const std::string>		 IncludeWildcards,
+										 std::span<const std::string>		 ExcludeWildcards,
 										 ChunkedFolderContent&				 OutRemoteContent,
 										 std::vector<ChunkBlockDescription>& OutBlockDescriptions,
 										 std::vector<IoHash>&				 OutLooseChunkHashes) {
@@ -9142,11 +9160,12 @@ namespace {
 									  OutRemoteContent.ChunkedContent.ChunkRawSizes,
 									  OutRemoteContent.ChunkedContent.ChunkOrders);
 
+			if (!IncludeWildcards.empty() || !ExcludeWildcards.empty())
 			{
 				std::vector<std::filesystem::path> DeletedPaths;
 				for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
 				{
-					if (!IncludePath(IncludeWildcard, ExcludeWildcard, RemotePath))
+					if (!IncludePath(IncludeWildcards, ExcludeWildcards, RemotePath))
 					{
 						DeletedPaths.push_back(RemotePath);
 					}
@@ -9178,8 +9197,8 @@ namespace {
 							   BuildId,
 							   BuildPartId,
 							   BuildPartManifest,
-							   IncludeWildcard,
-							   ExcludeWildcard,
+							   IncludeWildcards,
+							   ExcludeWildcards,
 							   OutPartContents[0],
 							   OutBlockDescriptions,
 							   OutLooseChunkHashes);
@@ -9211,8 +9230,8 @@ namespace {
 									   BuildId,
 									   OverlayBuildPartId,
 									   OverlayBuildPartManifest,
-									   IncludeWildcard,
-									   ExcludeWildcard,
+									   IncludeWildcards,
+									   ExcludeWildcards,
 									   OverlayPartContent,
 									   OverlayPartBlockDescriptions,
 									   OverlayPartLooseChunkHashes);
@@ -9267,8 +9286,8 @@ namespace {
 										 const std::filesystem::path&			StateFilePath,
 										 ChunkingController&					ChunkController,
 										 std::span<const std::filesystem::path> ReferencePaths,
-										 std::string_view						IncludeWildcard,
-										 std::string_view						ExcludeWildcard,
+										 std::span<const std::string>			IncludeWildcards,
+										 std::span<const std::string>			ExcludeWildcards,
 										 FolderContent&							OutLocalFolderContent)
 	{
 		FolderContent		 LocalFolderState;
@@ -9292,7 +9311,7 @@ namespace {
 
 			for (const std::filesystem::path& LocalPath : LocalFolderState.Paths)
 			{
-				if (IncludePath(IncludeWildcard, ExcludeWildcard, LocalPath))
+				if (IncludePath(IncludeWildcards, ExcludeWildcards, LocalPath))
 				{
 					FileSet.insert(LocalPath.generic_string());
 					PathsToCheck.push_back(LocalPath);
@@ -9301,7 +9320,7 @@ namespace {
 
 			for (const std::filesystem::path& RemotePath : ReferencePaths)
 			{
-				if (IncludePath(IncludeWildcard, ExcludeWildcard, RemotePath))
+				if (IncludePath(IncludeWildcards, ExcludeWildcards, RemotePath))
 				{
 					if (FileSet.insert(RemotePath.generic_string()).second)
 					{
@@ -9620,8 +9639,8 @@ namespace {
 		bool					 PrimeCacheOnly					= false;
 		bool					 EnableOtherDownloadsScavenging = true;
 		bool					 EnableTargetFolderScavenging	= true;
-		std::string				 IncludeWildcard;
-		std::string				 ExcludeWildcard;
+		std::vector<std::string> IncludeWildcards;
+		std::vector<std::string> ExcludeWildcards;
 	};
 
 	void DownloadFolder(StorageInstance&			 Storage,
@@ -9679,8 +9698,8 @@ namespace {
 		ChunkedFolderContent RemoteContent = GetRemoteContent(Storage,
 															  BuildId,
 															  AllBuildParts,
-															  Options.IncludeWildcard,
-															  Options.ExcludeWildcard,
+															  Options.IncludeWildcards,
+															  Options.ExcludeWildcards,
 															  ChunkController,
 															  PartContents,
 															  BlockDescriptions,
@@ -9707,8 +9726,8 @@ namespace {
 											   ZenStateFilePath(Options.ZenFolderPath),
 											   *ChunkController,
 											   RemoteContent.Paths,
-											   Options.IncludeWildcard,
-											   Options.ExcludeWildcard,
+											   Options.IncludeWildcards,
+											   Options.ExcludeWildcards,
 											   LocalFolderContent);
 			}
 			else
@@ -9924,8 +9943,8 @@ namespace {
 				   const Oid&					BuildId,
 				   const std::vector<Oid>&		BuildPartIds,
 				   std::span<const std::string> BuildPartNames,
-				   std::string_view				IncludeWildcard,
-				   std::string_view				ExcludeWildcard)
+				   std::span<const std::string> IncludeWildcards,
+				   std::span<const std::string> ExcludeWildcards)
 	{
 		std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
 
@@ -9988,7 +10007,7 @@ namespace {
 			for (size_t Index : Order)
 			{
 				const std::filesystem::path& Path = Paths[Index];
-				if (IncludePath(IncludeWildcard, ExcludeWildcard, Path))
+				if (IncludePath(IncludeWildcards, ExcludeWildcards, Path))
 				{
 					const IoHash&  RawHash	 = RawHashes[Index];
 					const uint64_t RawSize	 = RawSizes[Index];
@@ -10286,14 +10305,15 @@ BuildsCommand::BuildsCommand()
 		Ops.add_option("",
 					   "",
 					   "wildcard",
-					   "Windows style wildcard (using * and ?) to match file paths to include",
+					   "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;",
 					   cxxopts::value(m_IncludeWildcard),
 					   "<wildcard>");
 
 		Ops.add_option("",
 					   "",
 					   "exclude-wildcard",
-					   "Windows style wildcard (using * and ?) to match file paths to exclude. Applied after --wildcard include filter",
+					   "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard "
+					   "include filter",
 					   cxxopts::value(m_ExcludeWildcard),
 					   "<excludewildcard>");
 	};
@@ -11027,22 +11047,30 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		MakeSafeAbsolutePathÍnPlace(m_Path);
 	};
 
-	auto ParseFileFilters = [&]() {
-		for (auto It = begin(m_IncludeWildcard); It != end(m_IncludeWildcard); It++)
-		{
-			if (*It == '\\')
-			{
-				*It = '/';
-			}
-		}
+	auto ParseFileFilters = [&](std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) {
+		auto SplitWildcard = [](const std::string_view Wildcard) -> std::vector<std::string> {
+			std::vector<std::string> Wildcards;
+			ForEachStrTok(Wildcard, ';', [&Wildcards](std::string_view Wildcard) {
+				if (!Wildcard.empty())
+				{
+					std::string CleanWildcard(Wildcard);
+					for (auto It = begin(CleanWildcard); It != end(CleanWildcard); It++)
+					{
+						if (*It == '\\')
+						{
+							*It = '/';
+						}
+					}
 
-		for (auto It = begin(m_ExcludeWildcard); It != end(m_ExcludeWildcard); It++)
-		{
-			if (*It == '\\')
-			{
-				*It = '/';
-			}
-		}
+					Wildcards.emplace_back(std::move(CleanWildcard));
+				}
+				return true;
+			});
+			return Wildcards;
+		};
+
+		OutIncludeWildcards = SplitWildcard(m_IncludeWildcard);
+		OutExcludeWildcards = SplitWildcard(m_ExcludeWildcard);
 	};
 
 	auto ParseDiffPath = [&]() {
@@ -11549,7 +11577,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		ZenState InstanceState;
 
 		ParsePath();
-		ParseFileFilters();
+
+		std::vector<std::string> IncludeWildcards;
+		std::vector<std::string> ExcludeWildcards;
+		ParseFileFilters(IncludeWildcards, ExcludeWildcards);
 
 		if (m_ZenFolderPath.empty())
 		{
@@ -11607,8 +11638,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PrimeCacheOnly				   = m_PrimeCacheOnly,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
 									   .EnableTargetFolderScavenging   = !m_Force,
-									   .IncludeWildcard				   = m_IncludeWildcard,
-									   .ExcludeWildcard				   = m_ExcludeWildcard});
+									   .IncludeWildcards			   = IncludeWildcards,
+									   .ExcludeWildcards			   = ExcludeWildcards});
 
 		if (AbortFlag)
 		{
@@ -11625,7 +11656,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 		ZenState InstanceState;
 
-		ParseFileFilters();
+		std::vector<std::string> IncludeWildcards;
+		std::vector<std::string> ExcludeWildcards;
+		ParseFileFilters(IncludeWildcards, ExcludeWildcards);
 
 		if (m_ZenFolderPath.empty())
 		{
@@ -11647,7 +11680,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		std::vector<Oid>		 BuildPartIds	= ParseBuildPartIds();
 		std::vector<std::string> BuildPartNames = ParseBuildPartNames();
 
-		ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard);
+		ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards);
 
 		if (AbortFlag)
 		{