Merge remote-tracking branch 'origin/main' into de/zen-service-command

6 files changed, 51 insertions, 71 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 3974fb989..e976c061d 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -2,6 +2,7 @@
 
 #include <zenstore/blockstore.h>
 
+#include <zencore/enumflags.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -1427,8 +1428,8 @@ namespace blockstore::impl {
 	{
 		DirectoryContent DirectoryContent;
 		GetDirectoryContent(RootDir,
-							DirectoryContent::RecursiveFlag | (Files ? DirectoryContent::IncludeFilesFlag : 0) |
-								(Directories ? DirectoryContent::IncludeDirsFlag : 0),
+							DirectoryContentFlags::Recursive | (Files ? DirectoryContentFlags::IncludeFiles : DirectoryContentFlags::None) |
+								(Directories ? DirectoryContentFlags::IncludeDirs : DirectoryContentFlags::None),
 							DirectoryContent);
 		std::vector<std::filesystem::path> Result;
 		Result.insert(Result.end(), DirectoryContent.Directories.begin(), DirectoryContent.Directories.end());
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index cbc1d6e83..25f68330a 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -3858,7 +3858,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 	ZEN_TRACE_CPU("Z$::DiscoverBuckets");
 
 	DirectoryContent DirContent;
-	GetDirectoryContent(m_RootDir, DirectoryContent::IncludeDirsFlag, DirContent);
+	GetDirectoryContent(m_RootDir, DirectoryContentFlags::IncludeDirs, DirContent);
 
 	// Initialize buckets
 
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index c14ea73a8..133cb42d7 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -424,7 +424,7 @@ ZenCacheStore::ZenCacheStore(GcManager&					  Gc,
 	ZEN_INFO("initializing cache store at '{}'", m_BasePath);
 
 	DirectoryContent DirContent;
-	GetDirectoryContent(m_BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+	GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent);
 
 	std::vector<std::string> Namespaces;
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
diff --git a/src/zenstore/chunkedfile.cpp b/src/zenstore/chunkedfile.cpp
index 0f3cf5112..f200bc1ec 100644
--- a/src/zenstore/chunkedfile.cpp
+++ b/src/zenstore/chunkedfile.cpp
@@ -195,10 +195,10 @@ TEST_CASE("chunkedfile.findparams")
 {
 #		if 1
 	DirectoryContent SourceContent1;
-	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContent::IncludeFilesFlag, SourceContent1);
+	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContentFlags::IncludeFiles, SourceContent1);
 	const std::vector<std::filesystem::path>& SourceFiles1 = SourceContent1.Files;
 	DirectoryContent						  SourceContent2;
-	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContent::IncludeFilesFlag, SourceContent2);
+	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContentFlags::IncludeFiles, SourceContent2);
 	const std::vector<std::filesystem::path>& SourceFiles2 = SourceContent2.Files;
 #		else
 	std::filesystem::path SourcePath1 =
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 110baaf5f..14123528c 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -185,7 +185,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 			// in this folder as well
 			struct Visitor : public FileSystemTraversal::TreeVisitor
 			{
-				virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override
+				virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t, uint32_t) override
 				{
 					// We don't care about files
 				}
@@ -193,8 +193,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 				{
 					return std::find(&HexChars[0], &HexChars[16], C) != &HexChars[16];
 				}
-				virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
-											[[maybe_unused]] const path_view&			  DirectoryName) override
+				virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName, uint32_t) override
 				{
 					if (DirectoryName.length() == 3)
 					{
@@ -1175,7 +1174,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
 	struct Visitor : public FileSystemTraversal::TreeVisitor
 	{
 		Visitor(const std::filesystem::path& RootDir, std::vector<FileCasIndexEntry>& Entries) : RootDirectory(RootDir), Entries(Entries) {}
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
 		{
 			std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory);
 
@@ -1201,11 +1200,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
 			}
 		}
 
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
-									[[maybe_unused]] const path_view&			  DirectoryName) override
-		{
-			return true;
-		}
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
 
 		const std::filesystem::path&	RootDirectory;
 		std::vector<FileCasIndexEntry>& Entries;
diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp
index d399ffc88..02a83d2a6 100644
--- a/src/zenstore/workspaces.cpp
+++ b/src/zenstore/workspaces.cpp
@@ -252,18 +252,43 @@ FolderStructure::FolderStructure(std::vector<FileEntry>&& InEntries, std::vector
 }
 
 namespace {
-	struct FolderScanner
+	struct FolderScanner : public GetDirectoryContentVisitor
 	{
 		FolderScanner(LoggerRef& Log, WorkerThreadPool& WorkerPool, const std::filesystem::path& Path)
 		: m_Log(Log)
 		, Path(Path)
-		, WorkLatch(1)
 		, WorkerPool(WorkerPool)
 		{
 		}
 
 		void Traverse();
-		void Traverse(const std::filesystem::path& RelativeRoot, const std::filesystem::path& Path);
+
+		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+		{
+			std::vector<FolderStructure::FileEntry> FileEntries;
+			std::vector<Oid>						PathIds;
+			const size_t							FileCount = Content.FileNames.size();
+			FileEntries.reserve(FileCount);
+			PathIds.reserve(FileCount);
+
+			auto FileNameIt = Content.FileNames.begin();
+			auto FileSizeIt = Content.FileSizes.begin();
+			while (FileNameIt != Content.FileNames.end())
+			{
+				ZEN_ASSERT_SLOW(FileSizeIt != Content.FileSizes.end());
+
+				std::filesystem::path RelativePath = RelativeRoot.empty() ? *FileNameIt : RelativeRoot / *FileNameIt;
+				PathIds.emplace_back(Workspaces::PathToId(RelativePath));
+				FileEntries.emplace_back(FolderStructure::FileEntry{.RelativePath = std::move(RelativePath), .Size = *FileSizeIt});
+
+				FileNameIt++;
+				FileSizeIt++;
+			}
+			WorkLock.WithExclusiveLock([&]() {
+				FoundFiles.insert(FoundFiles.end(), FileEntries.begin(), FileEntries.end());
+				FoundFileIds.insert(FoundFileIds.end(), PathIds.begin(), PathIds.end());
+			});
+		}
 
 		LoggerRef&								Log() { return m_Log; }
 		LoggerRef&								m_Log;
@@ -271,75 +296,34 @@ namespace {
 		RwLock									WorkLock;
 		std::vector<FolderStructure::FileEntry> FoundFiles;
 		std::vector<Oid>						FoundFileIds;
-		Latch									WorkLatch;
 		WorkerThreadPool&						WorkerPool;
 	};
 
-	struct Visitor : public FileSystemTraversal::TreeVisitor
+	void FolderScanner::Traverse()
 	{
-		Visitor(FolderScanner& Data, const std::filesystem::path& RelativeRoot) : Data(Data), RelativeRoot(RelativeRoot) {}
+		Stopwatch Timer;
 
-		FileSystemTraversal						Traverser;
-		FolderScanner&							Data;
-		std::vector<FolderStructure::FileEntry> Entries;
-		std::vector<Oid>						FileIds;
-		std::filesystem::path					RelativeRoot;
+		const std::filesystem::path Root = std::filesystem::absolute(Path);
 
-		virtual void VisitFile(const std::filesystem::path&, const path_view& File, uint64_t FileSize)
-		{
-			std::filesystem::path RelativePath = RelativeRoot.empty() ? File : RelativeRoot / File;
-			Entries.push_back(FolderStructure::FileEntry{.RelativePath = RelativePath, .Size = FileSize});
-			FileIds.push_back(Workspaces::PathToId(RelativePath));
-		}
+		Latch WorkLatch(1);
 
-		virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName)
-		{
-			ZEN_ASSERT(!Parent.empty());
-			ZEN_ASSERT(!DirectoryName.empty());
-			FolderScanner* DataPtr = &Data;
-			Data.WorkLatch.AddCount(1);
-			Data.WorkerPool.ScheduleWork([DataPtr,
-										  RootDir	   = Parent / DirectoryName,
-										  RelativeRoot = RelativeRoot.empty() ? DirectoryName : RelativeRoot / DirectoryName]() {
-				auto _ = MakeGuard([DataPtr]() { DataPtr->WorkLatch.CountDown(); });
-				try
-				{
-					DataPtr->Traverse(RelativeRoot, RootDir);
-				}
-				catch (const std::exception& Ex)
-				{
-					ZEN_WARN("Exception while traversing path {} {}: {}", RelativeRoot, RootDir, Ex.what());
-				}
-			});
-			return false;
-		}
-	};
+		GetDirectoryContent(
+			Root,
+			DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes | DirectoryContentFlags::Recursive,
+			*this,
+			WorkerPool,
+			WorkLatch);
 
-	void FolderScanner::Traverse()
-	{
-		Stopwatch Timer;
-		Traverse({}, std::filesystem::absolute(Path));
 		WorkLatch.CountDown();
 		while (!WorkLatch.Wait(1000))
 		{
-			WorkLock.WithSharedLock([&]() { ZEN_INFO("Found {} files in '{}'...", FoundFiles.size(), Path.string()); });
+			WorkLock.WithSharedLock([&]() { ZEN_INFO("Found {} files in '{}'...", FoundFiles.size(), Root.string()); });
 		}
+
 		ZEN_ASSERT(FoundFiles.size() == FoundFileIds.size());
 		ZEN_INFO("Found {} files in '{}' in {}", FoundFiles.size(), Path.string(), NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
 	}
 
-	void FolderScanner::Traverse(const std::filesystem::path& RelativeRoot, const std::filesystem::path& AbsoluteRoot)
-	{
-		Visitor LeafVisitor(*this, RelativeRoot);
-		LeafVisitor.Traverser.TraverseFileSystem(AbsoluteRoot, LeafVisitor);
-		if (!LeafVisitor.Entries.empty())
-		{
-			WorkLock.WithExclusiveLock([&]() {
-				FoundFiles.insert(FoundFiles.end(), LeafVisitor.Entries.begin(), LeafVisitor.Entries.end());
-				FoundFileIds.insert(FoundFileIds.end(), LeafVisitor.FileIds.begin(), LeafVisitor.FileIds.end());
-			});
-		}
-	}
 }  // namespace
 
 std::unique_ptr<FolderStructure>