Add multithreading directory scanning in core/filesystem (#277)

add DirectoryContent::IncludeFileSizes
add DirectoryContent::IncludeAttributes
add multithreaded GetDirectoryContent
use multithreaded GetDirectoryContent in workspace folder scanning

16 files changed, 290 insertions, 131 deletions

diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 31e6886b2..995ed4136 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -737,14 +737,14 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	// Copy cache state
 	DirectoryContent CacheDirectoryContent;
-	GetDirectoryContent(CachePath, DirectoryContent::IncludeDirsFlag, CacheDirectoryContent);
+	GetDirectoryContent(CachePath, DirectoryContentFlags::IncludeDirs, CacheDirectoryContent);
 	for (const std::filesystem::path& NamespacePath : CacheDirectoryContent.Directories)
 	{
 		std::filesystem::path NamespaceName		  = NamespacePath.filename();
 		std::filesystem::path TargetNamespacePath = TargetCachePath / NamespaceName;
 
 		DirectoryContent CacheNamespaceContent;
-		GetDirectoryContent(NamespacePath, DirectoryContent::IncludeDirsFlag, CacheNamespaceContent);
+		GetDirectoryContent(NamespacePath, DirectoryContentFlags::IncludeDirs, CacheNamespaceContent);
 
 		for (const std::filesystem::path& BucketPath : CacheNamespaceContent.Directories)
 		{
diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp
index f39bfa71c..d42d3c107 100644
--- a/src/zen/cmds/copy_cmd.cpp
+++ b/src/zen/cmds/copy_cmd.cpp
@@ -120,7 +120,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			{
 			}
 
-			virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+			virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
 			{
 				ZEN_UNUSED(FileSize);
 				std::error_code				Ec;
@@ -157,7 +157,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				}
 			}
 
-			virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+			virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
 
 			std::filesystem::path BasePath;
 			std::filesystem::path TargetPath;
diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp
index ea9102b28..8e36e74ce 100644
--- a/src/zen/cmds/serve_cmd.cpp
+++ b/src/zen/cmds/serve_cmd.cpp
@@ -120,7 +120,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	struct FsVisitor : public FileSystemTraversal::TreeVisitor
 	{
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
 		{
 			std::filesystem::path ServerPath	   = std::filesystem::relative(Parent / File, RootPath);
 			std::string			  ServerPathString = reinterpret_cast<const char*>(ServerPath.generic_u8string().c_str());
@@ -133,7 +133,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			Files.emplace_back(FileEntry{ServerPathString, ServerPathString, FileSize});
 		}
 
-		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
 
 		struct FileEntry
 		{
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 52f2c4adc..b8c35212f 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -9,9 +9,11 @@
 #include <zencore/logging.h>
 #include <zencore/memory/memory.h>
 #include <zencore/process.h>
+#include <zencore/scopeguard.h>
 #include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/testing.h>
+#include <zencore/workthreadpool.h>
 
 #if ZEN_PLATFORM_WINDOWS
 #	include <zencore/windows.h>
@@ -681,7 +683,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 		{
 		}
 
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
 		{
 			std::error_code				Ec;
 			const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
@@ -727,7 +729,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 			}
 		}
 
-		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
 
 		std::filesystem::path BasePath;
 		std::filesystem::path TargetPath;
@@ -1215,7 +1217,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 				}
 				else
 				{
-					const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName);
+					const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
 
 					if (ShouldDescend)
 					{
@@ -1234,7 +1236,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 			}
 			else
 			{
-				Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
+				Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
 			}
 
 			const uint64_t NextOffset = DirInfo->NextEntryOffset;
@@ -1276,14 +1278,14 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 			{
 				/* nop */
 			}
-			else if (Visitor.VisitDirectory(RootDir, FileName))
+			else if (Visitor.VisitDirectory(RootDir, FileName, gsl::narrow<uint32_t>(Stat.st_mode)))
 			{
 				TraverseFileSystem(FullPath, Visitor);
 			}
 		}
 		else if (S_ISREG(Stat.st_mode))
 		{
-			Visitor.VisitFile(RootDir, FileName, Stat.st_size);
+			Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow<uint32_t>(Stat.st_mode));
 		}
 		else
 		{
@@ -1527,39 +1529,163 @@ MaximizeOpenFileCount()
 }
 
 void
-GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent)
+GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent)
 {
+	ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeDirs));
+	ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)
+				   ? true
+				   : (!EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes)));
+
 	FileSystemTraversal Traversal;
 	struct Visitor : public FileSystemTraversal::TreeVisitor
 	{
-		Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
+		Visitor(DirectoryContentFlags Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
 
-		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
-							   [[maybe_unused]] const path_view&			 File,
-							   [[maybe_unused]] uint64_t					 FileSize) override
+		virtual void VisitFile(const std::filesystem::path& Parent,
+							   const path_view&				File,
+							   uint64_t						FileSize,
+							   uint32_t						NativeModeOrAttributes) override
 		{
-			if (Flags & DirectoryContent::IncludeFilesFlag)
+			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
 			{
 				Content.Files.push_back(Parent / File);
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes))
+				{
+					Content.FileSizes.push_back(FileSize);
+				}
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+				{
+					Content.FileAttributes.push_back(NativeModeOrAttributes);
+				}
 			}
 		}
 
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
+									const path_view&							  DirectoryName,
+									uint32_t									  NativeModeOrAttributes) override
 		{
-			if (Flags & DirectoryContent::IncludeDirsFlag)
+			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeDirs))
 			{
 				Content.Directories.push_back(Parent / DirectoryName);
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+				{
+					Content.DirectoryAttributes.push_back(NativeModeOrAttributes);
+				}
 			}
-			return (Flags & DirectoryContent::RecursiveFlag) != 0;
+			return EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive);
 		}
 
-		const uint8_t	  Flags;
-		DirectoryContent& Content;
+		const DirectoryContentFlags Flags;
+		DirectoryContent&			Content;
 	} Visit(Flags, OutContent);
 
 	Traversal.TraverseFileSystem(RootDir, Visit);
 }
 
+void
+GetDirectoryContent(const std::filesystem::path& RootDir,
+					DirectoryContentFlags		 Flags,
+					GetDirectoryContentVisitor&	 Visitor,
+					WorkerThreadPool&			 WorkerPool,
+					Latch&						 PendingWorkCount)
+{
+	ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeDirs));
+	ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)
+				   ? true
+				   : (!EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes)));
+
+	struct MultithreadedVisitor : public FileSystemTraversal::TreeVisitor
+	{
+		MultithreadedVisitor(WorkerThreadPool&			  InWorkerPool,
+							 Latch&						  InOutPendingWorkCount,
+							 const std::filesystem::path& InRelativeRoot,
+							 DirectoryContentFlags		  InFlags,
+							 GetDirectoryContentVisitor*  InVisitor)
+		: WorkerPool(InWorkerPool)
+		, PendingWorkCount(InOutPendingWorkCount)
+		, RelativeRoot(InRelativeRoot)
+		, Flags(InFlags)
+		, Visitor(InVisitor)
+		{
+		}
+
+		virtual void VisitFile(const std::filesystem::path&,
+							   const path_view& File,
+							   uint64_t			FileSize,
+							   uint32_t			NativeModeOrAttributes) override
+		{
+			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
+			{
+				Content.FileNames.push_back(File);
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes))
+				{
+					Content.FileSizes.push_back(FileSize);
+				}
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+				{
+					Content.FileAttributes.push_back(NativeModeOrAttributes);
+				}
+			}
+		}
+
+		virtual bool VisitDirectory(const std::filesystem::path& Parent,
+									const path_view&			 DirectoryName,
+									uint32_t					 NativeModeOrAttributes) override
+		{
+			std::filesystem::path Path = Parent / DirectoryName;
+			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeDirs))
+			{
+				Content.DirectoryNames.push_back(DirectoryName);
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+				{
+					Content.DirectoryAttributes.push_back(NativeModeOrAttributes);
+				}
+			}
+			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive))
+			{
+				PendingWorkCount.AddCount(1);
+				try
+				{
+					WorkerPool.ScheduleWork([WorkerPool		  = &WorkerPool,
+											 PendingWorkCount = &PendingWorkCount,
+											 Visitor		  = Visitor,
+											 Flags			  = Flags,
+											 Path			  = std::move(Path),
+											 RelativeRoot	  = RelativeRoot / DirectoryName]() {
+						ZEN_ASSERT(Visitor);
+						auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
+						{
+							MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
+							FileSystemTraversal	 Traversal;
+							Traversal.TraverseFileSystem(Path, SubVisitor);
+							Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
+						}
+					});
+				}
+				catch (const std::exception Ex)
+				{
+					ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what());
+					PendingWorkCount.CountDown();
+				}
+			}
+			return false;
+		}
+
+		WorkerThreadPool& WorkerPool;
+		Latch&			  PendingWorkCount;
+
+		const std::filesystem::path RelativeRoot;
+		const DirectoryContentFlags Flags;
+
+		GetDirectoryContentVisitor::DirectoryContent Content;
+		GetDirectoryContentVisitor*					 Visitor;
+	} WrapperVisitor(WorkerPool, PendingWorkCount, {}, Flags, &Visitor);
+
+	FileSystemTraversal Traversal;
+	Traversal.TraverseFileSystem(RootDir, WrapperVisitor);
+	Visitor.AsyncVisitDirectory(WrapperVisitor.RelativeRoot, std::move(WrapperVisitor.Content));
+}
+
 std::string
 GetEnvVariable(std::string_view VariableName)
 {
@@ -1802,12 +1928,12 @@ TEST_CASE("filesystem")
 	// Traversal
 	struct : public FileSystemTraversal::TreeVisitor
 	{
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t) override
 		{
 			bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
 		}
 
-		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
 
 		bool				  bFoundExpected = false;
 		std::filesystem::path Expected;
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index dba4981f0..ca8682cd7 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -4,6 +4,7 @@
 
 #include "zencore.h"
 
+#include <zencore/enumflags.h>
 #include <zencore/iobuffer.h>
 #include <zencore/string.h>
 
@@ -12,8 +13,10 @@
 
 namespace zen {
 
-class IoBuffer;
 class CompositeBuffer;
+class IoBuffer;
+class Latch;
+class WorkerThreadPool;
 
 /** Delete directory (after deleting any contents)
  */
@@ -195,28 +198,65 @@ class FileSystemTraversal
 public:
 	struct TreeVisitor
 	{
-		using path_view	  = std::basic_string_view<std::filesystem::path::value_type>;
-		using path_string = std::filesystem::path::string_type;
+		using path_view = std::basic_string_view<std::filesystem::path::value_type>;
 
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0;
+		virtual void VisitFile(const std::filesystem::path& Parent,
+							   const path_view&				File,
+							   uint64_t						FileSize,
+							   uint32_t						NativeModeOrAttributes) = 0;
 
 		// This should return true if we should recurse into the directory
-		virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0;
+		virtual bool VisitDirectory(const std::filesystem::path& Parent,
+									const path_view&			 DirectoryName,
+									uint32_t					 NativeModeOrAttributes) = 0;
 	};
 
 	void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
 };
 
+enum class DirectoryContentFlags : uint8_t
+{
+	None			  = 0,
+	IncludeDirs		  = 1u << 0,
+	IncludeFiles	  = 1u << 1,
+	Recursive		  = 1u << 2,
+	IncludeFileSizes  = 1u << 3,
+	IncludeAttributes = 1u << 4,
+	IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive
+};
+
+ENUM_CLASS_FLAGS(DirectoryContentFlags)
+
 struct DirectoryContent
 {
-	static const uint8_t			   IncludeDirsFlag	= 1u << 0;
-	static const uint8_t			   IncludeFilesFlag = 1u << 1;
-	static const uint8_t			   RecursiveFlag	= 1u << 2;
 	std::vector<std::filesystem::path> Files;
+	std::vector<uint64_t>			   FileSizes;
+	std::vector<uint32_t>			   FileAttributes;
 	std::vector<std::filesystem::path> Directories;
+	std::vector<uint32_t>			   DirectoryAttributes;
+};
+
+void GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent);
+
+struct GetDirectoryContentVisitor
+{
+public:
+	struct DirectoryContent
+	{
+		std::vector<std::filesystem::path> FileNames;
+		std::vector<uint64_t>			   FileSizes;
+		std::vector<uint32_t>			   FileAttributes;
+		std::vector<std::filesystem::path> DirectoryNames;
+		std::vector<uint32_t>			   DirectoryAttributes;
+	};
+	virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) = 0;
 };
 
-void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent);
+void GetDirectoryContent(const std::filesystem::path& RootDir,
+						 DirectoryContentFlags		  Flags,
+						 GetDirectoryContentVisitor&  Visitor,
+						 WorkerThreadPool&			  WorkerPool,
+						 Latch&						  PendingWorkCount);
 
 std::string GetEnvVariable(std::string_view VariableName);
 
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index b1da034d2..c51e8f69d 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -1012,7 +1012,7 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 #if ZEN_PLATFORM_LINUX
 	std::vector<uint32_t> RunningPids;
 	DirectoryContent	  ProcList;
-	GetDirectoryContent("/proc", DirectoryContent::IncludeDirsFlag, ProcList);
+	GetDirectoryContent("/proc", DirectoryContentFlags::IncludeDirs, ProcList);
 	for (const std::filesystem::path& EntryPath : ProcList.Directories)
 	{
 		std::string				EntryName = EntryPath.stem();
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 847ed5a50..2888f5450 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -21,6 +21,7 @@
 #include <zenstore/gc.h>
 
 #include <zenstore/cache/structuredcachestore.h>
+#include <zenutil/workerpools.h>
 #include "config.h"
 #include "projectstore/projectstore.h"
 
@@ -41,31 +42,43 @@ GetStatsForDirectory(std::filesystem::path Dir)
 	if (!std::filesystem::exists(Dir))
 		return {};
 
-	FileSystemTraversal Traversal;
-
-	struct StatsTraversal : public FileSystemTraversal::TreeVisitor
+	struct StatsTraversal : public GetDirectoryContentVisitor
 	{
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
 		{
-			ZEN_UNUSED(Parent, File);
-			++TotalFileCount;
-			TotalBytes += FileSize;
+			ZEN_UNUSED(RelativeRoot);
+
+			uint64_t FileCount = Content.FileNames.size();
+			uint64_t DirCount  = Content.DirectoryNames.size();
+			uint64_t FilesSize = 0;
+			for (uint64_t FileSize : Content.FileSizes)
+			{
+				FilesSize += FileSize;
+			}
+			TotalBytes += FilesSize;
+			TotalFileCount += FileCount;
+			TotalDirCount += DirCount;
 		}
-		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override
+
+		std::atomic_uint64_t TotalBytes		= 0;
+		std::atomic_uint64_t TotalFileCount = 0;
+		std::atomic_uint64_t TotalDirCount	= 0;
+
+		DirStats GetStats()
 		{
-			++TotalDirCount;
-			return true;
+			return {.FileCount = TotalFileCount.load(), .DirCount = TotalDirCount.load(), .ByteCount = TotalBytes.load()};
 		}
+	} DirTraverser;
 
-		uint64_t TotalBytes		= 0;
-		uint64_t TotalFileCount = 0;
-		uint64_t TotalDirCount	= 0;
-
-		DirStats GetStats() { return {.FileCount = TotalFileCount, .DirCount = TotalDirCount, .ByteCount = TotalBytes}; }
-	};
+	Latch PendingWorkCount(1);
 
-	StatsTraversal DirTraverser;
-	Traversal.TraverseFileSystem(Dir, DirTraverser);
+	GetDirectoryContent(Dir,
+						DirectoryContentFlags::IncludeAllEntries | DirectoryContentFlags::IncludeFileSizes,
+						DirTraverser,
+						GetSmallWorkerPool(EWorkloadType::Burst),
+						PendingWorkCount);
+	PendingWorkCount.CountDown();
+	PendingWorkCount.Wait();
 
 	return DirTraverser.GetStats();
 }
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index f3ad71356..809092378 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -63,7 +63,7 @@ ReadAllCentralManifests(const std::filesystem::path& SystemRoot)
 	std::vector<CbObject> Manifests;
 
 	DirectoryContent Content;
-	GetDirectoryContent(SystemRoot / "States", DirectoryContent::IncludeFilesFlag, Content);
+	GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content);
 
 	for (std::filesystem::path& File : Content.Files)
 	{
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index 530efbc97..b0212ab07 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -376,7 +376,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 			Writer.BeginArray("Contents"sv);
 		}
 
-		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize) override
+		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
 		{
 			const fs::path FullPath		= Parent / fs::path(File);
 			fs::path	   RelativePath = fs::relative(FullPath, BucketPath);
@@ -390,7 +390,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 			Writer.EndObject();
 		}
 
-		bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return false; }
+		bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; }
 
 		CbObject GetResult()
 		{
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 9e9fc0e77..46a236af9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3506,7 +3506,7 @@ ProjectStore::Project::ScanForOplogs() const
 	if (Project::Exists(m_OplogStoragePath))
 	{
 		DirectoryContent DirContent;
-		GetDirectoryContent(m_OplogStoragePath, DirectoryContent::IncludeDirsFlag, DirContent);
+		GetDirectoryContent(m_OplogStoragePath, DirectoryContentFlags::IncludeDirs, DirContent);
 		Oplogs.reserve(DirContent.Directories.size());
 		for (const std::filesystem::path& DirPath : DirContent.Directories)
 		{
@@ -3859,7 +3859,7 @@ ProjectStore::DiscoverProjects()
 	}
 
 	DirectoryContent DirContent;
-	GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+	GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent);
 
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
@@ -3965,7 +3965,7 @@ ProjectStore::StorageSize() const
 		if (std::filesystem::exists(m_ProjectBasePath))
 		{
 			DirectoryContent ProjectsFolderContent;
-			GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, ProjectsFolderContent);
+			GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent);
 
 			for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories)
 			{
@@ -3974,7 +3974,7 @@ ProjectStore::StorageSize() const
 				{
 					Result.DiskSize += Project::TotalSize(ProjectBasePath);
 					DirectoryContent DirContent;
-					GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+					GetDirectoryContent(ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent);
 					for (const std::filesystem::path& OplogBasePath : DirContent.Directories)
 					{
 						Result.DiskSize += Oplog::TotalSize(OplogBasePath);
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 3974fb989..e976c061d 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -2,6 +2,7 @@
 
 #include <zenstore/blockstore.h>
 
+#include <zencore/enumflags.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -1427,8 +1428,8 @@ namespace blockstore::impl {
 	{
 		DirectoryContent DirectoryContent;
 		GetDirectoryContent(RootDir,
-							DirectoryContent::RecursiveFlag | (Files ? DirectoryContent::IncludeFilesFlag : 0) |
-								(Directories ? DirectoryContent::IncludeDirsFlag : 0),
+							DirectoryContentFlags::Recursive | (Files ? DirectoryContentFlags::IncludeFiles : DirectoryContentFlags::None) |
+								(Directories ? DirectoryContentFlags::IncludeDirs : DirectoryContentFlags::None),
 							DirectoryContent);
 		std::vector<std::filesystem::path> Result;
 		Result.insert(Result.end(), DirectoryContent.Directories.begin(), DirectoryContent.Directories.end());
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index cbc1d6e83..25f68330a 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -3858,7 +3858,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 	ZEN_TRACE_CPU("Z$::DiscoverBuckets");
 
 	DirectoryContent DirContent;
-	GetDirectoryContent(m_RootDir, DirectoryContent::IncludeDirsFlag, DirContent);
+	GetDirectoryContent(m_RootDir, DirectoryContentFlags::IncludeDirs, DirContent);
 
 	// Initialize buckets
 
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index c14ea73a8..133cb42d7 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -424,7 +424,7 @@ ZenCacheStore::ZenCacheStore(GcManager&					  Gc,
 	ZEN_INFO("initializing cache store at '{}'", m_BasePath);
 
 	DirectoryContent DirContent;
-	GetDirectoryContent(m_BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+	GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent);
 
 	std::vector<std::string> Namespaces;
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
diff --git a/src/zenstore/chunkedfile.cpp b/src/zenstore/chunkedfile.cpp
index 0f3cf5112..f200bc1ec 100644
--- a/src/zenstore/chunkedfile.cpp
+++ b/src/zenstore/chunkedfile.cpp
@@ -195,10 +195,10 @@ TEST_CASE("chunkedfile.findparams")
 {
 #		if 1
 	DirectoryContent SourceContent1;
-	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContent::IncludeFilesFlag, SourceContent1);
+	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContentFlags::IncludeFiles, SourceContent1);
 	const std::vector<std::filesystem::path>& SourceFiles1 = SourceContent1.Files;
 	DirectoryContent						  SourceContent2;
-	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContent::IncludeFilesFlag, SourceContent2);
+	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContentFlags::IncludeFiles, SourceContent2);
 	const std::vector<std::filesystem::path>& SourceFiles2 = SourceContent2.Files;
 #		else
 	std::filesystem::path SourcePath1 =
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 110baaf5f..14123528c 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -185,7 +185,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 			// in this folder as well
 			struct Visitor : public FileSystemTraversal::TreeVisitor
 			{
-				virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override
+				virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t, uint32_t) override
 				{
 					// We don't care about files
 				}
@@ -193,8 +193,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 				{
 					return std::find(&HexChars[0], &HexChars[16], C) != &HexChars[16];
 				}
-				virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
-											[[maybe_unused]] const path_view&			  DirectoryName) override
+				virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName, uint32_t) override
 				{
 					if (DirectoryName.length() == 3)
 					{
@@ -1175,7 +1174,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
 	struct Visitor : public FileSystemTraversal::TreeVisitor
 	{
 		Visitor(const std::filesystem::path& RootDir, std::vector<FileCasIndexEntry>& Entries) : RootDirectory(RootDir), Entries(Entries) {}
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
 		{
 			std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory);
 
@@ -1201,11 +1200,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
 			}
 		}
 
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
-									[[maybe_unused]] const path_view&			  DirectoryName) override
-		{
-			return true;
-		}
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
 
 		const std::filesystem::path&	RootDirectory;
 		std::vector<FileCasIndexEntry>& Entries;
diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp
index d399ffc88..02a83d2a6 100644
--- a/src/zenstore/workspaces.cpp
+++ b/src/zenstore/workspaces.cpp
@@ -252,18 +252,43 @@ FolderStructure::FolderStructure(std::vector<FileEntry>&& InEntries, std::vector
 }
 
 namespace {
-	struct FolderScanner
+	struct FolderScanner : public GetDirectoryContentVisitor
 	{
 		FolderScanner(LoggerRef& Log, WorkerThreadPool& WorkerPool, const std::filesystem::path& Path)
 		: m_Log(Log)
 		, Path(Path)
-		, WorkLatch(1)
 		, WorkerPool(WorkerPool)
 		{
 		}
 
 		void Traverse();
-		void Traverse(const std::filesystem::path& RelativeRoot, const std::filesystem::path& Path);
+
+		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+		{
+			std::vector<FolderStructure::FileEntry> FileEntries;
+			std::vector<Oid>						PathIds;
+			const size_t							FileCount = Content.FileNames.size();
+			FileEntries.reserve(FileCount);
+			PathIds.reserve(FileCount);
+
+			auto FileNameIt = Content.FileNames.begin();
+			auto FileSizeIt = Content.FileSizes.begin();
+			while (FileNameIt != Content.FileNames.end())
+			{
+				ZEN_ASSERT_SLOW(FileSizeIt != Content.FileSizes.end());
+
+				std::filesystem::path RelativePath = RelativeRoot.empty() ? *FileNameIt : RelativeRoot / *FileNameIt;
+				PathIds.emplace_back(Workspaces::PathToId(RelativePath));
+				FileEntries.emplace_back(FolderStructure::FileEntry{.RelativePath = std::move(RelativePath), .Size = *FileSizeIt});
+
+				FileNameIt++;
+				FileSizeIt++;
+			}
+			WorkLock.WithExclusiveLock([&]() {
+				FoundFiles.insert(FoundFiles.end(), FileEntries.begin(), FileEntries.end());
+				FoundFileIds.insert(FoundFileIds.end(), PathIds.begin(), PathIds.end());
+			});
+		}
 
 		LoggerRef&								Log() { return m_Log; }
 		LoggerRef&								m_Log;
@@ -271,75 +296,34 @@ namespace {
 		RwLock									WorkLock;
 		std::vector<FolderStructure::FileEntry> FoundFiles;
 		std::vector<Oid>						FoundFileIds;
-		Latch									WorkLatch;
 		WorkerThreadPool&						WorkerPool;
 	};
 
-	struct Visitor : public FileSystemTraversal::TreeVisitor
+	void FolderScanner::Traverse()
 	{
-		Visitor(FolderScanner& Data, const std::filesystem::path& RelativeRoot) : Data(Data), RelativeRoot(RelativeRoot) {}
+		Stopwatch Timer;
 
-		FileSystemTraversal						Traverser;
-		FolderScanner&							Data;
-		std::vector<FolderStructure::FileEntry> Entries;
-		std::vector<Oid>						FileIds;
-		std::filesystem::path					RelativeRoot;
+		const std::filesystem::path Root = std::filesystem::absolute(Path);
 
-		virtual void VisitFile(const std::filesystem::path&, const path_view& File, uint64_t FileSize)
-		{
-			std::filesystem::path RelativePath = RelativeRoot.empty() ? File : RelativeRoot / File;
-			Entries.push_back(FolderStructure::FileEntry{.RelativePath = RelativePath, .Size = FileSize});
-			FileIds.push_back(Workspaces::PathToId(RelativePath));
-		}
+		Latch WorkLatch(1);
 
-		virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName)
-		{
-			ZEN_ASSERT(!Parent.empty());
-			ZEN_ASSERT(!DirectoryName.empty());
-			FolderScanner* DataPtr = &Data;
-			Data.WorkLatch.AddCount(1);
-			Data.WorkerPool.ScheduleWork([DataPtr,
-										  RootDir	   = Parent / DirectoryName,
-										  RelativeRoot = RelativeRoot.empty() ? DirectoryName : RelativeRoot / DirectoryName]() {
-				auto _ = MakeGuard([DataPtr]() { DataPtr->WorkLatch.CountDown(); });
-				try
-				{
-					DataPtr->Traverse(RelativeRoot, RootDir);
-				}
-				catch (const std::exception& Ex)
-				{
-					ZEN_WARN("Exception while traversing path {} {}: {}", RelativeRoot, RootDir, Ex.what());
-				}
-			});
-			return false;
-		}
-	};
+		GetDirectoryContent(
+			Root,
+			DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes | DirectoryContentFlags::Recursive,
+			*this,
+			WorkerPool,
+			WorkLatch);
 
-	void FolderScanner::Traverse()
-	{
-		Stopwatch Timer;
-		Traverse({}, std::filesystem::absolute(Path));
 		WorkLatch.CountDown();
 		while (!WorkLatch.Wait(1000))
 		{
-			WorkLock.WithSharedLock([&]() { ZEN_INFO("Found {} files in '{}'...", FoundFiles.size(), Path.string()); });
+			WorkLock.WithSharedLock([&]() { ZEN_INFO("Found {} files in '{}'...", FoundFiles.size(), Root.string()); });
 		}
+
 		ZEN_ASSERT(FoundFiles.size() == FoundFileIds.size());
 		ZEN_INFO("Found {} files in '{}' in {}", FoundFiles.size(), Path.string(), NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
 	}
 
-	void FolderScanner::Traverse(const std::filesystem::path& RelativeRoot, const std::filesystem::path& AbsoluteRoot)
-	{
-		Visitor LeafVisitor(*this, RelativeRoot);
-		LeafVisitor.Traverser.TraverseFileSystem(AbsoluteRoot, LeafVisitor);
-		if (!LeafVisitor.Entries.empty())
-		{
-			WorkLock.WithExclusiveLock([&]() {
-				FoundFiles.insert(FoundFiles.end(), LeafVisitor.Entries.begin(), LeafVisitor.Entries.end());
-				FoundFileIds.insert(FoundFileIds.end(), LeafVisitor.FileIds.begin(), LeafVisitor.FileIds.end());
-			});
-		}
-	}
 }  // namespace
 
 std::unique_ptr<FolderStructure>