Add multithreading directory scanning in core/filesystem (#277)

add DirectoryContent::IncludeFileSizes
add DirectoryContent::IncludeAttributes
add multithreaded GetDirectoryContent
use multithreaded GetDirectoryContent in workspace folder scanning

4 files changed, 38 insertions, 25 deletions

diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 847ed5a50..2888f5450 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -21,6 +21,7 @@
 #include <zenstore/gc.h>
 
 #include <zenstore/cache/structuredcachestore.h>
+#include <zenutil/workerpools.h>
 #include "config.h"
 #include "projectstore/projectstore.h"
 
@@ -41,31 +42,43 @@ GetStatsForDirectory(std::filesystem::path Dir)
 	if (!std::filesystem::exists(Dir))
 		return {};
 
-	FileSystemTraversal Traversal;
-
-	struct StatsTraversal : public FileSystemTraversal::TreeVisitor
+	struct StatsTraversal : public GetDirectoryContentVisitor
 	{
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
 		{
-			ZEN_UNUSED(Parent, File);
-			++TotalFileCount;
-			TotalBytes += FileSize;
+			ZEN_UNUSED(RelativeRoot);
+
+			uint64_t FileCount = Content.FileNames.size();
+			uint64_t DirCount  = Content.DirectoryNames.size();
+			uint64_t FilesSize = 0;
+			for (uint64_t FileSize : Content.FileSizes)
+			{
+				FilesSize += FileSize;
+			}
+			TotalBytes += FilesSize;
+			TotalFileCount += FileCount;
+			TotalDirCount += DirCount;
 		}
-		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override
+
+		std::atomic_uint64_t TotalBytes		= 0;
+		std::atomic_uint64_t TotalFileCount = 0;
+		std::atomic_uint64_t TotalDirCount	= 0;
+
+		DirStats GetStats()
 		{
-			++TotalDirCount;
-			return true;
+			return {.FileCount = TotalFileCount.load(), .DirCount = TotalDirCount.load(), .ByteCount = TotalBytes.load()};
 		}
+	} DirTraverser;
 
-		uint64_t TotalBytes		= 0;
-		uint64_t TotalFileCount = 0;
-		uint64_t TotalDirCount	= 0;
-
-		DirStats GetStats() { return {.FileCount = TotalFileCount, .DirCount = TotalDirCount, .ByteCount = TotalBytes}; }
-	};
+	Latch PendingWorkCount(1);
 
-	StatsTraversal DirTraverser;
-	Traversal.TraverseFileSystem(Dir, DirTraverser);
+	GetDirectoryContent(Dir,
+						DirectoryContentFlags::IncludeAllEntries | DirectoryContentFlags::IncludeFileSizes,
+						DirTraverser,
+						GetSmallWorkerPool(EWorkloadType::Burst),
+						PendingWorkCount);
+	PendingWorkCount.CountDown();
+	PendingWorkCount.Wait();
 
 	return DirTraverser.GetStats();
 }
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index f3ad71356..809092378 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -63,7 +63,7 @@ ReadAllCentralManifests(const std::filesystem::path& SystemRoot)
 	std::vector<CbObject> Manifests;
 
 	DirectoryContent Content;
-	GetDirectoryContent(SystemRoot / "States", DirectoryContent::IncludeFilesFlag, Content);
+	GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content);
 
 	for (std::filesystem::path& File : Content.Files)
 	{
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index 530efbc97..b0212ab07 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -376,7 +376,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 			Writer.BeginArray("Contents"sv);
 		}
 
-		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize) override
+		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
 		{
 			const fs::path FullPath		= Parent / fs::path(File);
 			fs::path	   RelativePath = fs::relative(FullPath, BucketPath);
@@ -390,7 +390,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 			Writer.EndObject();
 		}
 
-		bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return false; }
+		bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; }
 
 		CbObject GetResult()
 		{
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 9e9fc0e77..46a236af9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3506,7 +3506,7 @@ ProjectStore::Project::ScanForOplogs() const
 	if (Project::Exists(m_OplogStoragePath))
 	{
 		DirectoryContent DirContent;
-		GetDirectoryContent(m_OplogStoragePath, DirectoryContent::IncludeDirsFlag, DirContent);
+		GetDirectoryContent(m_OplogStoragePath, DirectoryContentFlags::IncludeDirs, DirContent);
 		Oplogs.reserve(DirContent.Directories.size());
 		for (const std::filesystem::path& DirPath : DirContent.Directories)
 		{
@@ -3859,7 +3859,7 @@ ProjectStore::DiscoverProjects()
 	}
 
 	DirectoryContent DirContent;
-	GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+	GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent);
 
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
@@ -3965,7 +3965,7 @@ ProjectStore::StorageSize() const
 		if (std::filesystem::exists(m_ProjectBasePath))
 		{
 			DirectoryContent ProjectsFolderContent;
-			GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, ProjectsFolderContent);
+			GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent);
 
 			for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories)
 			{
@@ -3974,7 +3974,7 @@ ProjectStore::StorageSize() const
 				{
 					Result.DiskSize += Project::TotalSize(ProjectBasePath);
 					DirectoryContent DirContent;
-					GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+					GetDirectoryContent(ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent);
 					for (const std::filesystem::path& OplogBasePath : DirContent.Directories)
 					{
 						Result.DiskSize += Oplog::TotalSize(OplogBasePath);