From b6bac83e8456905c37c8abb4f57fd45d61c37c72 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Wed, 22 Jan 2025 15:26:55 +0100
Subject: Add multithreading directory scanning in core/filesystem (#277)

add DirectoryContent::IncludeFileSizes
add DirectoryContent::IncludeAttributes
add multithreaded GetDirectoryContent
use multithreaded GetDirectoryContent in workspace folder scanning
---
 src/zencore/include/zencore/filesystem.h | 58 +++++++++++++++++++++++++++-----
 1 file changed, 49 insertions(+), 9 deletions(-)

(limited to 'src/zencore/include')

diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index dba4981f0..ca8682cd7 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -4,6 +4,7 @@
 
 #include "zencore.h"
 
+#include <zencore/enumflags.h>
 #include <zencore/iobuffer.h>
 #include <zencore/string.h>
 
@@ -12,8 +13,10 @@
 
 namespace zen {
 
-class IoBuffer;
 class CompositeBuffer;
+class IoBuffer;
+class Latch;
+class WorkerThreadPool;
 
 /** Delete directory (after deleting any contents)
  */
@@ -195,28 +198,65 @@ class FileSystemTraversal
 public:
 	struct TreeVisitor
 	{
-		using path_view	  = std::basic_string_view<std::filesystem::path::value_type>;
-		using path_string = std::filesystem::path::string_type;
+		using path_view = std::basic_string_view<std::filesystem::path::value_type>;
 
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0;
+		virtual void VisitFile(const std::filesystem::path& Parent,
+							   const path_view&				File,
+							   uint64_t						FileSize,
+							   uint32_t						NativeModeOrAttributes) = 0;
 
 		// This should return true if we should recurse into the directory
-		virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0;
+		virtual bool VisitDirectory(const std::filesystem::path& Parent,
+									const path_view&			 DirectoryName,
+									uint32_t					 NativeModeOrAttributes) = 0;
 	};
 
 	void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
 };
 
+enum class DirectoryContentFlags : uint8_t
+{
+	None			  = 0,
+	IncludeDirs		  = 1u << 0,
+	IncludeFiles	  = 1u << 1,
+	Recursive		  = 1u << 2,
+	IncludeFileSizes  = 1u << 3,
+	IncludeAttributes = 1u << 4,
+	IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive
+};
+
+ENUM_CLASS_FLAGS(DirectoryContentFlags)
+
 struct DirectoryContent
 {
-	static const uint8_t			   IncludeDirsFlag	= 1u << 0;
-	static const uint8_t			   IncludeFilesFlag = 1u << 1;
-	static const uint8_t			   RecursiveFlag	= 1u << 2;
 	std::vector<std::filesystem::path> Files;
+	std::vector<uint64_t>			   FileSizes;
+	std::vector<uint32_t>			   FileAttributes;
 	std::vector<std::filesystem::path> Directories;
+	std::vector<uint32_t>			   DirectoryAttributes;
+};
+
+void GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent);
+
+struct GetDirectoryContentVisitor
+{
+public:
+	struct DirectoryContent
+	{
+		std::vector<std::filesystem::path> FileNames;
+		std::vector<uint64_t>			   FileSizes;
+		std::vector<uint32_t>			   FileAttributes;
+		std::vector<std::filesystem::path> DirectoryNames;
+		std::vector<uint32_t>			   DirectoryAttributes;
+	};
+	virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) = 0;
 };
 
-void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent);
+void GetDirectoryContent(const std::filesystem::path& RootDir,
+						 DirectoryContentFlags		  Flags,
+						 GetDirectoryContentVisitor&  Visitor,
+						 WorkerThreadPool&			  WorkerPool,
+						 Latch&						  PendingWorkCount);
 
 std::string GetEnvVariable(std::string_view VariableName);
 
-- 
cgit v1.2.3