aboutsummaryrefslogtreecommitdiff
path: root/src/zencore/filesystem.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zencore/filesystem.cpp')
-rw-r--r--src/zencore/filesystem.cpp164
1 files changed, 145 insertions, 19 deletions
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 52f2c4adc..b8c35212f 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -9,9 +9,11 @@
#include <zencore/logging.h>
#include <zencore/memory/memory.h>
#include <zencore/process.h>
+#include <zencore/scopeguard.h>
#include <zencore/stream.h>
#include <zencore/string.h>
#include <zencore/testing.h>
+#include <zencore/workthreadpool.h>
#if ZEN_PLATFORM_WINDOWS
# include <zencore/windows.h>
@@ -681,7 +683,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
{
}
- virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
{
std::error_code Ec;
const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
@@ -727,7 +729,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
}
}
- virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
std::filesystem::path BasePath;
std::filesystem::path TargetPath;
@@ -1215,7 +1217,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
}
else
{
- const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName);
+ const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
if (ShouldDescend)
{
@@ -1234,7 +1236,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
}
else
{
- Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
+ Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
}
const uint64_t NextOffset = DirInfo->NextEntryOffset;
@@ -1276,14 +1278,14 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
{
/* nop */
}
- else if (Visitor.VisitDirectory(RootDir, FileName))
+ else if (Visitor.VisitDirectory(RootDir, FileName, gsl::narrow<uint32_t>(Stat.st_mode)))
{
TraverseFileSystem(FullPath, Visitor);
}
}
else if (S_ISREG(Stat.st_mode))
{
- Visitor.VisitFile(RootDir, FileName, Stat.st_size);
+ Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow<uint32_t>(Stat.st_mode));
}
else
{
@@ -1527,39 +1529,163 @@ MaximizeOpenFileCount()
}
void
-GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent)
+GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent)
{
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeDirs));
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)
+ ? true
+ : (!EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes)));
+
FileSystemTraversal Traversal;
struct Visitor : public FileSystemTraversal::TreeVisitor
{
- Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
+ Visitor(DirectoryContentFlags Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
- virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
- [[maybe_unused]] const path_view& File,
- [[maybe_unused]] uint64_t FileSize) override
+ virtual void VisitFile(const std::filesystem::path& Parent,
+ const path_view& File,
+ uint64_t FileSize,
+ uint32_t NativeModeOrAttributes) override
{
- if (Flags & DirectoryContent::IncludeFilesFlag)
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
{
Content.Files.push_back(Parent / File);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes))
+ {
+ Content.FileSizes.push_back(FileSize);
+ }
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.FileAttributes.push_back(NativeModeOrAttributes);
+ }
}
}
- virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+ virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
+ const path_view& DirectoryName,
+ uint32_t NativeModeOrAttributes) override
{
- if (Flags & DirectoryContent::IncludeDirsFlag)
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeDirs))
{
Content.Directories.push_back(Parent / DirectoryName);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.DirectoryAttributes.push_back(NativeModeOrAttributes);
+ }
}
- return (Flags & DirectoryContent::RecursiveFlag) != 0;
+ return EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive);
}
- const uint8_t Flags;
- DirectoryContent& Content;
+ const DirectoryContentFlags Flags;
+ DirectoryContent& Content;
} Visit(Flags, OutContent);
Traversal.TraverseFileSystem(RootDir, Visit);
}
+void
+GetDirectoryContent(const std::filesystem::path& RootDir,
+ DirectoryContentFlags Flags,
+ GetDirectoryContentVisitor& Visitor,
+ WorkerThreadPool& WorkerPool,
+ Latch& PendingWorkCount)
+{
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeDirs));
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)
+ ? true
+ : (!EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes)));
+
+ struct MultithreadedVisitor : public FileSystemTraversal::TreeVisitor
+ {
+ MultithreadedVisitor(WorkerThreadPool& InWorkerPool,
+ Latch& InOutPendingWorkCount,
+ const std::filesystem::path& InRelativeRoot,
+ DirectoryContentFlags InFlags,
+ GetDirectoryContentVisitor* InVisitor)
+ : WorkerPool(InWorkerPool)
+ , PendingWorkCount(InOutPendingWorkCount)
+ , RelativeRoot(InRelativeRoot)
+ , Flags(InFlags)
+ , Visitor(InVisitor)
+ {
+ }
+
+ virtual void VisitFile(const std::filesystem::path&,
+ const path_view& File,
+ uint64_t FileSize,
+ uint32_t NativeModeOrAttributes) override
+ {
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
+ {
+ Content.FileNames.push_back(File);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes))
+ {
+ Content.FileSizes.push_back(FileSize);
+ }
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.FileAttributes.push_back(NativeModeOrAttributes);
+ }
+ }
+ }
+
+ virtual bool VisitDirectory(const std::filesystem::path& Parent,
+ const path_view& DirectoryName,
+ uint32_t NativeModeOrAttributes) override
+ {
+ std::filesystem::path Path = Parent / DirectoryName;
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeDirs))
+ {
+ Content.DirectoryNames.push_back(DirectoryName);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.DirectoryAttributes.push_back(NativeModeOrAttributes);
+ }
+ }
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive))
+ {
+ PendingWorkCount.AddCount(1);
+ try
+ {
+ WorkerPool.ScheduleWork([WorkerPool = &WorkerPool,
+ PendingWorkCount = &PendingWorkCount,
+ Visitor = Visitor,
+ Flags = Flags,
+ Path = std::move(Path),
+ RelativeRoot = RelativeRoot / DirectoryName]() {
+ ZEN_ASSERT(Visitor);
+ auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
+ {
+ MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
+ FileSystemTraversal Traversal;
+ Traversal.TraverseFileSystem(Path, SubVisitor);
+ Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
+ }
+ });
+ }
+ catch (const std::exception Ex)
+ {
+ ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what());
+ PendingWorkCount.CountDown();
+ }
+ }
+ return false;
+ }
+
+ WorkerThreadPool& WorkerPool;
+ Latch& PendingWorkCount;
+
+ const std::filesystem::path RelativeRoot;
+ const DirectoryContentFlags Flags;
+
+ GetDirectoryContentVisitor::DirectoryContent Content;
+ GetDirectoryContentVisitor* Visitor;
+ } WrapperVisitor(WorkerPool, PendingWorkCount, {}, Flags, &Visitor);
+
+ FileSystemTraversal Traversal;
+ Traversal.TraverseFileSystem(RootDir, WrapperVisitor);
+ Visitor.AsyncVisitDirectory(WrapperVisitor.RelativeRoot, std::move(WrapperVisitor.Content));
+}
+
std::string
GetEnvVariable(std::string_view VariableName)
{
@@ -1802,12 +1928,12 @@ TEST_CASE("filesystem")
// Traversal
struct : public FileSystemTraversal::TreeVisitor
{
- virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t) override
{
bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
}
- virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
bool bFoundExpected = false;
std::filesystem::path Expected;
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-11 12:43:30 +0000