From 0e20ec796d79650e9e9ebf5c2561b1633b4b2020 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 13 Jan 2026 15:00:28 +0100 Subject: added options to configure exclude folders and extensions to zen build commands (#706) * added `--exclude-folders` to `zen upload`, `zen download` and `zen diff` added `--exclude-extensions` to `zen upload` and `zen diff` excluded folder names are now matched by folder name in subfolders in addition to root level folders * allow multiple token separators --- src/zenremotestore/filesystemutils.cpp | 367 ++++++++++++++++++++------------- 1 file changed, 218 insertions(+), 149 deletions(-) (limited to 'src/zenremotestore/filesystemutils.cpp') diff --git a/src/zenremotestore/filesystemutils.cpp b/src/zenremotestore/filesystemutils.cpp index 8dff05c6b..fa1ce6f78 100644 --- a/src/zenremotestore/filesystemutils.cpp +++ b/src/zenremotestore/filesystemutils.cpp @@ -11,6 +11,11 @@ #include #include +#if ZEN_WITH_TESTS +# include +# include +#endif // ZEN_WITH_TESTS + namespace zen { BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path, @@ -349,13 +354,14 @@ CleanDirectory( std::atomic DeletedItemCount = 0; std::atomic DeletedByteCount = 0; - CleanDirectoryResult Result; - RwLock ResultLock; - auto _ = MakeGuard([&]() { - Result.DeletedCount = DeletedItemCount.load(); - Result.DeletedByteCount = DeletedByteCount.load(); - Result.FoundCount = DiscoveredItemCount.load(); - }); + std::vector DirectoriesToDelete; + CleanDirectoryResult Result; + RwLock ResultLock; + auto _ = MakeGuard([&]() { + Result.DeletedCount = DeletedItemCount.load(); + Result.DeletedByteCount = DeletedByteCount.load(); + Result.FoundCount = DiscoveredItemCount.load(); + }); ParallelWork Work(AbortFlag, PauseFlag, @@ -363,119 +369,133 @@ CleanDirectory( struct AsyncVisitor : public GetDirectoryContentVisitor { - AsyncVisitor(const std::filesystem::path& InPath, - std::atomic& InAbortFlag, - std::atomic& InDiscoveredItemCount, - std::atomic& InDeletedItemCount, - std::atomic& InDeletedByteCount, - std::span InExcludeDirectories, - CleanDirectoryResult& InResult, - RwLock& InResultLock) + AsyncVisitor(const std::filesystem::path& InPath, + std::atomic& InAbortFlag, + std::atomic& InDiscoveredItemCount, + std::atomic& InDeletedItemCount, + std::atomic& InDeletedByteCount, + std::span InExcludeDirectories, + std::vector& OutDirectoriesToDelete, + CleanDirectoryResult& InResult, + RwLock& InResultLock) : Path(InPath) , AbortFlag(InAbortFlag) , DiscoveredItemCount(InDiscoveredItemCount) , DeletedItemCount(InDeletedItemCount) , DeletedByteCount(InDeletedByteCount) , ExcludeDirectories(InExcludeDirectories) + , DirectoriesToDelete(OutDirectoriesToDelete) , Result(InResult) , ResultLock(InResultLock) { } + + virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const override + { + ZEN_UNUSED(Parent); + + if (AbortFlag) + { + return false; + } + const std::string DirectoryString = DirectoryName.string(); + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (DirectoryString == ExcludeDirectory) + { + return false; + } + } + return true; + } + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); if (!AbortFlag) { - if (!Content.FileNames.empty()) + DiscoveredItemCount += Content.FileNames.size(); + + ZEN_TRACE_CPU("DeleteFiles"); + std::vector> FailedRemovePaths; + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) { - DiscoveredItemCount += Content.FileNames.size(); + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); - const std::string RelativeRootString = RelativeRoot.generic_string(); - bool RemoveContent = true; - for (const std::string_view ExcludeDirectory : ExcludeDirectories) + bool IsRemoved = false; + std::error_code Ec; + (void)SetFileReadOnly(FilePath, false, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) { - if (RelativeRootString.starts_with(ExcludeDirectory)) + if (!IsFileWithRetry(FilePath)) { - if (RelativeRootString.length() > ExcludeDirectory.length()) - { - const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()]; - if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' || - MaybePathDelimiter == std::filesystem::path::preferred_separator) - { - RemoveContent = false; - break; - } - } - else - { - RemoveContent = false; - break; - } + IsRemoved = true; + Ec.clear(); + break; } + Sleep(100 + int(Retries * 50)); + Ec.clear(); + (void)SetFileReadOnly(FilePath, false, Ec); } - if (RemoveContent) + if (!IsRemoved && !Ec) { - ZEN_TRACE_CPU("DeleteFiles"); - for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + (void)RemoveFile(FilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 6; Retries++) { - const std::filesystem::path& FileName = Content.FileNames[FileIndex]; - const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); - - bool IsRemoved = false; - std::error_code Ec; - (void)SetFileReadOnly(FilePath, false, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + if (!IsFileWithRetry(FilePath)) { - if (!IsFileWithRetry(FilePath)) - { - IsRemoved = true; - Ec.clear(); - break; - } - Sleep(100 + int(Retries * 50)); + IsRemoved = true; Ec.clear(); - (void)SetFileReadOnly(FilePath, false, Ec); - } - if (!IsRemoved && !Ec) - { - (void)RemoveFile(FilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 6; Retries++) - { - if (!IsFileWithRetry(FilePath)) - { - IsRemoved = true; - Ec.clear(); - return; - } - Sleep(100 + int(Retries * 50)); - Ec.clear(); - (void)RemoveFile(FilePath, Ec); - } - } - if (!IsRemoved && Ec) - { - RwLock::ExclusiveLockScope _(ResultLock); - Result.FailedRemovePaths.push_back(std::make_pair(FilePath, Ec)); - } - else - { - DeletedItemCount++; - DeletedByteCount += Content.FileSizes[FileIndex]; + break; } + Sleep(100 + int(Retries * 50)); + Ec.clear(); + (void)RemoveFile(FilePath, Ec); } } + if (!IsRemoved && Ec) + { + FailedRemovePaths.push_back(std::make_pair(FilePath, Ec)); + } + else + { + DeletedItemCount++; + DeletedByteCount += Content.FileSizes[FileIndex]; + } + } + + if (!FailedRemovePaths.empty()) + { + RwLock::ExclusiveLockScope _(ResultLock); + FailedRemovePaths.insert(FailedRemovePaths.end(), FailedRemovePaths.begin(), FailedRemovePaths.end()); + } + else if (!RelativeRoot.empty()) + { + DiscoveredItemCount++; + RwLock::ExclusiveLockScope _(ResultLock); + DirectoriesToDelete.push_back(RelativeRoot); } } } - const std::filesystem::path& Path; - std::atomic& AbortFlag; - std::atomic& DiscoveredItemCount; - std::atomic& DeletedItemCount; - std::atomic& DeletedByteCount; - std::span ExcludeDirectories; - CleanDirectoryResult& Result; - RwLock& ResultLock; - } Visitor(Path, AbortFlag, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories, Result, ResultLock); + const std::filesystem::path& Path; + std::atomic& AbortFlag; + std::atomic& DiscoveredItemCount; + std::atomic& DeletedItemCount; + std::atomic& DeletedByteCount; + std::span ExcludeDirectories; + std::vector& DirectoriesToDelete; + CleanDirectoryResult& Result; + RwLock& ResultLock; + } Visitor(Path, + AbortFlag, + DiscoveredItemCount, + DeletedItemCount, + DeletedByteCount, + ExcludeDirectories, + DirectoriesToDelete, + Result, + ResultLock); GetDirectoryContent(Path, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes, @@ -483,29 +503,6 @@ CleanDirectory( IOWorkerPool, Work.PendingWork()); - DirectoryContent LocalDirectoryContent; - GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); - DiscoveredItemCount += LocalDirectoryContent.Directories.size(); - std::vector DirectoriesToDelete; - DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size()); - for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) - { - bool Leave = false; - for (const std::string_view ExcludeDirectory : ExcludeDirectories) - { - if (LocalDirPath == (Path / ExcludeDirectory)) - { - Leave = true; - break; - } - } - if (!Leave) - { - DirectoriesToDelete.emplace_back(std::move(LocalDirPath)); - DiscoveredItemCount++; - } - } - uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); if (ProgressFunc && ProgressUpdateDelayMS != 0) @@ -528,6 +525,15 @@ CleanDirectory( { ZEN_TRACE_CPU("DeleteDirs"); + + std::sort(DirectoriesToDelete.begin(), + DirectoriesToDelete.end(), + [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) { + auto DistanceLhs = std::distance(Lhs.begin(), Lhs.end()); + auto DistanceRhs = std::distance(Rhs.begin(), Rhs.end()); + return DistanceLhs > DistanceRhs; + }); + for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete) { if (AbortFlag) @@ -542,53 +548,48 @@ CleanDirectory( } } - { - std::error_code Ec; - zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec); - if (Ec) - { - Sleep(200); - Ec.clear(); - zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec); - } + const std::filesystem::path FullPath = Path / DirectoryToDelete; - if (!Ec) + std::error_code Ec; + RemoveDir(FullPath, Ec); + if (Ec) + { + for (size_t Retries = 0; Ec && Retries < 3; Retries++) { - RemoveDir(DirectoryToDelete, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + if (!IsDir(FullPath)) { - if (!IsDir(DirectoryToDelete)) - { - Ec.clear(); - break; - } - Sleep(100 + int(Retries * 50)); Ec.clear(); - RemoveDir(DirectoryToDelete, Ec); + break; } - } - if (Ec) - { - RwLock::ExclusiveLockScope __(ResultLock); - Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec)); - } - else - { - DeletedItemCount++; + Sleep(100 + int(Retries * 50)); + Ec.clear(); + RemoveDir(FullPath, Ec); } } + if (Ec) + { + RwLock::ExclusiveLockScope __(ResultLock); + Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec)); + } + else + { + DeletedItemCount++; + } - uint64_t NowMs = Timer.GetElapsedTimeMs(); - - if ((NowMs - LastUpdateTimeMs) >= ProgressUpdateDelayMS) + if (ProgressFunc) { - LastUpdateTimeMs = NowMs; + uint64_t NowMs = Timer.GetElapsedTimeMs(); + + if ((NowMs - LastUpdateTimeMs) > 0) + { + LastUpdateTimeMs = NowMs; - uint64_t Deleted = DeletedItemCount.load(); - uint64_t DeletedBytes = DeletedByteCount.load(); - uint64_t Discovered = DiscoveredItemCount.load(); - std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); - ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag); + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); + ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag); + } } } } @@ -625,4 +626,72 @@ CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, return false; } +#if ZEN_WITH_TESTS + +void +filesystemutils_forcelink() +{ +} + +namespace { + void GenerateFile(const std::filesystem::path& Path) { BasicFile _(Path, BasicFile::Mode::kTruncate); } +} // namespace + +TEST_CASE("filesystemutils.CleanDirectory") +{ + ScopedTemporaryDirectory TmpDir; + + CreateDirectories(TmpDir.Path() / ".keepme"); + GenerateFile(TmpDir.Path() / ".keepme" / "keep"); + GenerateFile(TmpDir.Path() / "deleteme1"); + GenerateFile(TmpDir.Path() / "deleteme2"); + GenerateFile(TmpDir.Path() / "deleteme3"); + CreateDirectories(TmpDir.Path() / ".keepmenot"); + CreateDirectories(TmpDir.Path() / "no.keepme"); + + CreateDirectories(TmpDir.Path() / "DeleteMe"); + GenerateFile(TmpDir.Path() / "DeleteMe" / "delete1"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete1"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete3"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe" / ".keepme"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete3"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept"); + GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1"); + GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme"); + + WorkerThreadPool Pool(4); + std::atomic AbortFlag; + std::atomic PauseFlag; + + CleanDirectory(Pool, AbortFlag, PauseFlag, TmpDir.Path(), std::vector{".keepme"}, {}, 0); + + CHECK(IsDir(TmpDir.Path() / ".keepme")); + CHECK(IsFile(TmpDir.Path() / ".keepme" / "keep")); + CHECK(!IsFile(TmpDir.Path() / "deleteme1")); + CHECK(!IsFile(TmpDir.Path() / "deleteme2")); + CHECK(!IsFile(TmpDir.Path() / "deleteme3")); + CHECK(!IsFile(TmpDir.Path() / ".keepmenot")); + CHECK(!IsFile(TmpDir.Path() / "no.keepme")); + + CHECK(!IsDir(TmpDir.Path() / "DeleteMe")); + CHECK(!IsDir(TmpDir.Path() / "DeleteMe2")); + + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe" / ".keepme")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept")); + CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1")); + CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2")); + CHECK(!IsFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme")); +} + +#endif + } // namespace zen -- cgit v1.2.3