// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #include #include #include #if ZEN_WITH_TESTS # include # include #endif // ZEN_WITH_TESTS namespace zen { BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path, std::atomic& OpenReadCount, std::atomic& CurrentOpenFileCount, std::atomic& ReadCount, std::atomic& ReadByteCount) : m_Source(Path, BasicFile::Mode::kRead) , m_SourceSize(m_Source.FileSize()) , m_OpenReadCount(OpenReadCount) , m_CurrentOpenFileCount(CurrentOpenFileCount) , m_ReadCount(ReadCount) , m_ReadByteCount(ReadByteCount) { m_OpenReadCount++; m_CurrentOpenFileCount++; } BufferedOpenFile::~BufferedOpenFile() { m_CurrentOpenFileCount--; } CompositeBuffer BufferedOpenFile::GetRange(uint64_t Offset, uint64_t Size) { ZEN_TRACE_CPU("BufferedOpenFile::GetRange"); ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); auto _ = MakeGuard([&]() { ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); }); ZEN_ASSERT((Offset + Size) <= m_SourceSize); const uint64_t BlockIndexStart = Offset / BlockSize; const uint64_t BlockIndexEnd = (Offset + Size - 1) / BlockSize; std::vector BufferRanges; BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1); uint64_t ReadOffset = Offset; for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++) { const uint64_t BlockStartOffset = BlockIndex * BlockSize; if (m_CacheBlockIndex != BlockIndex) { uint64_t CacheSize = Min(BlockSize, m_SourceSize - BlockStartOffset); ZEN_ASSERT(CacheSize > 0); m_Cache = IoBuffer(CacheSize); m_Source.Read(m_Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset); m_ReadCount++; m_ReadByteCount += CacheSize; m_CacheBlockIndex = BlockIndex; } const uint64_t BytesRead = ReadOffset - Offset; ZEN_ASSERT(BlockStartOffset <= ReadOffset); const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset; ZEN_ASSERT(OffsetIntoBlock < m_Cache.GetSize()); const uint64_t BlockBytes = Min(m_Cache.GetSize() - OffsetIntoBlock, Size - BytesRead); BufferRanges.emplace_back(SharedBuffer(IoBuffer(m_Cache, OffsetIntoBlock, BlockBytes))); ReadOffset += BlockBytes; } CompositeBuffer Result(std::move(BufferRanges)); ZEN_ASSERT(Result.GetSize() == Size); return Result; } bool IsFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; bool Result = IsFile(Path, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { Sleep(100 + int(Retries * 50)); Ec.clear(); Result = IsFile(Path, Ec); } if (Ec) { throw std::system_error(std::error_code(Ec.value(), std::system_category()), fmt::format("Failed to check path '{}' is file, reason: ({}) {}", Path, Ec.value(), Ec.message())); } return Result; } bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) { std::error_code Ec; bool Result = SetFileReadOnly(Path, ReadOnly, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { if (!IsFileWithRetry(Path)) { return false; } Sleep(100 + int(Retries * 50)); Ec.clear(); Result = SetFileReadOnly(Path, ReadOnly, Ec); } if (Ec) { throw std::system_error(std::error_code(Ec.value(), std::system_category()), fmt::format("Failed {} read only flag for file '{}', reason: ({}) {}", ReadOnly ? "setting" : "clearing", Path, Ec.value(), Ec.message())); } return Result; } std::error_code RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) { std::error_code Ec; RenameFile(SourcePath, TargetPath, Ec); for (size_t Retries = 0; Ec && Retries < 5; Retries++) { ZEN_ASSERT_SLOW(IsFile(SourcePath)); Sleep(50 + int(Retries * 150)); Ec.clear(); RenameFile(SourcePath, TargetPath, Ec); } return Ec; } std::error_code RenameDirectoryWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) { std::error_code Ec; RenameDirectory(SourcePath, TargetPath, Ec); for (size_t Retries = 0; Ec && Retries < 5; Retries++) { ZEN_ASSERT_SLOW(IsDir(SourcePath)); Sleep(50 + int(Retries * 150)); Ec.clear(); RenameDirectory(SourcePath, TargetPath, Ec); } return Ec; } std::error_code TryRemoveFile(const std::filesystem::path& Path) { std::error_code Ec; RemoveFile(Path, Ec); if (Ec) { if (IsFile(Path, Ec)) { Ec.clear(); RemoveFile(Path, Ec); if (Ec) { return Ec; } } } return {}; } void RemoveFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; RemoveFile(Path, Ec); for (size_t Retries = 0; Ec && Retries < 6; Retries++) { if (!IsFileWithRetry(Path)) { return; } Sleep(100 + int(Retries * 50)); Ec.clear(); RemoveFile(Path, Ec); } if (Ec) { throw std::system_error(std::error_code(Ec.value(), std::system_category()), fmt::format("Failed removing file '{}', reason: ({}) {}", Path, Ec.value(), Ec.message())); } } void FastCopyFile(bool AllowFileClone, bool UseSparseFiles, const std::filesystem::path& SourceFilePath, const std::filesystem::path& TargetFilePath, uint64_t RawSize, std::atomic& WriteCount, std::atomic& WriteByteCount, std::atomic& CloneCount, std::atomic& CloneByteCount) { ZEN_TRACE_CPU("CopyFile"); if (AllowFileClone && TryCloneFile(SourceFilePath, TargetFilePath)) { WriteCount += 1; WriteByteCount += RawSize; CloneCount += 1; CloneByteCount += RawSize; } else { BasicFile TargetFile(TargetFilePath, BasicFile::Mode::kTruncate); if (UseSparseFiles) { PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize); } uint64_t Offset = 0; if (!ScanFile(SourceFilePath, 512u * 1024u, [&](const void* Data, size_t Size) { TargetFile.Write(Data, Size, Offset); Offset += Size; WriteCount++; WriteByteCount += Size; })) { throw std::runtime_error(fmt::format("Failed to copy file '{}' to '{}'", SourceFilePath, TargetFilePath)); } } } void GetDirectoryContent(WorkerThreadPool& WorkerPool, const std::filesystem::path& Path, DirectoryContentFlags Flags, DirectoryContent& OutContent) { struct Visitor : public GetDirectoryContentVisitor { Visitor(zen::DirectoryContent& OutContent, const std::filesystem::path& InRootPath) : Content(OutContent), RootPath(InRootPath) {} virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const { ZEN_UNUSED(Parent, DirectoryName); return true; } virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& InContent) { std::vector Files; std::vector Directories; if (!InContent.FileNames.empty()) { Files.reserve(InContent.FileNames.size()); for (const std::filesystem::path& FileName : InContent.FileNames) { if (RelativeRoot.empty()) { Files.push_back(RootPath / FileName); } else { Files.push_back(RootPath / RelativeRoot / FileName); } } } if (!InContent.DirectoryNames.empty()) { Directories.reserve(InContent.DirectoryNames.size()); for (const std::filesystem::path& DirName : InContent.DirectoryNames) { if (RelativeRoot.empty()) { Directories.push_back(RootPath / DirName); } else { Directories.push_back(RootPath / RelativeRoot / DirName); } } } Lock.WithExclusiveLock([&]() { if (!InContent.FileNames.empty()) { for (const std::filesystem::path& FileName : InContent.FileNames) { if (RelativeRoot.empty()) { Content.Files.push_back(RootPath / FileName); } else { Content.Files.push_back(RootPath / RelativeRoot / FileName); } } } if (!InContent.FileSizes.empty()) { Content.FileSizes.insert(Content.FileSizes.end(), InContent.FileSizes.begin(), InContent.FileSizes.end()); } if (!InContent.FileAttributes.empty()) { Content.FileAttributes.insert(Content.FileAttributes.end(), InContent.FileAttributes.begin(), InContent.FileAttributes.end()); } if (!InContent.FileModificationTicks.empty()) { Content.FileModificationTicks.insert(Content.FileModificationTicks.end(), InContent.FileModificationTicks.begin(), InContent.FileModificationTicks.end()); } if (!InContent.DirectoryNames.empty()) { for (const std::filesystem::path& DirName : InContent.DirectoryNames) { if (RelativeRoot.empty()) { Content.Directories.push_back(RootPath / DirName); } else { Content.Directories.push_back(RootPath / RelativeRoot / DirName); } } } if (!InContent.DirectoryAttributes.empty()) { Content.DirectoryAttributes.insert(Content.DirectoryAttributes.end(), InContent.DirectoryAttributes.begin(), InContent.DirectoryAttributes.end()); } }); } RwLock Lock; zen::DirectoryContent& Content; const std::filesystem::path& RootPath; }; Visitor RootVisitor(OutContent, Path); Latch PendingWork(1); GetDirectoryContent(Path, Flags, RootVisitor, WorkerPool, PendingWork); PendingWork.CountDown(); PendingWork.Wait(); } CleanDirectoryResult CleanDirectory( WorkerThreadPool& IOWorkerPool, std::atomic& AbortFlag, std::atomic& PauseFlag, const std::filesystem::path& Path, std::span ExcludeDirectories, std::function&& ProgressFunc, uint32_t ProgressUpdateDelayMS) { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; std::atomic DiscoveredItemCount = 0; std::atomic DeletedItemCount = 0; std::atomic DeletedByteCount = 0; std::vector DirectoriesToDelete; CleanDirectoryResult Result; RwLock ResultLock; auto _ = MakeGuard([&]() { Result.DeletedCount = DeletedItemCount.load(); Result.DeletedByteCount = DeletedByteCount.load(); Result.FoundCount = DiscoveredItemCount.load(); }); ParallelWork Work(AbortFlag, PauseFlag, ProgressFunc ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog); struct AsyncVisitor : public GetDirectoryContentVisitor { AsyncVisitor(const std::filesystem::path& InPath, std::atomic& InAbortFlag, std::atomic& InDiscoveredItemCount, std::atomic& InDeletedItemCount, std::atomic& InDeletedByteCount, std::span InExcludeDirectories, std::vector& OutDirectoriesToDelete, CleanDirectoryResult& InResult, RwLock& InResultLock) : Path(InPath) , AbortFlag(InAbortFlag) , DiscoveredItemCount(InDiscoveredItemCount) , DeletedItemCount(InDeletedItemCount) , DeletedByteCount(InDeletedByteCount) , ExcludeDirectories(InExcludeDirectories) , DirectoriesToDelete(OutDirectoriesToDelete) , Result(InResult) , ResultLock(InResultLock) { } virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const override { ZEN_UNUSED(Parent); if (AbortFlag) { return false; } const std::string DirectoryString = DirectoryName.string(); for (const std::string_view ExcludeDirectory : ExcludeDirectories) { if (DirectoryString == ExcludeDirectory) { return false; } } return true; } virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); if (!AbortFlag) { DiscoveredItemCount += Content.FileNames.size(); ZEN_TRACE_CPU("DeleteFiles"); std::vector> FailedRemovePaths; for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) { const std::filesystem::path& FileName = Content.FileNames[FileIndex]; const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); bool IsRemoved = false; std::error_code Ec; (void)SetFileReadOnly(FilePath, false, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { if (!IsFileWithRetry(FilePath)) { IsRemoved = true; Ec.clear(); break; } Sleep(100 + int(Retries * 50)); Ec.clear(); (void)SetFileReadOnly(FilePath, false, Ec); } if (!IsRemoved && !Ec) { (void)RemoveFile(FilePath, Ec); for (size_t Retries = 0; Ec && Retries < 6; Retries++) { if (!IsFileWithRetry(FilePath)) { IsRemoved = true; Ec.clear(); break; } Sleep(100 + int(Retries * 50)); Ec.clear(); (void)RemoveFile(FilePath, Ec); } } if (!IsRemoved && Ec) { FailedRemovePaths.push_back(std::make_pair(FilePath, Ec)); } else { DeletedItemCount++; DeletedByteCount += Content.FileSizes[FileIndex]; } } if (!FailedRemovePaths.empty()) { RwLock::ExclusiveLockScope _(ResultLock); Result.FailedRemovePaths.insert(Result.FailedRemovePaths.end(), FailedRemovePaths.begin(), FailedRemovePaths.end()); } else if (!RelativeRoot.empty()) { DiscoveredItemCount++; RwLock::ExclusiveLockScope _(ResultLock); DirectoriesToDelete.push_back(RelativeRoot); } } } const std::filesystem::path& Path; std::atomic& AbortFlag; std::atomic& DiscoveredItemCount; std::atomic& DeletedItemCount; std::atomic& DeletedByteCount; std::span ExcludeDirectories; std::vector& DirectoriesToDelete; CleanDirectoryResult& Result; RwLock& ResultLock; } Visitor(Path, AbortFlag, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories, DirectoriesToDelete, Result, ResultLock); GetDirectoryContent(Path, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes, Visitor, IOWorkerPool, Work.PendingWork()); uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); if (ProgressFunc && ProgressUpdateDelayMS != 0) { Work.Wait(ProgressUpdateDelayMS, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); LastUpdateTimeMs = Timer.GetElapsedTimeMs(); uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); ProgressFunc(Details, Discovered, Discovered - Deleted, IsPaused, IsAborted); }); } else { Work.Wait(); } { ZEN_TRACE_CPU("DeleteDirs"); std::sort(DirectoriesToDelete.begin(), DirectoriesToDelete.end(), [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) { auto DistanceLhs = std::distance(Lhs.begin(), Lhs.end()); auto DistanceRhs = std::distance(Rhs.begin(), Rhs.end()); return DistanceLhs > DistanceRhs; }); for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete) { if (AbortFlag) { break; } else { while (PauseFlag && !AbortFlag) { Sleep(2000); } } const std::filesystem::path FullPath = Path / DirectoryToDelete; std::error_code Ec; RemoveDir(FullPath, Ec); if (Ec) { for (size_t Retries = 0; Ec && Retries < 3; Retries++) { if (!IsDir(FullPath)) { Ec.clear(); break; } Sleep(100 + int(Retries * 50)); Ec.clear(); RemoveDir(FullPath, Ec); } } if (Ec) { RwLock::ExclusiveLockScope __(ResultLock); Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec)); } else { DeletedItemCount++; } if (ProgressFunc) { uint64_t NowMs = Timer.GetElapsedTimeMs(); if ((NowMs - LastUpdateTimeMs) > 0) { LastUpdateTimeMs = NowMs; uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag); } } } } return Result; } bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, std::atomic& AbortFlag, std::atomic& PauseFlag, const std::filesystem::path& Directory) { if (!IsDir(Directory)) { return true; } if (CleanDirectoryResult Res = CleanDirectory( WorkerPool, AbortFlag, PauseFlag, Directory, {}, [](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) { ZEN_UNUSED(Details, TotalCount, RemainingCount, IsPaused, IsAborted); }, 1000); Res.FailedRemovePaths.empty()) { std::error_code Ec; RemoveDir(Directory, Ec); return !Ec; } return false; } #if ZEN_WITH_TESTS void filesystemutils_forcelink() { } namespace { void GenerateFile(const std::filesystem::path& Path) { BasicFile _(Path, BasicFile::Mode::kTruncate); } } // namespace TEST_SUITE_BEGIN("util.filesystemutils"); TEST_CASE("filesystemutils.CleanDirectory") { ScopedTemporaryDirectory TmpDir; CreateDirectories(TmpDir.Path() / ".keepme"); GenerateFile(TmpDir.Path() / ".keepme" / "keep"); GenerateFile(TmpDir.Path() / "deleteme1"); GenerateFile(TmpDir.Path() / "deleteme2"); GenerateFile(TmpDir.Path() / "deleteme3"); CreateDirectories(TmpDir.Path() / ".keepmenot"); CreateDirectories(TmpDir.Path() / "no.keepme"); CreateDirectories(TmpDir.Path() / "DeleteMe"); GenerateFile(TmpDir.Path() / "DeleteMe" / "delete1"); CreateDirectories(TmpDir.Path() / "CantDeleteMe"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete1"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete2"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete3"); CreateDirectories(TmpDir.Path() / "CantDeleteMe" / ".keepme"); CreateDirectories(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete2"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete3"); CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme"); CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept"); GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1"); GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2"); GenerateFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme"); WorkerThreadPool Pool(4); std::atomic AbortFlag; std::atomic PauseFlag; CleanDirectory(Pool, AbortFlag, PauseFlag, TmpDir.Path(), std::vector{".keepme"}, {}, 0); CHECK(IsDir(TmpDir.Path() / ".keepme")); CHECK(IsFile(TmpDir.Path() / ".keepme" / "keep")); CHECK(!IsFile(TmpDir.Path() / "deleteme1")); CHECK(!IsFile(TmpDir.Path() / "deleteme2")); CHECK(!IsFile(TmpDir.Path() / "deleteme3")); CHECK(!IsFile(TmpDir.Path() / ".keepmenot")); CHECK(!IsFile(TmpDir.Path() / "no.keepme")); CHECK(!IsDir(TmpDir.Path() / "DeleteMe")); CHECK(!IsDir(TmpDir.Path() / "DeleteMe2")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe" / ".keepme")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept")); CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1")); CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2")); CHECK(!IsFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme")); } TEST_SUITE_END(); #endif } // namespace zen