// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #include #include #include #include #if ZEN_WITH_TESTS # include # include #endif // ZEN_WITH_TESTS namespace zen { BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path, std::atomic& OpenReadCount, std::atomic& CurrentOpenFileCount, std::atomic& ReadCount, std::atomic& ReadByteCount) : m_Source(Path, BasicFile::Mode::kRead) , m_SourceSize(m_Source.FileSize()) , m_OpenReadCount(OpenReadCount) , m_CurrentOpenFileCount(CurrentOpenFileCount) , m_ReadCount(ReadCount) , m_ReadByteCount(ReadByteCount) { m_OpenReadCount++; m_CurrentOpenFileCount++; } BufferedOpenFile::~BufferedOpenFile() { m_CurrentOpenFileCount--; } CompositeBuffer BufferedOpenFile::GetRange(uint64_t Offset, uint64_t Size) { ZEN_TRACE_CPU("BufferedOpenFile::GetRange"); ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); auto _ = MakeGuard([&]() { ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); }); ZEN_ASSERT((Offset + Size) <= m_SourceSize); const uint64_t BlockIndexStart = Offset / BlockSize; const uint64_t BlockIndexEnd = (Offset + Size - 1) / BlockSize; std::vector BufferRanges; BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1); uint64_t ReadOffset = Offset; for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++) { const uint64_t BlockStartOffset = BlockIndex * BlockSize; if (m_CacheBlockIndex != BlockIndex) { uint64_t CacheSize = Min(BlockSize, m_SourceSize - BlockStartOffset); ZEN_ASSERT(CacheSize > 0); m_Cache = IoBuffer(CacheSize); m_Source.Read(m_Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset); m_ReadCount++; m_ReadByteCount += CacheSize; m_CacheBlockIndex = BlockIndex; } const uint64_t BytesRead = ReadOffset - Offset; ZEN_ASSERT(BlockStartOffset <= ReadOffset); const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset; ZEN_ASSERT(OffsetIntoBlock < m_Cache.GetSize()); const uint64_t BlockBytes = Min(m_Cache.GetSize() - OffsetIntoBlock, Size - BytesRead); BufferRanges.emplace_back(SharedBuffer(IoBuffer(m_Cache, OffsetIntoBlock, BlockBytes))); ReadOffset += BlockBytes; } CompositeBuffer Result(std::move(BufferRanges)); ZEN_ASSERT(Result.GetSize() == Size); return Result; } ReadFileCache::ReadFileCache(std::atomic& OpenReadCount, std::atomic& CurrentOpenFileCount, std::atomic& ReadCount, std::atomic& ReadByteCount, const std::filesystem::path& Path, const ChunkedFolderContent& LocalContent, const ChunkedContentLookup& LocalLookup, size_t MaxOpenFileCount) : m_Path(Path) , m_LocalContent(LocalContent) , m_LocalLookup(LocalLookup) , m_OpenReadCount(OpenReadCount) , m_CurrentOpenFileCount(CurrentOpenFileCount) , m_ReadCount(ReadCount) , m_ReadByteCount(ReadByteCount) { m_OpenFiles.reserve(MaxOpenFileCount); } ReadFileCache::~ReadFileCache() { m_OpenFiles.clear(); } CompositeBuffer ReadFileCache::GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size) { ZEN_TRACE_CPU("ReadFileCache::GetRange"); auto CacheIt = std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { return Lhs.first == SequenceIndex; }); if (CacheIt != m_OpenFiles.end()) { if (CacheIt != m_OpenFiles.begin()) { auto CachedFile(std::move(CacheIt->second)); m_OpenFiles.erase(CacheIt); m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile))); } CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); return Result; } const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex]; const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); if (Size == m_LocalContent.RawSizes[LocalPathIndex]) { IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath); return CompositeBuffer(SharedBuffer(Result)); } if (m_OpenFiles.size() == m_OpenFiles.capacity()) { m_OpenFiles.pop_back(); } m_OpenFiles.insert( m_OpenFiles.begin(), std::make_pair( SequenceIndex, std::make_unique(LocalFilePath, m_OpenReadCount, m_CurrentOpenFileCount, m_ReadCount, m_ReadByteCount))); CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); return Result; } uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS if (SourcePlatform == SourcePlatform::Windows) { SetFileAttributesToPath(FilePath, Attributes); return Attributes; } else { uint32_t CurrentAttributes = GetFileAttributesFromPath(FilePath); uint32_t NewAttributes = zen::MakeFileAttributeReadOnly(CurrentAttributes, zen::IsFileModeReadOnly(Attributes)); if (CurrentAttributes != NewAttributes) { SetFileAttributesToPath(FilePath, NewAttributes); } return NewAttributes; } #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC if (SourcePlatform != SourcePlatform::Windows) { zen::SetFileMode(FilePath, Attributes); return Attributes; } else { uint32_t CurrentMode = zen::GetFileMode(FilePath); uint32_t NewMode = zen::MakeFileModeReadOnly(CurrentMode, zen::IsFileAttributeReadOnly(Attributes)); if (CurrentMode != NewMode) { zen::SetFileMode(FilePath, NewMode); } return NewMode; } #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC }; uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath) { #if ZEN_PLATFORM_WINDOWS return GetFileAttributesFromPath(FilePath); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC return GetFileMode(FilePath); #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC } bool IsFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; bool Result = IsFile(Path, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { Sleep(100 + int(Retries * 50)); Ec.clear(); Result = IsFile(Path, Ec); } if (Ec) { throw std::system_error(std::error_code(Ec.value(), std::system_category()), fmt::format("Failed to check path '{}' is file, reason: ({}) {}", Path, Ec.value(), Ec.message())); } return Result; } bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) { std::error_code Ec; bool Result = SetFileReadOnly(Path, ReadOnly, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { if (!IsFileWithRetry(Path)) { return false; } Sleep(100 + int(Retries * 50)); Ec.clear(); Result = SetFileReadOnly(Path, ReadOnly, Ec); } if (Ec) { throw std::system_error(std::error_code(Ec.value(), std::system_category()), fmt::format("Failed {} read only flag for file '{}', reason: ({}) {}", ReadOnly ? "setting" : "clearing", Path, Ec.value(), Ec.message())); } return Result; } std::error_code RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) { std::error_code Ec; RenameFile(SourcePath, TargetPath, Ec); for (size_t Retries = 0; Ec && Retries < 5; Retries++) { ZEN_ASSERT_SLOW(IsFile(SourcePath)); Sleep(50 + int(Retries * 150)); Ec.clear(); RenameFile(SourcePath, TargetPath, Ec); } return Ec; } std::error_code TryRemoveFile(const std::filesystem::path& Path) { std::error_code Ec; RemoveFile(Path, Ec); if (Ec) { if (IsFile(Path, Ec)) { Ec.clear(); RemoveFile(Path, Ec); if (Ec) { return Ec; } } } return {}; } void RemoveFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; RemoveFile(Path, Ec); for (size_t Retries = 0; Ec && Retries < 6; Retries++) { if (!IsFileWithRetry(Path)) { return; } Sleep(100 + int(Retries * 50)); Ec.clear(); RemoveFile(Path, Ec); } if (Ec) { throw std::system_error(std::error_code(Ec.value(), std::system_category()), fmt::format("Failed removing file '{}', reason: ({}) {}", Path, Ec.value(), Ec.message())); } } void FastCopyFile(bool AllowFileClone, bool UseSparseFiles, const std::filesystem::path& SourceFilePath, const std::filesystem::path& TargetFilePath, uint64_t RawSize, std::atomic& WriteCount, std::atomic& WriteByteCount, std::atomic& CloneCount, std::atomic& CloneByteCount) { ZEN_TRACE_CPU("CopyFile"); if (AllowFileClone && TryCloneFile(SourceFilePath, TargetFilePath)) { WriteCount += 1; WriteByteCount += RawSize; CloneCount += 1; CloneByteCount += RawSize; } else { BasicFile TargetFile(TargetFilePath, BasicFile::Mode::kTruncate); if (UseSparseFiles) { PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize); } uint64_t Offset = 0; if (!ScanFile(SourceFilePath, 512u * 1024u, [&](const void* Data, size_t Size) { TargetFile.Write(Data, Size, Offset); Offset += Size; WriteCount++; WriteByteCount += Size; })) { throw std::runtime_error(fmt::format("Failed to copy file '{}' to '{}'", SourceFilePath, TargetFilePath)); } } } CleanDirectoryResult CleanDirectory( WorkerThreadPool& IOWorkerPool, std::atomic& AbortFlag, std::atomic& PauseFlag, const std::filesystem::path& Path, std::span ExcludeDirectories, std::function&& ProgressFunc, uint32_t ProgressUpdateDelayMS) { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; std::atomic DiscoveredItemCount = 0; std::atomic DeletedItemCount = 0; std::atomic DeletedByteCount = 0; std::vector DirectoriesToDelete; CleanDirectoryResult Result; RwLock ResultLock; auto _ = MakeGuard([&]() { Result.DeletedCount = DeletedItemCount.load(); Result.DeletedByteCount = DeletedByteCount.load(); Result.FoundCount = DiscoveredItemCount.load(); }); ParallelWork Work(AbortFlag, PauseFlag, ProgressFunc ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog); struct AsyncVisitor : public GetDirectoryContentVisitor { AsyncVisitor(const std::filesystem::path& InPath, std::atomic& InAbortFlag, std::atomic& InDiscoveredItemCount, std::atomic& InDeletedItemCount, std::atomic& InDeletedByteCount, std::span InExcludeDirectories, std::vector& OutDirectoriesToDelete, CleanDirectoryResult& InResult, RwLock& InResultLock) : Path(InPath) , AbortFlag(InAbortFlag) , DiscoveredItemCount(InDiscoveredItemCount) , DeletedItemCount(InDeletedItemCount) , DeletedByteCount(InDeletedByteCount) , ExcludeDirectories(InExcludeDirectories) , DirectoriesToDelete(OutDirectoriesToDelete) , Result(InResult) , ResultLock(InResultLock) { } virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const override { ZEN_UNUSED(Parent); if (AbortFlag) { return false; } const std::string DirectoryString = DirectoryName.string(); for (const std::string_view ExcludeDirectory : ExcludeDirectories) { if (DirectoryString == ExcludeDirectory) { return false; } } return true; } virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); if (!AbortFlag) { DiscoveredItemCount += Content.FileNames.size(); ZEN_TRACE_CPU("DeleteFiles"); std::vector> FailedRemovePaths; for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) { const std::filesystem::path& FileName = Content.FileNames[FileIndex]; const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); bool IsRemoved = false; std::error_code Ec; (void)SetFileReadOnly(FilePath, false, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { if (!IsFileWithRetry(FilePath)) { IsRemoved = true; Ec.clear(); break; } Sleep(100 + int(Retries * 50)); Ec.clear(); (void)SetFileReadOnly(FilePath, false, Ec); } if (!IsRemoved && !Ec) { (void)RemoveFile(FilePath, Ec); for (size_t Retries = 0; Ec && Retries < 6; Retries++) { if (!IsFileWithRetry(FilePath)) { IsRemoved = true; Ec.clear(); break; } Sleep(100 + int(Retries * 50)); Ec.clear(); (void)RemoveFile(FilePath, Ec); } } if (!IsRemoved && Ec) { FailedRemovePaths.push_back(std::make_pair(FilePath, Ec)); } else { DeletedItemCount++; DeletedByteCount += Content.FileSizes[FileIndex]; } } if (!FailedRemovePaths.empty()) { RwLock::ExclusiveLockScope _(ResultLock); FailedRemovePaths.insert(FailedRemovePaths.end(), FailedRemovePaths.begin(), FailedRemovePaths.end()); } else if (!RelativeRoot.empty()) { DiscoveredItemCount++; RwLock::ExclusiveLockScope _(ResultLock); DirectoriesToDelete.push_back(RelativeRoot); } } } const std::filesystem::path& Path; std::atomic& AbortFlag; std::atomic& DiscoveredItemCount; std::atomic& DeletedItemCount; std::atomic& DeletedByteCount; std::span ExcludeDirectories; std::vector& DirectoriesToDelete; CleanDirectoryResult& Result; RwLock& ResultLock; } Visitor(Path, AbortFlag, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories, DirectoriesToDelete, Result, ResultLock); GetDirectoryContent(Path, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes, Visitor, IOWorkerPool, Work.PendingWork()); uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); if (ProgressFunc && ProgressUpdateDelayMS != 0) { Work.Wait(ProgressUpdateDelayMS, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); LastUpdateTimeMs = Timer.GetElapsedTimeMs(); uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); ProgressFunc(Details, Discovered, Discovered - Deleted, IsPaused, IsAborted); }); } else { Work.Wait(); } { ZEN_TRACE_CPU("DeleteDirs"); std::sort(DirectoriesToDelete.begin(), DirectoriesToDelete.end(), [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) { auto DistanceLhs = std::distance(Lhs.begin(), Lhs.end()); auto DistanceRhs = std::distance(Rhs.begin(), Rhs.end()); return DistanceLhs > DistanceRhs; }); for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete) { if (AbortFlag) { break; } else { while (PauseFlag && !AbortFlag) { Sleep(2000); } } const std::filesystem::path FullPath = Path / DirectoryToDelete; std::error_code Ec; RemoveDir(FullPath, Ec); if (Ec) { for (size_t Retries = 0; Ec && Retries < 3; Retries++) { if (!IsDir(FullPath)) { Ec.clear(); break; } Sleep(100 + int(Retries * 50)); Ec.clear(); RemoveDir(FullPath, Ec); } } if (Ec) { RwLock::ExclusiveLockScope __(ResultLock); Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec)); } else { DeletedItemCount++; } if (ProgressFunc) { uint64_t NowMs = Timer.GetElapsedTimeMs(); if ((NowMs - LastUpdateTimeMs) > 0) { LastUpdateTimeMs = NowMs; uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag); } } } } return Result; } bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, std::atomic& AbortFlag, std::atomic& PauseFlag, const std::filesystem::path& Directory) { if (!IsDir(Directory)) { return true; } if (CleanDirectoryResult Res = CleanDirectory( WorkerPool, AbortFlag, PauseFlag, Directory, {}, [](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) { ZEN_UNUSED(Details, TotalCount, RemainingCount, IsPaused, IsAborted); }, 1000); Res.FailedRemovePaths.empty()) { std::error_code Ec; RemoveDir(Directory, Ec); return !Ec; } return false; } #if ZEN_WITH_TESTS void filesystemutils_forcelink() { } namespace { void GenerateFile(const std::filesystem::path& Path) { BasicFile _(Path, BasicFile::Mode::kTruncate); } } // namespace TEST_CASE("filesystemutils.CleanDirectory") { ScopedTemporaryDirectory TmpDir; CreateDirectories(TmpDir.Path() / ".keepme"); GenerateFile(TmpDir.Path() / ".keepme" / "keep"); GenerateFile(TmpDir.Path() / "deleteme1"); GenerateFile(TmpDir.Path() / "deleteme2"); GenerateFile(TmpDir.Path() / "deleteme3"); CreateDirectories(TmpDir.Path() / ".keepmenot"); CreateDirectories(TmpDir.Path() / "no.keepme"); CreateDirectories(TmpDir.Path() / "DeleteMe"); GenerateFile(TmpDir.Path() / "DeleteMe" / "delete1"); CreateDirectories(TmpDir.Path() / "CantDeleteMe"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete1"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete2"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete3"); CreateDirectories(TmpDir.Path() / "CantDeleteMe" / ".keepme"); CreateDirectories(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete2"); GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete3"); CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme"); CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept"); GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1"); GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2"); GenerateFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme"); WorkerThreadPool Pool(4); std::atomic AbortFlag; std::atomic PauseFlag; CleanDirectory(Pool, AbortFlag, PauseFlag, TmpDir.Path(), std::vector{".keepme"}, {}, 0); CHECK(IsDir(TmpDir.Path() / ".keepme")); CHECK(IsFile(TmpDir.Path() / ".keepme" / "keep")); CHECK(!IsFile(TmpDir.Path() / "deleteme1")); CHECK(!IsFile(TmpDir.Path() / "deleteme2")); CHECK(!IsFile(TmpDir.Path() / "deleteme3")); CHECK(!IsFile(TmpDir.Path() / ".keepmenot")); CHECK(!IsFile(TmpDir.Path() / "no.keepme")); CHECK(!IsDir(TmpDir.Path() / "DeleteMe")); CHECK(!IsDir(TmpDir.Path() / "DeleteMe2")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe" / ".keepme")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme")); CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept")); CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1")); CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2")); CHECK(!IsFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme")); } #endif } // namespace zen