multithreaded clean (#331)

- Improvement: Faster cleaning of directories
- Improvement: Faster initial scanning of local state

3 files changed, 393 insertions, 92 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index d2ba20e78..2bbb21012 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -316,25 +316,108 @@ namespace {
 	bool CleanDirectory(const std::filesystem::path& Path, std::span<const std::string_view> ExcludeDirectories)
 	{
 		ZEN_TRACE_CPU("CleanDirectory");
+		Stopwatch Timer;
 
-		bool CleanWipe = true;
+		ProgressBar Progress(UsePlainProgress);
 
-		DirectoryContent LocalDirectoryContent;
-		GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
-		for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files)
+		std::atomic<bool>	  CleanWipe			  = true;
+		std::atomic<uint64_t> DiscoveredItemCount = 0;
+		std::atomic<uint64_t> DeletedItemCount	  = 0;
+		std::atomic<uint64_t> DeletedByteCount	  = 0;
+		ParallellWork		  Work(AbortFlag);
+
+		struct AsyncVisitor : public GetDirectoryContentVisitor
 		{
-			try
+			AsyncVisitor(const std::filesystem::path&	   InPath,
+						 std::atomic<bool>&				   InCleanWipe,
+						 std::atomic<uint64_t>&			   InDiscoveredItemCount,
+						 std::atomic<uint64_t>&			   InDeletedItemCount,
+						 std::atomic<uint64_t>&			   InDeletedByteCount,
+						 std::span<const std::string_view> InExcludeDirectories)
+			: Path(InPath)
+			, CleanWipe(InCleanWipe)
+			, DiscoveredItemCount(InDiscoveredItemCount)
+			, DeletedItemCount(InDeletedItemCount)
+			, DeletedByteCount(InDeletedByteCount)
+			, ExcludeDirectories(InExcludeDirectories)
 			{
-				SetFileReadOnlyWithRetry(LocalFilePath, false);
-				RemoveFileWithRetry(LocalFilePath);
 			}
-			catch (const std::exception& Ex)
+			virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
 			{
-				ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what());
-				CleanWipe = false;
+				ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
+				if (!AbortFlag)
+				{
+					if (!Content.FileNames.empty())
+					{
+						DiscoveredItemCount += Content.FileNames.size();
+
+						const std::string RelativeRootString = RelativeRoot.generic_string();
+						bool			  RemoveContent		 = true;
+						for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+						{
+							if (RelativeRootString.starts_with(ExcludeDirectory))
+							{
+								if (RelativeRootString.length() > ExcludeDirectory.length())
+								{
+									const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
+									if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
+										MaybePathDelimiter == std::filesystem::path::preferred_separator)
+									{
+										RemoveContent = false;
+										break;
+									}
+								}
+								else
+								{
+									RemoveContent = false;
+									break;
+								}
+							}
+						}
+						if (RemoveContent)
+						{
+							ZEN_TRACE_CPU("DeleteFiles");
+							for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+							{
+								const std::filesystem::path& FileName = Content.FileNames[FileIndex];
+								const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
+								try
+								{
+									SetFileReadOnlyWithRetry(FilePath, false);
+									RemoveFileWithRetry(FilePath);
+									DeletedItemCount++;
+									DeletedByteCount += Content.FileSizes[FileIndex];
+								}
+								catch (const std::exception& Ex)
+								{
+									ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what());
+									CleanWipe = false;
+								}
+							}
+						}
+					}
+				}
 			}
-		}
+			const std::filesystem::path&	  Path;
+			std::atomic<bool>&				  CleanWipe;
+			std::atomic<uint64_t>&			  DiscoveredItemCount;
+			std::atomic<uint64_t>&			  DeletedItemCount;
+			std::atomic<uint64_t>&			  DeletedByteCount;
+			std::span<const std::string_view> ExcludeDirectories;
+		} Visitor(Path, CleanWipe, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories);
 
+		GetDirectoryContent(
+			Path,
+			DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes,
+			Visitor,
+			GetIOWorkerPool(),
+			Work.PendingWork());
+
+		DirectoryContent LocalDirectoryContent;
+		GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
+		DiscoveredItemCount += LocalDirectoryContent.Directories.size();
+		std::vector<std::filesystem::path> DirectoriesToDelete;
+		DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size());
 		for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
 		{
 			bool Leave = false;
@@ -348,31 +431,78 @@ namespace {
 			}
 			if (!Leave)
 			{
-				try
-				{
-					std::error_code Ec;
-					zen::CleanDirectory(LocalDirPath, true, Ec);
-					if (Ec)
-					{
-						Sleep(200);
-						zen::CleanDirectory(LocalDirPath, true);
-						Ec.clear();
-					}
+				DirectoriesToDelete.emplace_back(std::move(LocalDirPath));
+				DiscoveredItemCount++;
+			}
+		}
 
-					RemoveDir(LocalDirPath, Ec);
-					if (Ec)
-					{
-						Sleep(200);
-						RemoveDir(LocalDirPath);
-					}
-				}
-				catch (const std::exception& Ex)
+		uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+		Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) {
+			ZEN_UNUSED(IsAborted, PendingWork);
+			LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+			uint64_t Deleted	  = DeletedItemCount.load();
+			uint64_t DeletedBytes = DeletedByteCount.load();
+			uint64_t Discovered	  = DiscoveredItemCount.load();
+			Progress.UpdateState({.Task			  = "Cleaning folder  ",
+								  .Details		  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+								  .TotalCount	  = Discovered,
+								  .RemainingCount = Discovered - Deleted},
+								 false);
+		});
+
+		for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete)
+		{
+			ZEN_TRACE_CPU("DeleteDirs");
+			try
+			{
+				std::error_code Ec;
+				zen::CleanDirectory(DirectoryToDelete, true, Ec);
+				if (Ec)
 				{
-					ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what());
-					CleanWipe = false;
+					Sleep(200);
+					zen::CleanDirectory(DirectoryToDelete, true);
+					Ec.clear();
 				}
+
+				RemoveDirWithRetry(DirectoryToDelete);
+
+				DeletedItemCount++;
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
+				CleanWipe = false;
+			}
+
+			uint64_t NowMs = Timer.GetElapsedTimeMs();
+			if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200))
+			{
+				LastUpdateTimeMs = NowMs;
+
+				uint64_t Deleted	  = DeletedItemCount.load();
+				uint64_t DeletedBytes = DeletedByteCount.load();
+				uint64_t Discovered	  = DiscoveredItemCount.load();
+				Progress.UpdateState({.Task		  = "Cleaning folder  ",
+									  .Details	  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+									  .TotalCount = Discovered,
+									  .RemainingCount = Discovered - Deleted},
+									 false);
 			}
 		}
+
+		Progress.Finish();
+
+		uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
+		if (ElapsedTimeMs >= 200)
+		{
+			ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}",
+						Path,
+						DiscoveredItemCount.load(),
+						NiceBytes(DeletedByteCount.load()),
+						NiceTimeSpanMs(ElapsedTimeMs));
+		}
 		return CleanWipe;
 	}
 
@@ -1959,7 +2089,7 @@ namespace {
 
 			BasicFile		CompressedFile;
 			std::error_code Ec;
-			CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncate, Ec);
+			CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec);
 			if (Ec)
 			{
 				throw std::runtime_error(
@@ -2406,7 +2536,7 @@ namespace {
 										PartPayload.SetContentType(ZenContentType::kBinary);
 										return PartPayload;
 									},
-									[&, Payload, RawSize](uint64_t SentBytes, bool IsComplete) {
+									[&, RawSize](uint64_t SentBytes, bool IsComplete) {
 										UploadStats.ChunksBytes += SentBytes;
 										UploadedCompressedChunkSize += SentBytes;
 										if (IsComplete)
@@ -4159,7 +4289,10 @@ namespace {
 
 	bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters)
 	{
-		return SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1;
+		uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1);
+		ZEN_ASSERT(PreviousValue >= 1);
+		ZEN_ASSERT(PreviousValue != (uint32_t)-1);
+		return PreviousValue == 1;
 	}
 
 	std::vector<uint32_t> CompleteChunkTargets(const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
@@ -5680,7 +5813,31 @@ namespace {
 												{
 													BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash);
 												}
-												if (!BuildBlob)
+												if (BuildBlob)
+												{
+													uint64_t BlobSize = BuildBlob.GetSize();
+													DownloadStats.DownloadedChunkCount++;
+													DownloadStats.DownloadedChunkByteCount += BlobSize;
+													DownloadStats.RequestsCompleteCount++;
+													if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+													{
+														FilteredDownloadedBytesPerSecond.Stop();
+													}
+													AsyncWriteDownloadedChunk(ZenFolderPath,
+																			  RemoteContent,
+																			  RemoteLookup,
+																			  RemoteChunkIndex,
+																			  std::move(ChunkTargetPtrs),
+																			  Work,
+																			  WritePool,
+																			  std::move(BuildBlob),
+																			  SequenceIndexChunksLeftToWriteCounters,
+																			  WritePartsComplete,
+																			  TotalPartWriteCount,
+																			  FilteredWrittenBytesPerSecond,
+																			  DiskStats);
+												}
+												else
 												{
 													if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize)
 													{
@@ -6420,6 +6577,10 @@ namespace {
 						const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex];
 						ZEN_ASSERT(!IncompletePath.empty());
 						const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
+						ZEN_CONSOLE("{}: Max count {}, Current count {}",
+									IncompletePath,
+									ExpectedSequenceCount,
+									SequenceIndexChunksLeftToWriteCounter.load());
 						ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount);
 					}
 				}
@@ -6453,8 +6614,14 @@ namespace {
 		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceHashToLocalPathIndex;
 		std::vector<uint32_t>							 RemoveLocalPathIndexes;
 
-		if (!WipeTargetFolder)
+		if (AbortFlag)
+		{
+			return;
+		}
+
 		{
+			ZEN_TRACE_CPU("UpdateFolder_PrepareTarget");
+
 			tsl::robin_set<IoHash, IoHash::Hasher> CachedRemoteSequences;
 			tsl::robin_map<std::string, uint32_t>  RemotePathToRemoteIndex;
 			RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size());
@@ -6463,14 +6630,21 @@ namespace {
 				RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex});
 			}
 
-			uint64_t MatchCount		   = 0;
-			uint64_t PathMismatchCount = 0;
-			uint64_t HashMismatchCount = 0;
-			uint64_t CachedCount	   = 0;
-			uint64_t SkippedCount	   = 0;
-			uint64_t DeleteCount	   = 0;
+			std::vector<uint32_t> FilesToCache;
+
+			uint64_t			  MatchCount		= 0;
+			uint64_t			  PathMismatchCount = 0;
+			uint64_t			  HashMismatchCount = 0;
+			std::atomic<uint64_t> CachedCount		= 0;
+			std::atomic<uint64_t> CachedByteCount	= 0;
+			uint64_t			  SkippedCount		= 0;
+			uint64_t			  DeleteCount		= 0;
 			for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++)
 			{
+				if (AbortFlag)
+				{
+					break;
+				}
 				const IoHash&				 RawHash   = LocalContent.RawHashes[LocalPathIndex];
 				const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex];
 
@@ -6502,22 +6676,16 @@ namespace {
 				}
 				if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash))
 				{
-					const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
 					if (!CachedRemoteSequences.contains(RawHash))
 					{
+						ZEN_TRACE_CPU("MoveToCache");
 						// We need it
-						ZEN_ASSERT_SLOW(!IsFile(CacheFilePath));
-						const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred();
-
-						RenameFileWithRetry(LocalFilePath, CacheFilePath);
-
+						FilesToCache.push_back(LocalPathIndex);
 						CachedRemoteSequences.insert(RawHash);
-						CachedCount++;
 					}
 					else
 					{
 						// We already have it
-						ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
 						SkippedCount++;
 					}
 				}
@@ -6529,15 +6697,80 @@ namespace {
 				}
 			}
 
-			ZEN_DEBUG(
-				"Local state prep: MatchCount: {}, PathMismatchCount: {}, HashMismatchCount: {}, CachedCount: {}, SkippedCount: {}, "
-				"DeleteCount: {}",
-				MatchCount,
-				PathMismatchCount,
-				HashMismatchCount,
-				CachedCount,
-				SkippedCount,
-				DeleteCount);
+			if (AbortFlag)
+			{
+				return;
+			}
+
+			{
+				ZEN_TRACE_CPU("UpdateFolder_CopyToCache");
+
+				Stopwatch Timer;
+
+				WorkerThreadPool& WritePool = GetIOWorkerPool();
+
+				ProgressBar	  CacheLocalProgressBar(UsePlainProgress);
+				ParallellWork Work(AbortFlag);
+
+				for (uint32_t LocalPathIndex : FilesToCache)
+				{
+					if (AbortFlag)
+					{
+						break;
+					}
+					Work.ScheduleWork(
+						WritePool,
+						[&, LocalPathIndex](std::atomic<bool>&) {
+							ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache");
+							if (!AbortFlag)
+							{
+								const IoHash&				 RawHash	   = LocalContent.RawHashes[LocalPathIndex];
+								const std::filesystem::path& LocalPath	   = LocalContent.Paths[LocalPathIndex];
+								const std::filesystem::path	 CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
+								ZEN_ASSERT_SLOW(!IsFile(CacheFilePath));
+								const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred();
+								RenameFileWithRetry(LocalFilePath, CacheFilePath);
+								CachedCount++;
+								CachedByteCount += LocalContent.RawSizes[LocalPathIndex];
+							}
+						},
+						Work.DefaultErrorFunction());
+				}
+
+				{
+					ZEN_TRACE_CPU("CacheLocal_Wait");
+
+					Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
+						ZEN_UNUSED(IsAborted, PendingWork);
+						const uint64_t WorkTotal	= FilesToCache.size();
+						const uint64_t WorkComplete = CachedCount.load();
+						std::string	   Details		= fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(CachedByteCount));
+						CacheLocalProgressBar.UpdateState({.Task		   = "Caching local    ",
+														   .Details		   = Details,
+														   .TotalCount	   = gsl::narrow<uint64_t>(WorkTotal),
+														   .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete)},
+														  false);
+					});
+				}
+
+				if (AbortFlag)
+				{
+					return;
+				}
+
+				CacheLocalProgressBar.Finish();
+
+				ZEN_DEBUG(
+					"Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, "
+					"Delete: {}",
+					MatchCount,
+					PathMismatchCount,
+					HashMismatchCount,
+					CachedCount.load(),
+					NiceBytes(CachedByteCount.load()),
+					SkippedCount,
+					DeleteCount);
+			}
 		}
 
 		if (WipeTargetFolder)
@@ -6546,30 +6779,16 @@ namespace {
 			Stopwatch Timer;
 
 			// Clean target folder
-			ZEN_CONSOLE("Wiping {}", Path);
 			if (!CleanDirectory(Path, DefaultExcludeFolders))
 			{
 				ZEN_WARN("Some files in {} could not be removed", Path);
 			}
 			RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs();
 		}
-		else
-		{
-			ZEN_TRACE_CPU("UpdateFolder_RemoveUnused");
-			Stopwatch Timer;
-
-			// Remove unused tracked files
 
-			if (!RemoveLocalPathIndexes.empty())
-			{
-				ZEN_CONSOLE("Cleaning {} removed files from {}", RemoveLocalPathIndexes.size(), Path);
-				for (uint32_t LocalPathIndex : RemoveLocalPathIndexes)
-				{
-					const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
-					SetFileReadOnlyWithRetry(LocalFilePath, false);
-					RemoveFileWithRetry(LocalFilePath);
-				}
-			}
+		if (AbortFlag)
+		{
+			return;
 		}
 
 		{
@@ -6586,6 +6805,28 @@ namespace {
 			OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size());
 			OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size());
 
+			std::atomic<uint64_t> DeletedCount = 0;
+
+			for (uint32_t LocalPathIndex : RemoveLocalPathIndexes)
+			{
+				if (AbortFlag)
+				{
+					break;
+				}
+				Work.ScheduleWork(
+					WritePool,
+					[&, LocalPathIndex](std::atomic<bool>&) {
+						if (!AbortFlag)
+						{
+							const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+							SetFileReadOnlyWithRetry(LocalFilePath, false);
+							RemoveFileWithRetry(LocalFilePath);
+							DeletedCount++;
+						}
+					},
+					Work.DefaultErrorFunction());
+			}
+
 			std::atomic<uint64_t> TargetsComplete = 0;
 
 			struct FinalizeTarget
@@ -6638,6 +6879,7 @@ namespace {
 
 							if (RawHash == IoHash::Zero)
 							{
+								ZEN_TRACE_CPU("ZeroSize");
 								while (TargetOffset < (BaseTargetOffset + TargetCount))
 								{
 									const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
@@ -6674,6 +6916,7 @@ namespace {
 							}
 							else
 							{
+								ZEN_TRACE_CPU("Files");
 								ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash));
 								const uint32_t				 FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex;
 								const std::filesystem::path& FirstTargetPath	  = RemoteContent.Paths[FirstRemotePathIndex];
@@ -6698,16 +6941,19 @@ namespace {
 									if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash);
 										InplaceIt != SequenceHashToLocalPathIndex.end())
 									{
+										ZEN_TRACE_CPU("Copy");
 										const uint32_t				 LocalPathIndex = InplaceIt->second;
 										const std::filesystem::path& SourcePath		= LocalContent.Paths[LocalPathIndex];
 										std::filesystem::path		 SourceFilePath = (Path / SourcePath).make_preferred();
 										ZEN_ASSERT_SLOW(IsFile(SourceFilePath));
 
+										ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath);
 										CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false});
 										RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
 									}
 									else
 									{
+										ZEN_TRACE_CPU("Rename");
 										const std::filesystem::path CacheFilePath =
 											GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
 										ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
@@ -6747,6 +6993,7 @@ namespace {
 									}
 									else
 									{
+										ZEN_TRACE_CPU("Copy");
 										if (IsFile(TargetFilePath))
 										{
 											SetFileReadOnlyWithRetry(TargetFilePath, false);
@@ -6757,6 +7004,7 @@ namespace {
 										}
 
 										ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath));
+										ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath);
 										CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false});
 										RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
 									}
@@ -6788,11 +7036,13 @@ namespace {
 
 				Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
 					ZEN_UNUSED(IsAborted, PendingWork);
-					std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size());
+					const uint64_t WorkTotal	= Targets.size() + RemoveLocalPathIndexes.size();
+					const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load();
+					std::string	   Details		= fmt::format("{}/{} files", WorkComplete, WorkTotal);
 					RebuildProgressBar.UpdateState({.Task			= "Rebuilding state ",
 													.Details		= Details,
-													.TotalCount		= gsl::narrow<uint64_t>(Targets.size()),
-													.RemainingCount = gsl::narrow<uint64_t>(Targets.size() - TargetsComplete.load())},
+													.TotalCount		= gsl::narrow<uint64_t>(WorkTotal),
+													.RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete)},
 												   false);
 				});
 			}
@@ -7312,7 +7562,7 @@ namespace {
 
 				while (PathIndex < PathCount)
 				{
-					uint32_t PathRangeCount = Min(1024u, PathCount - PathIndex);
+					uint32_t PathRangeCount = Min(128u, PathCount - PathIndex);
 					Work.ScheduleWork(
 						GetIOWorkerPool(),
 						[PathIndex,
@@ -7326,17 +7576,16 @@ namespace {
 							{
 								const std::filesystem::path& FilePath	   = PathsToCheck[PathRangeIndex];
 								std::filesystem::path		 LocalFilePath = (Path / FilePath).make_preferred();
-								if (IsFile(LocalFilePath))
+								if (TryGetFileProperties(LocalFilePath,
+														 OutLocalFolderContent.RawSizes[PathRangeIndex],
+														 OutLocalFolderContent.ModificationTicks[PathRangeIndex],
+														 OutLocalFolderContent.Attributes[PathRangeIndex]))
 								{
-									const uint64_t FileSize									= FileSizeFromPath(LocalFilePath);
-									OutLocalFolderContent.Paths[PathRangeIndex]				= FilePath;
-									OutLocalFolderContent.RawSizes[PathRangeIndex]			= FileSize;
-									OutLocalFolderContent.Attributes[PathRangeIndex]		= GetNativeFileAttributes(LocalFilePath);
-									OutLocalFolderContent.ModificationTicks[PathRangeIndex] = GetModificationTickFromPath(LocalFilePath);
+									OutLocalFolderContent.Paths[PathRangeIndex] = std::move(FilePath);
 									LocalFolderScanStats.FoundFileCount++;
-									LocalFolderScanStats.FoundFileByteCount += FileSize;
+									LocalFolderScanStats.FoundFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex];
 									LocalFolderScanStats.AcceptedFileCount++;
-									LocalFolderScanStats.AcceptedFileByteCount += FileSize;
+									LocalFolderScanStats.AcceptedFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex];
 								}
 								CompletedPathCount++;
 							}
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 6ff4dd053..4ec563ba3 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -280,16 +280,17 @@ bool
 RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec)
 {
 #if ZEN_PLATFORM_WINDOWS
-	BOOL Success = ::DeleteFile(Path.native().c_str());
+	const std::filesystem::path::value_type* NativePath = Path.native().c_str();
+	BOOL									 Success	= ::DeleteFile(NativePath);
 	if (!Success)
 	{
 		if (ForceRemoveReadOnlyFiles)
 		{
-			DWORD FileAttributes = ::GetFileAttributes(Path.native().c_str());
+			DWORD FileAttributes = ::GetFileAttributes(NativePath);
 			if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0)
 			{
-				::SetFileAttributes(Path.native().c_str(), MakeFileAttributeReadOnly(FileAttributes, false));
-				Success = ::DeleteFile(Path.native().c_str());
+				::SetFileAttributes(NativePath, MakeFileAttributeReadOnly(FileAttributes, false));
+				Success = ::DeleteFile(NativePath);
 			}
 		}
 		if (!Success)
@@ -1964,6 +1965,52 @@ GetModificationTickFromPath(const std::filesystem::path& Filename)
 #endif
 }
 
+bool
+TryGetFileProperties(const std::filesystem::path& Path,
+					 uint64_t&					  OutSize,
+					 uint64_t&					  OutModificationTick,
+					 uint32_t&					  OutNativeModeOrAttributes)
+{
+#if ZEN_PLATFORM_WINDOWS
+	const std::filesystem::path::value_type* NativePath = Path.native().c_str();
+	{
+		void* Handle = CreateFileW(NativePath,
+								   GENERIC_READ,
+								   FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+								   nullptr,
+								   OPEN_EXISTING,
+								   0,
+								   nullptr);
+		if (Handle == INVALID_HANDLE_VALUE)
+		{
+			return false;
+		}
+		auto _ = MakeGuard([Handle]() { CloseHandle(Handle); });
+
+		BY_HANDLE_FILE_INFORMATION Bhfh = {};
+		if (!GetFileInformationByHandle(Handle, &Bhfh))
+		{
+			return false;
+		}
+		OutSize					  = uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow;
+		OutModificationTick		  = ((uint64_t(Bhfh.ftLastWriteTime.dwHighDateTime) << 32) | Bhfh.ftLastWriteTime.dwLowDateTime);
+		OutNativeModeOrAttributes = Bhfh.dwFileAttributes;
+		return true;
+	}
+#else
+	struct stat Stat;
+	int			err = stat(Path.native().c_str(), &Stat);
+	if (err)
+	{
+		return false;
+	}
+	OutModificationTick		  = gsl::narrow<uint64_t>(Stat.st_mtime);
+	OutSize					  = size_t(Stat.st_size);
+	OutNativeModeOrAttributes = (uint32_t)Stat.st_mode;
+	return true;
+#endif
+}
+
 void
 RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
 {
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index c23f16d03..66deffa6f 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -110,6 +110,11 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro
  */
 ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename);
 
+ZENCORE_API bool TryGetFileProperties(const std::filesystem::path& Path,
+									  uint64_t&					   OutSize,
+									  uint64_t&					   OutModificationTick,
+									  uint32_t&					   OutNativeModeOrAttributes);
+
 /** Move a file, if the files are not on the same drive the function will fail
  */
 ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath);