optimize scavenge (#697)

* optimize FindScavengeContent
* optimize GetValidFolderContent

1 files changed, 19 insertions, 85 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index a2b2fb0f6..25f66e0ee 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -875,87 +875,6 @@ namespace {
 		}
 	}
 
-	FolderContent GetValidFolderContent(TransferThreadWorkers&												   Workers,
-										GetFolderContentStatistics&											   LocalFolderScanStats,
-										const std::filesystem::path&										   Path,
-										std::span<const std::filesystem::path>								   PathsToCheck,
-										std::function<void(uint64_t PathCount, uint64_t CompletedPathCount)>&& ProgressCallback)
-	{
-		ZEN_TRACE_CPU("GetValidFolderContent");
-		FolderContent  Result;
-		const uint32_t PathCount = gsl::narrow<uint32_t>(PathsToCheck.size());
-
-		Result.Paths.resize(PathCount);
-		Result.RawSizes.resize(PathCount);
-		Result.Attributes.resize(PathCount);
-		Result.ModificationTicks.resize(PathCount);
-
-		{
-			Stopwatch Timer;
-			auto _ = MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
-
-			ParallelWork		  Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-			std::atomic<uint64_t> CompletedPathCount = 0;
-			uint32_t			  PathIndex			 = 0;
-
-			while (PathIndex < PathCount)
-			{
-				uint32_t PathRangeCount = Min(128u, PathCount - PathIndex);
-				Work.ScheduleWork(Workers.GetIOWorkerPool(),
-								  [PathIndex, PathRangeCount, &PathsToCheck, &Path, &Result, &CompletedPathCount, &LocalFolderScanStats](
-									  std::atomic<bool>&) {
-									  for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount;
-										   PathRangeIndex++)
-									  {
-										  const std::filesystem::path& FilePath		 = PathsToCheck[PathRangeIndex];
-										  std::filesystem::path		   LocalFilePath = (Path / FilePath).make_preferred();
-										  if (TryGetFileProperties(LocalFilePath,
-																   Result.RawSizes[PathRangeIndex],
-																   Result.ModificationTicks[PathRangeIndex],
-																   Result.Attributes[PathRangeIndex]))
-										  {
-											  Result.Paths[PathRangeIndex] = std::move(FilePath);
-											  LocalFolderScanStats.FoundFileCount++;
-											  LocalFolderScanStats.FoundFileByteCount += Result.RawSizes[PathRangeIndex];
-											  LocalFolderScanStats.AcceptedFileCount++;
-											  LocalFolderScanStats.AcceptedFileByteCount += Result.RawSizes[PathRangeIndex];
-										  }
-										  CompletedPathCount++;
-									  }
-								  });
-				PathIndex += PathRangeCount;
-			}
-			Work.Wait(200, [&](bool, bool, ptrdiff_t) {
-				if (ProgressCallback)
-				{
-					ProgressCallback(PathCount, CompletedPathCount.load());
-				}
-			});
-		}
-
-		uint32_t WritePathIndex = 0;
-		for (uint32_t ReadPathIndex = 0; ReadPathIndex < PathCount; ReadPathIndex++)
-		{
-			if (!Result.Paths[ReadPathIndex].empty())
-			{
-				if (WritePathIndex < ReadPathIndex)
-				{
-					Result.Paths[WritePathIndex]			 = std::move(Result.Paths[ReadPathIndex]);
-					Result.RawSizes[WritePathIndex]			 = Result.RawSizes[ReadPathIndex];
-					Result.Attributes[WritePathIndex]		 = Result.Attributes[ReadPathIndex];
-					Result.ModificationTicks[WritePathIndex] = Result.ModificationTicks[ReadPathIndex];
-				}
-				WritePathIndex++;
-			}
-		}
-
-		Result.Paths.resize(WritePathIndex);
-		Result.RawSizes.resize(WritePathIndex);
-		Result.Attributes.resize(WritePathIndex);
-		Result.ModificationTicks.resize(WritePathIndex);
-		return Result;
-	}
-
 	std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix)
 	{
 		ExtendableStringBuilder<512>					 SB;
@@ -1380,7 +1299,7 @@ namespace {
 		{
 			ProgressBar ProgressBar(ProgressMode, "Check Files");
 			FolderState = GetValidFolderContent(
-				Workers,
+				Workers.GetIOWorkerPool(),
 				LocalFolderScanStats,
 				Path,
 				PathsToCheck,
@@ -1393,7 +1312,10 @@ namespace {
 											 .RemainingCount = PathCount - CompletedPathCount,
 											 .Status		 = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
 											false);
-				});
+				},
+				GetUpdateDelayMS(ProgressMode),
+				AbortFlag,
+				PauseFlag);
 			ProgressBar.Finish();
 		}
 
@@ -1478,7 +1400,7 @@ namespace {
 		{
 			ProgressBar ProgressBar(ProgressMode, "Check Known Files");
 			CurrentLocalFolderState = GetValidFolderContent(
-				Workers,
+				Workers.GetIOWorkerPool(),
 				LocalFolderScanStats,
 				Path,
 				SavedLocalState.FolderState.Paths,
@@ -1491,7 +1413,10 @@ namespace {
 											 .RemainingCount = PathCount - CompletedPathCount,
 											 .Status		 = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
 											false);
-				});
+				},
+				GetUpdateDelayMS(ProgressMode),
+				AbortFlag,
+				PauseFlag);
 			ProgressBar.Finish();
 		}
 		if (AbortFlag)
@@ -2007,6 +1932,7 @@ namespace {
 
 					ZEN_CONSOLE(
 						"Downloaded build {}, parts:{} in {}\n"
+						"    Scavenge:   {} (Target: {}, Cache: {}, Others: {})\n"
 						"    Download:   {} ({}) {}bits/s{}\n"
 						"    Write:      {} ({}) {}B/s{}\n"
 						"    Clean:      {}\n"
@@ -2016,6 +1942,14 @@ namespace {
 						BuildPartString.ToView(),
 						NiceTimeSpanMs(DownloadTimeMs),
 
+						NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs +
+										Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs +
+										Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) /
+									   1000),
+						NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000),
+						NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000),
+						NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000),
+
 						DownloadCount,
 						NiceBytes(DownloadByteCount),
 						NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),