use state file if available when doing builds diff command (#369)

* use state file if available when doing builds diff command
* remove dead code

1 files changed, 62 insertions, 81 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 6c97645cd..855d7012f 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -672,75 +672,6 @@ namespace {
 		return CacheFolderPath / RawHash.ToHexString();
 	}
 
-	ChunkedFolderContent ScanAndChunkFolder(
-		GetFolderContentStatistics&																 GetFolderContentStats,
-		ChunkingStatistics&																		 ChunkingStats,
-		const std::filesystem::path&															 Path,
-		std::function<bool(const std::string_view& RelativePath)>&&								 IsAcceptedFolder,
-		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
-		ChunkingController&																		 ChunkController)
-	{
-		ZEN_TRACE_CPU("ScanAndChunkFolder");
-
-		FolderContent Content = GetFolderContent(
-			GetFolderContentStats,
-			Path,
-			std::move(IsAcceptedFolder),
-			std::move(IsAcceptedFile),
-			GetIOWorkerPool(),
-			UsePlainProgress ? 5000 : 200,
-			[](bool, std::ptrdiff_t) {},
-			AbortFlag);
-		if (AbortFlag)
-		{
-			return {};
-		}
-
-		ProgressBar	 ProgressBar(UsePlainProgress);
-		FilteredRate FilteredBytesHashed;
-		FilteredBytesHashed.Start();
-		ChunkedFolderContent FolderContent = ChunkFolderContent(
-			ChunkingStats,
-			GetIOWorkerPool(),
-			Path,
-			Content,
-			ChunkController,
-			UsePlainProgress ? 5000 : 200,
-			[&](bool, std::ptrdiff_t) {
-				FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
-				std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
-												  ChunkingStats.FilesProcessed.load(),
-												  GetFolderContentStats.AcceptedFileCount.load(),
-												  NiceBytes(ChunkingStats.BytesHashed.load()),
-												  NiceBytes(GetFolderContentStats.FoundFileByteCount),
-												  NiceNum(FilteredBytesHashed.GetCurrent()),
-												  ChunkingStats.UniqueChunksFound.load(),
-												  NiceBytes(ChunkingStats.UniqueBytesFound.load()));
-				ProgressBar.UpdateState({.Task			 = "Scanning files   ",
-										 .Details		 = Details,
-										 .TotalCount	 = GetFolderContentStats.AcceptedFileByteCount,
-										 .RemainingCount = GetFolderContentStats.AcceptedFileByteCount - ChunkingStats.BytesHashed.load()},
-										false);
-			},
-			AbortFlag);
-		if (AbortFlag)
-		{
-			return {};
-		}
-		FilteredBytesHashed.Stop();
-		ProgressBar.Finish();
-
-		ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
-					ChunkingStats.FilesProcessed.load(),
-					NiceBytes(ChunkingStats.BytesHashed.load()),
-					ChunkingStats.UniqueChunksFound.load(),
-					NiceBytes(ChunkingStats.UniqueBytesFound.load()),
-					Path,
-					NiceTimeSpanMs((GetFolderContentStats.ElapsedWallTimeUS + ChunkingStats.ElapsedWallTimeUS) / 1000),
-					NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
-		return FolderContent;
-	};
-
 	struct DiskStatistics
 	{
 		std::atomic<uint64_t> OpenReadCount		   = 0;
@@ -8158,13 +8089,13 @@ namespace {
 		return RemoteContent;
 	}
 
-	ChunkedFolderContent GetLocalContent(GetFolderContentStatistics&  LocalFolderScanStats,
-										 ChunkingStatistics&		  ChunkingStats,
-										 const std::filesystem::path& Path,
-										 const std::filesystem::path& StateFilePath,
-										 ChunkingController&		  ChunkController,
-										 const ChunkedFolderContent&  ReferenceContent,
-										 FolderContent&				  OutLocalFolderContent)
+	ChunkedFolderContent GetLocalContent(GetFolderContentStatistics&			LocalFolderScanStats,
+										 ChunkingStatistics&					ChunkingStats,
+										 const std::filesystem::path&			Path,
+										 const std::filesystem::path&			StateFilePath,
+										 ChunkingController&					ChunkController,
+										 std::span<const std::filesystem::path> ReferencePaths,
+										 FolderContent&							OutLocalFolderContent)
 	{
 		FolderContent		 LocalFolderState;
 		ChunkedFolderContent LocalContent;
@@ -8176,7 +8107,7 @@ namespace {
 			ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
 		}
 		{
-			const uint32_t LocalPathCount  = gsl::narrow<uint32_t>(ReferenceContent.Paths.size());
+			const uint32_t LocalPathCount  = gsl::narrow<uint32_t>(ReferencePaths.size());
 			const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size());
 
 			std::vector<std::filesystem::path> PathsToCheck;
@@ -8191,7 +8122,7 @@ namespace {
 				PathsToCheck.push_back(LocalPath);
 			}
 
-			for (const std::filesystem::path& RemotePath : ReferenceContent.Paths)
+			for (const std::filesystem::path& RemotePath : ReferencePaths)
 			{
 				if (FileSet.insert(RemotePath.generic_string()).second)
 				{
@@ -8334,6 +8265,56 @@ namespace {
 		return LocalContent;
 	}
 
+	ChunkedFolderContent ScanAndChunkFolder(
+		GetFolderContentStatistics&																 GetFolderContentStats,
+		ChunkingStatistics&																		 ChunkingStats,
+		const std::filesystem::path&															 Path,
+		std::function<bool(const std::string_view& RelativePath)>&&								 IsAcceptedFolder,
+		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
+		ChunkingController&																		 ChunkController)
+	{
+		Stopwatch Timer;
+
+		ZEN_TRACE_CPU("ScanAndChunkFolder");
+
+		FolderContent Content = GetFolderContent(
+			GetFolderContentStats,
+			Path,
+			std::move(IsAcceptedFolder),
+			std::move(IsAcceptedFile),
+			GetIOWorkerPool(),
+			UsePlainProgress ? 5000 : 200,
+			[](bool, std::ptrdiff_t) {},
+			AbortFlag);
+		if (AbortFlag)
+		{
+			return {};
+		}
+
+		FolderContent		 _;
+		ChunkedFolderContent Result = GetLocalContent(GetFolderContentStats,
+													  ChunkingStats,
+													  Path,
+													  ZenStateFilePath(Path / ZenFolderName),
+													  ChunkController,
+													  Content.Paths,
+													  _);
+
+		const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
+		const uint64_t ChunkedRawSize =
+			std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
+
+		ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+					Result.Paths.size(),
+					NiceBytes(TotalRawSize),
+					Result.ChunkedContent.ChunkHashes.size(),
+					NiceBytes(ChunkedRawSize),
+					Path,
+					NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+					NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+		return Result;
+	};
+
 	void DownloadFolder(StorageInstance&			 Storage,
 						const Oid&					 BuildId,
 						const std::vector<Oid>&		 BuildPartIds,
@@ -8398,7 +8379,7 @@ namespace {
 												   Path,
 												   ZenStateFilePath(ZenFolderPath),
 												   *ChunkController,
-												   RemoteContent,
+												   RemoteContent.Paths,
 												   LocalFolderContent);
 				}
 			}
@@ -8695,7 +8676,7 @@ namespace {
 
 		double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0;
 
-		ZEN_CONSOLE("{} ({}) files removed, {} ({}) files added, {} ({} {:.1f}%) files kept",
+		ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept",
 					RemovedHashes.size(),
 					NiceBytes(RemovedSize),
 					AddedHashes.size(),
@@ -8730,7 +8711,7 @@ namespace {
 		double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0;
 		double NewPercent	= CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0;
 
-		ZEN_CONSOLE("Found {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
+		ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
 					FoundChunkCount,
 					NiceBytes(FoundChunkSize),
 					FoundPercent,