check file from local track state during download (#329)

1 files changed, 147 insertions, 118 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index d4add0e04..e3dc20621 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -7163,8 +7163,79 @@ namespace {
 										 const ChunkedFolderContent&  ReferenceContent,
 										 FolderContent&				  OutLocalFolderContent)
 	{
+		FolderContent		 LocalFolderState;
+		ChunkedFolderContent LocalContent;
+
+		bool HasLocalState = false;
+		if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath)))
+		{
+			try
+			{
+				Stopwatch ReadStateTimer;
+				CbObject  CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object;
+				if (CurrentStateObject)
+				{
+					Oid								  CurrentBuildId;
+					std::vector<Oid>				  SavedBuildPartIds;
+					std::vector<std::string>		  SavedBuildPartsNames;
+					std::vector<ChunkedFolderContent> SavedPartContents;
+					if (ReadStateObject(CurrentStateObject,
+										CurrentBuildId,
+										SavedBuildPartIds,
+										SavedBuildPartsNames,
+										SavedPartContents,
+										LocalFolderState))
+					{
+						if (!SavedPartContents.empty())
+						{
+							if (SavedPartContents.size() == 1)
+							{
+								LocalContent = std::move(SavedPartContents[0]);
+							}
+							else
+							{
+								LocalContent =
+									MergeChunkedFolderContents(SavedPartContents[0],
+															   std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1));
+							}
+							HasLocalState = true;
+						}
+					}
+				}
+				ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what());
+			}
+		}
+
 		{
-			const uint32_t PathCount = gsl::narrow<uint32_t>(ReferenceContent.Paths.size());
+			const uint32_t LocalPathCount  = gsl::narrow<uint32_t>(ReferenceContent.Paths.size());
+			const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size());
+
+			std::vector<std::filesystem::path> PathsToCheck;
+			PathsToCheck.reserve(LocalPathCount + RemotePathCount);
+
+			tsl::robin_set<std::string> FileSet;
+			FileSet.reserve(LocalPathCount + RemotePathCount);
+
+			for (const std::filesystem::path& LocalPath : LocalFolderState.Paths)
+			{
+				FileSet.insert(LocalPath.generic_string());
+				PathsToCheck.push_back(LocalPath);
+			}
+
+			for (const std::filesystem::path& RemotePath : ReferenceContent.Paths)
+			{
+				if (FileSet.insert(RemotePath.generic_string()).second)
+				{
+					PathsToCheck.push_back(RemotePath);
+				}
+			}
+
+			const uint32_t PathCount = gsl::narrow<uint32_t>(PathsToCheck.size());
+
 			OutLocalFolderContent.Paths.resize(PathCount);
 			OutLocalFolderContent.RawSizes.resize(PathCount);
 			OutLocalFolderContent.Attributes.resize(PathCount);
@@ -7188,14 +7259,14 @@ namespace {
 						GetIOWorkerPool(),
 						[PathIndex,
 						 PathRangeCount,
-						 &ReferenceContent,
+						 &PathsToCheck,
 						 &Path,
 						 &OutLocalFolderContent,
 						 &CompletedPathCount,
 						 &LocalFolderScanStats](std::atomic<bool>&) {
 							for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++)
 							{
-								const std::filesystem::path& FilePath	   = ReferenceContent.Paths[PathRangeIndex];
+								const std::filesystem::path& FilePath	   = PathsToCheck[PathRangeIndex];
 								std::filesystem::path		 LocalFilePath = (Path / FilePath).make_preferred();
 								if (std::filesystem::exists(LocalFilePath))
 								{
@@ -7252,132 +7323,90 @@ namespace {
 			OutLocalFolderContent.ModificationTicks.resize(WritePathIndex);
 		}
 
-		FolderContent		 LocalFolderState;
-		ChunkedFolderContent LocalContent;
-
 		bool				  ScanContent = true;
 		std::vector<uint32_t> PathIndexesOufOfDate;
-		if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath)))
+		if (HasLocalState)
 		{
-			try
+			if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent))
 			{
-				Stopwatch ReadStateTimer;
-				CbObject  CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object;
-				if (CurrentStateObject)
+				const size_t					   LocaStatePathCount = LocalFolderState.Paths.size();
+				std::vector<std::filesystem::path> DeletedPaths;
+				FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths);
+				if (!DeletedPaths.empty())
 				{
-					Oid								  CurrentBuildId;
-					std::vector<Oid>				  SavedBuildPartIds;
-					std::vector<std::string>		  SavedBuildPartsNames;
-					std::vector<ChunkedFolderContent> SavedPartContents;
-					if (ReadStateObject(CurrentStateObject,
-										CurrentBuildId,
-										SavedBuildPartIds,
-										SavedBuildPartsNames,
-										SavedPartContents,
-										LocalFolderState))
-					{
-						if (!SavedPartContents.empty())
-						{
-							if (SavedPartContents.size() == 1)
-							{
-								LocalContent = std::move(SavedPartContents[0]);
-							}
-							else
-							{
-								LocalContent =
-									MergeChunkedFolderContents(SavedPartContents[0],
-															   std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1));
-							}
-
-							if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent))
-							{
-								const size_t					   LocaStatePathCount = LocalFolderState.Paths.size();
-								std::vector<std::filesystem::path> DeletedPaths;
-								FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths);
-								if (!DeletedPaths.empty())
-								{
-									LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
-								}
+					LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
+				}
 
-								ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
-											DeletedPaths.size(),
-											UpdatedContent.Paths.size(),
-											LocaStatePathCount);
-								if (UpdatedContent.Paths.size() > 0)
-								{
-									uint64_t ByteCountToScan = 0;
-									for (const uint64_t RawSize : UpdatedContent.RawSizes)
-									{
-										ByteCountToScan += RawSize;
-									}
-									ProgressBar	 ProgressBar(false);
-									FilteredRate FilteredBytesHashed;
-									FilteredBytesHashed.Start();
-									ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
-										ChunkingStats,
-										GetIOWorkerPool(),
-										Path,
-										UpdatedContent,
-										ChunkController,
-										UsePlainProgress ? 5000 : 200,
-										[&](bool, std::ptrdiff_t) {
-											FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
-											std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
-																			  ChunkingStats.FilesProcessed.load(),
-																			  UpdatedContent.Paths.size(),
-																			  NiceBytes(ChunkingStats.BytesHashed.load()),
-																			  NiceBytes(ByteCountToScan),
-																			  NiceNum(FilteredBytesHashed.GetCurrent()),
-																			  ChunkingStats.UniqueChunksFound.load(),
-																			  NiceBytes(ChunkingStats.UniqueBytesFound.load()));
-											ProgressBar.UpdateState({.Task			 = "Scanning files   ",
-																	 .Details		 = Details,
-																	 .TotalCount	 = ByteCountToScan,
-																	 .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()},
-																	false);
-										},
-										AbortFlag);
-									if (AbortFlag)
-									{
-										return {};
-									}
-									FilteredBytesHashed.Stop();
-									ProgressBar.Finish();
-									LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}});
-								}
-							}
-							else
-							{
-								// Remove files from LocalContent no longer in LocalFolderState
-								tsl::robin_set<std::string> LocalFolderPaths;
-								LocalFolderPaths.reserve(LocalFolderState.Paths.size());
-								for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths)
-								{
-									LocalFolderPaths.insert(LocalFolderPath.generic_string());
-								}
-								std::vector<std::filesystem::path> DeletedPaths;
-								for (const std::filesystem::path& LocalContentPath : LocalContent.Paths)
-								{
-									if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
-									{
-										DeletedPaths.push_back(LocalContentPath);
-									}
-								}
-								if (!DeletedPaths.empty())
-								{
-									LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
-								}
-							}
-							ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
-							ScanContent = false;
-						}
+				ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
+							DeletedPaths.size(),
+							UpdatedContent.Paths.size(),
+							LocaStatePathCount);
+				if (UpdatedContent.Paths.size() > 0)
+				{
+					uint64_t ByteCountToScan = 0;
+					for (const uint64_t RawSize : UpdatedContent.RawSizes)
+					{
+						ByteCountToScan += RawSize;
 					}
+					ProgressBar	 ProgressBar(false);
+					FilteredRate FilteredBytesHashed;
+					FilteredBytesHashed.Start();
+					ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
+						ChunkingStats,
+						GetIOWorkerPool(),
+						Path,
+						UpdatedContent,
+						ChunkController,
+						UsePlainProgress ? 5000 : 200,
+						[&](bool, std::ptrdiff_t) {
+							FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
+							std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
+															  ChunkingStats.FilesProcessed.load(),
+															  UpdatedContent.Paths.size(),
+															  NiceBytes(ChunkingStats.BytesHashed.load()),
+															  NiceBytes(ByteCountToScan),
+															  NiceNum(FilteredBytesHashed.GetCurrent()),
+															  ChunkingStats.UniqueChunksFound.load(),
+															  NiceBytes(ChunkingStats.UniqueBytesFound.load()));
+							ProgressBar.UpdateState({.Task			 = "Scanning files   ",
+													 .Details		 = Details,
+													 .TotalCount	 = ByteCountToScan,
+													 .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()},
+													false);
+						},
+						AbortFlag);
+					if (AbortFlag)
+					{
+						return {};
+					}
+					FilteredBytesHashed.Stop();
+					ProgressBar.Finish();
+					LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}});
 				}
 			}
-			catch (const std::exception& Ex)
+			else
 			{
-				ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what());
+				// Remove files from LocalContent no longer in LocalFolderState
+				tsl::robin_set<std::string> LocalFolderPaths;
+				LocalFolderPaths.reserve(LocalFolderState.Paths.size());
+				for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths)
+				{
+					LocalFolderPaths.insert(LocalFolderPath.generic_string());
+				}
+				std::vector<std::filesystem::path> DeletedPaths;
+				for (const std::filesystem::path& LocalContentPath : LocalContent.Paths)
+				{
+					if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
+					{
+						DeletedPaths.push_back(LocalContentPath);
+					}
+				}
+				if (!DeletedPaths.empty())
+				{
+					LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
+				}
 			}
+			ScanContent = false;
 		}
 
 		if (ScanContent)