optimize scavenge (#697)

* optimize FindScavengeContent
* optimize GetValidFolderContent

2 files changed, 16 insertions, 8 deletions

diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index eca654223..d78ee29c1 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -259,11 +259,6 @@ private:
 
 	std::filesystem::path FindDownloadedChunk(const IoHash& ChunkHash);
 
-	FolderContent GetValidFolderContent(GetFolderContentStatistics&											   LocalFolderScanStats,
-										const std::filesystem::path&										   Path,
-										std::span<const std::filesystem::path>								   PathsToCheck,
-										std::function<void(uint64_t PathCount, uint64_t CompletedPathCount)>&& ProgressCallback);
-
 	std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> GetRemainingChunkTargets(
 		std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
 		uint32_t							   ChunkIndex);
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
index e4be7923a..78f20a727 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
@@ -71,6 +71,15 @@ FolderContent GetFolderContent(GetFolderContentStatistics&																Stats,
 							   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						UpdateCallback,
 							   std::atomic<bool>&																		AbortFlag);
 
+FolderContent GetValidFolderContent(WorkerThreadPool&													   WorkerPool,
+									GetFolderContentStatistics&											   FolderScanStats,
+									const std::filesystem::path&										   Path,
+									std::span<const std::filesystem::path>								   PathsToCheck,
+									std::function<void(uint64_t PathCount, uint64_t CompletedPathCount)>&& ProgressCallback,
+									uint32_t															   ProgressUpdateDelayMS,
+									std::atomic<bool>&													   AbortFlag,
+									std::atomic<bool>&													   PauseFlag);
+
 struct ChunkedContentData
 {
 	// To describe one asset with a particular RawHash, find the index of the hash in SequenceRawHashes
@@ -116,9 +125,10 @@ void				 SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Con
 ChunkedFolderContent LoadChunkedFolderContentFromCompactBinary(CbObjectView Input);
 
 ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays);
-ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent&			  Base,
-												   const ChunkedContentLookup&			  BaseContentLookup,
-												   std::span<const std::filesystem::path> DeletedPaths);
+ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent&							   Base,
+												   const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& RawHashToSequenceIndex,
+												   std::vector<uint32_t>								   SequenceIndexChunkOrderOffset,
+												   std::span<const std::filesystem::path>				   DeletedPaths);
 ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span<const std::filesystem::path> DeletedPaths);
 
 bool CompareChunkedContent(const ChunkedFolderContent& Lhs, const ChunkedFolderContent& Rhs);
@@ -161,6 +171,9 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&			 Stats,
 										std::atomic<bool>&																 AbortFlag,
 										std::atomic<bool>&																 PauseFlag);
 
+tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BuildHashLookup(std::span<const IoHash> Hashes);
+std::vector<uint32_t>							 BuildChunkOrderOffset(std::span<const uint32_t> ChunkCounts);
+
 ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content);
 
 inline std::pair<size_t, uint32_t>