caller controls threshold for bulk-loading chunks in IterateChunks (#222)

* Allow caller to control threshold for bulk-loading chunks in IterateChunks
* use smaller batch chunk reading for /fileinfos and /chunkinfos as we do not intend to read the payload
* use smaller batch read buffer when just querying for size of attachments

1 files changed, 11 insertions, 6 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index a43c9e0e2..3bcc3c51d 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1922,15 +1922,17 @@ ProjectStore::Oplog::GetChunkByRawHash(const IoHash& RawHash)
 bool
 ProjectStore::Oplog::IterateChunks(std::span<IoHash>												 RawHashes,
 								   const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
-								   WorkerThreadPool*												 OptionalWorkerPool)
+								   WorkerThreadPool*												 OptionalWorkerPool,
+								   uint64_t															 LargeSizeLimit)
 {
-	return m_CidStore.IterateChunks(RawHashes, AsyncCallback, OptionalWorkerPool);
+	return m_CidStore.IterateChunks(RawHashes, AsyncCallback, OptionalWorkerPool, LargeSizeLimit);
 }
 
 bool
 ProjectStore::Oplog::IterateChunks(std::span<Oid>													 ChunkIds,
 								   const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
-								   WorkerThreadPool*												 OptionalWorkerPool)
+								   WorkerThreadPool*												 OptionalWorkerPool,
+								   uint64_t															 LargeSizeLimit)
 {
 	std::vector<size_t>				   CidChunkIndexes;
 	std::vector<IoHash>				   CidChunkHashes;
@@ -1961,7 +1963,8 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid>													 ChunkIds,
 	m_CidStore.IterateChunks(
 		CidChunkHashes,
 		[&](size_t Index, const IoBuffer& Payload) { return AsyncCallback(CidChunkIndexes[Index], Payload); },
-		OptionalWorkerPool);
+		OptionalWorkerPool,
+		LargeSizeLimit);
 
 	if (OptionalWorkerPool)
 	{
@@ -3995,7 +3998,8 @@ ProjectStore::GetProjectFiles(const std::string_view				 ProjectId,
 				}
 				return true;
 			},
-			&GetSmallWorkerPool(EWorkloadType::Burst));
+			&GetSmallWorkerPool(EWorkloadType::Burst),
+			8u * 1024u);
 	}
 
 	CbObjectWriter Response;
@@ -4149,7 +4153,8 @@ ProjectStore::GetProjectChunkInfos(const std::string_view				  ProjectId,
 				}
 				return true;
 			},
-			&WorkerPool);
+			&WorkerPool,
+			8u * 1024u);
 	}
 
 	CbObjectWriter Response;