From bcb81b326a373aa86d7e6a046febc8ba74f21c04 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 25 Nov 2024 14:49:04 +0100 Subject: caller controls threshold for bulk-loading chunks in IterateChunks (#222) * Allow caller to control threshold for bulk-loading chunks in IterateChunks * use smaller batch chunk reading for /fileinfos and /chunkinfos as we do not intend to read the payload * use smaller batch read buffer when just querying for size of attachments --- src/zenstore/compactcas.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/zenstore/compactcas.cpp') diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index bc30301d1..792854af6 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -305,7 +305,8 @@ CasContainerStrategy::FilterChunks(HashKeySet& InOutChunks) bool CasContainerStrategy::IterateChunks(std::span ChunkHashes, const std::function& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool) + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) { if (ChunkHashes.size() < 3) { @@ -344,7 +345,8 @@ CasContainerStrategy::IterateChunks(std::span ChunkHashes, }, [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { return AsyncCallback(FoundChunkIndexes[ChunkIndex], File.GetChunk(Offset, Size)); - }); + }, + LargeSizeLimit); }; Latch WorkLatch(1); @@ -498,7 +500,7 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) }; m_BlockStore.IterateChunks(ChunkLocations, [&](uint32_t, std::span ChunkIndexes) { - return m_BlockStore.IterateBlock(ChunkLocations, ChunkIndexes, ValidateSmallChunk, ValidateLargeChunk); + return m_BlockStore.IterateBlock(ChunkLocations, ChunkIndexes, ValidateSmallChunk, ValidateLargeChunk, 0); }); } catch (const ScrubDeadlineExpiredException&) -- cgit v1.2.3