diff options
| author | Dan Engelbrecht <[email protected]> | 2024-11-25 14:49:04 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2024-11-25 14:49:04 +0100 |
| commit | bcb81b326a373aa86d7e6a046febc8ba74f21c04 (patch) | |
| tree | b20c6d59cefd299b4daac0754c8fab7ec7019b9c /src/zenserver | |
| parent | stronger validation of payload existance (#229) (diff) | |
| download | zen-bcb81b326a373aa86d7e6a046febc8ba74f21c04.tar.xz zen-bcb81b326a373aa86d7e6a046febc8ba74f21c04.zip | |
caller controls threshold for bulk-loading chunks in IterateChunks (#222)
* Allow caller to control threshold for bulk-loading chunks in IterateChunks
* use smaller batch chunk reading for /fileinfos and /chunkinfos as we do not intend to read the payload
* use smaller batch read buffer when just querying for size of attachments
Diffstat (limited to 'src/zenserver')
| -rw-r--r-- | src/zenserver/cache/httpstructuredcache.cpp | 6 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 17 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.h | 6 |
3 files changed, 19 insertions, 10 deletions
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index 551b5a76d..f49f6a645 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -672,7 +672,8 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque AttachmentsSize += Payload.GetSize(); return true; }, - &WorkerPool); + &WorkerPool, + 8u * 1024u); ResponseWriter << "Count" << AllAttachments.size(); ResponseWriter << "Size" << AttachmentsSize; @@ -765,7 +766,8 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, AttachmentsSize += Payload.GetSize(); return true; }, - &WorkerPool); + &WorkerPool, + 8u * 1024u); ResponseWriter << "AttachmentsSize" << AttachmentsSize; } diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index a43c9e0e2..3bcc3c51d 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1922,15 +1922,17 @@ ProjectStore::Oplog::GetChunkByRawHash(const IoHash& RawHash) bool ProjectStore::Oplog::IterateChunks(std::span<IoHash> RawHashes, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool) + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) { - return m_CidStore.IterateChunks(RawHashes, AsyncCallback, OptionalWorkerPool); + return m_CidStore.IterateChunks(RawHashes, AsyncCallback, OptionalWorkerPool, LargeSizeLimit); } bool ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool) + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) { std::vector<size_t> CidChunkIndexes; std::vector<IoHash> CidChunkHashes; @@ -1961,7 +1963,8 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, m_CidStore.IterateChunks( CidChunkHashes, [&](size_t Index, const IoBuffer& Payload) { return AsyncCallback(CidChunkIndexes[Index], Payload); }, - OptionalWorkerPool); + OptionalWorkerPool, + LargeSizeLimit); if (OptionalWorkerPool) { @@ -3995,7 +3998,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, } return true; }, - &GetSmallWorkerPool(EWorkloadType::Burst)); + &GetSmallWorkerPool(EWorkloadType::Burst), + 8u * 1024u); } CbObjectWriter Response; @@ -4149,7 +4153,8 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, } return true; }, - &WorkerPool); + &WorkerPool, + 8u * 1024u); } CbObjectWriter Response; diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index d8b13585b..860f2c17d 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -118,10 +118,12 @@ public: IoBuffer GetChunkByRawHash(const IoHash& RawHash); bool IterateChunks(std::span<IoHash> RawHashes, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool); + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); bool IterateChunks(std::span<Oid> ChunkIds, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool); + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); inline static const uint32_t kInvalidOp = ~0u; /** Persist a new oplog entry |