diff options
| author | Dan Engelbrecht <[email protected]> | 2025-12-15 15:19:32 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-12-15 15:19:32 +0100 |
| commit | 215ef33ab4a4f8deb0f08bedee5f9e7f744cebf4 (patch) | |
| tree | c065f3a0c294b54f050575d73e54ff5e39b073f2 | |
| parent | oplog download size (#690) (diff) | |
| download | zen-215ef33ab4a4f8deb0f08bedee5f9e7f744cebf4.tar.xz zen-215ef33ab4a4f8deb0f08bedee5f9e7f744cebf4.zip | |
remove found chunks as they are found in blocks (#691)
| -rw-r--r-- | CHANGELOG.md | 1 | ||||
| -rw-r--r-- | src/zenremotestore/projectstore/remoteprojectstore.cpp | 25 |
2 files changed, 19 insertions, 7 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index ec8cb4906..aa32841e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Bugfix: Upload of oplogs could reference multiple blocks for the same chunk causing redundant downloads of blocks - Improvement: At end of `zen builds download` and `zen oplog-import` we now show information over the source of the data - cache, cloud storage, s3/azure, replication - Improvement: Use the improved block reuse selection function from `zen builds upload` in `zen oplog-export` to reduce oplog download size +- Improvement: Remove duplicate download of chunks from different blocks in `zen oplog-export` ## 5.7.14 - Improvement: asio http server now supports the `--dedicated` option which prevents port remapping diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index 0e18cc6b0..b566e5bed 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -2830,9 +2830,12 @@ ParseOplogContainer(const CbObject& ContainerObject, for (CbFieldView ChunkField : ChunksArray) { IoHash ChunkHash = ChunkField.AsBinaryAttachment(); - if (OpsAttachments.contains(ChunkHash) && !HasAttachment(ChunkHash)) + if (OpsAttachments.erase(ChunkHash) == 1) { - NeededChunks.emplace_back(ChunkHash); + if (!HasAttachment(ChunkHash)) + { + NeededChunks.emplace_back(ChunkHash); + } } } } @@ -2841,9 +2844,12 @@ ParseOplogContainer(const CbObject& ContainerObject, for (CbFieldView ChunkField : ChunksArray) { const IoHash ChunkHash = ChunkField.AsHash(); - if (OpsAttachments.contains(ChunkHash) && !HasAttachment(ChunkHash)) + if (OpsAttachments.erase(ChunkHash) == 1) { - NeededChunks.emplace_back(ChunkHash); + if (!HasAttachment(ChunkHash)) + { + NeededChunks.emplace_back(ChunkHash); + } } } } @@ -2871,11 +2877,16 @@ ParseOplogContainer(const CbObject& ContainerObject, for (CbFieldView LargeChunksField : LargeChunksArray) { IoHash AttachmentHash = LargeChunksField.AsBinaryAttachment(); - if (OpsAttachments.contains(AttachmentHash) && !HasAttachment(AttachmentHash)) + + if (OpsAttachments.erase(AttachmentHash) == 1) { - OnNeedAttachment(AttachmentHash); - NeedAttachmentCount++; + if (!HasAttachment(AttachmentHash)) + { + OnNeedAttachment(AttachmentHash); + NeedAttachmentCount++; + } } + if (remotestore_impl::IsCancelled(OptionalContext)) { return RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int>(HttpResponseCode::OK), |