diff options
| author | Dan Engelbrecht <[email protected]> | 2025-12-16 12:08:56 +0100 |
|---|---|---|
| committer | Dan Engelbrecht <[email protected]> | 2025-12-16 13:09:33 +0100 |
| commit | a81b605fce23b776700ec61cec431ab966843e8a (patch) | |
| tree | 5e010ee7e8b18d3952a9082eedd696e87ecfafe9 | |
| parent | optimized BlockStore::WriteChunks when only one chunk is written to a block (diff) | |
| download | zen-de/oplog-import-improvements.tar.xz zen-de/oplog-import-improvements.zip | |
ensure oplog chunk blocks do no exceed size limitde/oplog-import-improvements
| -rw-r--r-- | src/zenremotestore/projectstore/remoteprojectstore.cpp | 31 |
1 files changed, 18 insertions, 13 deletions
diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index efa1a872d..61a7cef17 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -2044,6 +2044,15 @@ BuildContainer(CidStore& ChunkStore, if (BlockAttachmentHashes.insert(AttachmentHash).second) { + if (BuildBlocks && ChunksInBlock.size() > 0) + { + if (((BlockSize + PayloadSize) > MaxBlockSize || (ChunksInBlock.size() + 1) > MaxChunksPerBlock) && + (CurrentOpKey != LastOpKey)) + { + NewBlock(); + } + } + if (auto It = LooseUploadAttachments.find(RawHash); It != LooseUploadAttachments.end()) { ChunksInBlock.emplace_back(std::make_pair( @@ -2081,11 +2090,6 @@ BuildContainer(CidStore& ChunkStore, })); } BlockSize += PayloadSize; - - if ((BlockSize >= MaxBlockSize || ChunksInBlock.size() > MaxChunksPerBlock) && (CurrentOpKey != LastOpKey)) - { - NewBlock(); - } LastOpKey = CurrentOpKey; ChunksAssembled++; } @@ -2129,6 +2133,14 @@ BuildContainer(CidStore& ChunkStore, if (BlockAttachmentHashes.insert(ChunkHash).second) { const ChunkSource& Source = Chunked.ChunkSources[ChunkIndex]; + uint32_t ChunkSize = gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + Source.Size); + if (BuildBlocks && ChunksInBlock.size() > 0) + { + if ((BlockSize + ChunkSize) > MaxBlockSize || (ChunksInBlock.size() + 1) > MaxChunksPerBlock) + { + NewBlock(); + } + } ChunksInBlock.emplace_back( std::make_pair(ChunkHash, [Source = ChunkedFile.Source, Offset = Source.Offset, Size = Source.Size]( @@ -2138,14 +2150,7 @@ BuildContainer(CidStore& ChunkStore, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; })); - BlockSize += CompressedBuffer::GetHeaderSizeForNoneEncoder() + Source.Size; - if (BuildBlocks) - { - if (BlockSize >= MaxBlockSize || ChunksInBlock.size() > MaxChunksPerBlock) - { - NewBlock(); - } - } + BlockSize += ChunkSize; ChunksAssembled++; } ChunkedHashes.erase(FindIt); |