diff options
| author | Dan Engelbrecht <[email protected]> | 2025-05-07 10:23:42 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-05-07 10:23:42 +0200 |
| commit | 68938614c95635045a394ff0a52786b82f01ffc4 (patch) | |
| tree | 6193f85d4fe3ec154e78bb494b979a91733f9ae6 /src/zenstore/blockstore.cpp | |
| parent | added logic to handle empty directories correctly (#383) (diff) | |
| download | zen-68938614c95635045a394ff0a52786b82f01ffc4.tar.xz zen-68938614c95635045a394ff0a52786b82f01ffc4.zip | |
optimize block store CompactBlocks (#384)
- Improvement: Optimize block compact reducing memcpy operations
- Improvement: Handle padding of block store blocks when compacting to avoid excessive flusing of write buffer
- Improvement: Handle padding when writing oplog index snapshot to avoid unnecessary flushing of write buffer
Diffstat (limited to 'src/zenstore/blockstore.cpp')
| -rw-r--r-- | src/zenstore/blockstore.cpp | 33 |
1 files changed, 19 insertions, 14 deletions
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index c58080e6a..e0f371061 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -1062,11 +1062,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, std::sort(SortedChunkIndexes.begin(), SortedChunkIndexes.end(), [&ChunkLocations](size_t Lhs, size_t Rhs) { return ChunkLocations[Lhs].Offset < ChunkLocations[Rhs].Offset; }); - BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(65536u, OldBlockSize)); + BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(256u * 1024u, OldBlockSize)); - uint64_t WrittenBytesToBlock = 0; - uint64_t MovedFromBlock = 0; - std::vector<uint8_t> Chunk; + uint64_t MovedFromBlock = 0; + std::vector<uint8_t> ChunkBuffer; for (const size_t& ChunkIndex : SortedChunkIndexes) { const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex]; @@ -1084,10 +1083,15 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, continue; } - Chunk.resize(ChunkLocation.Size); - SourceFileBuffer.Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset); + MemoryView ChunkView = SourceFileBuffer.MakeView(ChunkLocation.Size, ChunkLocation.Offset); + if (ChunkView.GetSize() != ChunkLocation.Size) + { + ChunkBuffer.resize(ChunkLocation.Size); + SourceFileBuffer.Read(ChunkBuffer.data(), ChunkLocation.Size, ChunkLocation.Offset); + ChunkView = MemoryView(ChunkBuffer.data(), ChunkLocation.Size); + } - if ((WriteOffset + Chunk.size()) > m_MaxBlockSize) + if ((WriteOffset + ChunkView.GetSize()) > m_MaxBlockSize) { TargetFileBuffer.reset(); if (NewBlockFile) @@ -1174,20 +1178,21 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, NewBlockFile->Create(m_MaxBlockSize); NewBlockIndex = NextBlockIndex; WriteOffset = 0; - AddedSize += WrittenBytesToBlock; - WrittenBytesToBlock = 0; - TargetFileBuffer = std::make_unique<BasicFileWriter>(NewBlockFile->GetBasicFile(), Min(65536u, m_MaxBlockSize)); + AddedSize += WriteOffset; + WriteOffset = 0; + TargetFileBuffer = std::make_unique<BasicFileWriter>(NewBlockFile->GetBasicFile(), Min(256u * 1024u, m_MaxBlockSize)); } - TargetFileBuffer->Write(Chunk.data(), ChunkLocation.Size, WriteOffset); + WriteOffset = TargetFileBuffer->AlignTo(PayloadAlignment); + + TargetFileBuffer->Write(ChunkView.GetData(), ChunkLocation.Size, WriteOffset); MovedChunks.push_back( {ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}}); - WrittenBytesToBlock = WriteOffset + ChunkLocation.Size; + WriteOffset += ChunkLocation.Size; MovedFromBlock += RoundUp(ChunkLocation.Offset + ChunkLocation.Size, PayloadAlignment) - ChunkLocation.Offset; - WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment); } - AddedSize += WrittenBytesToBlock; + AddedSize += WriteOffset; ZEN_INFO("{}moved {} chunks ({}) from '{}' to new block, freeing {}", LogPrefix, KeepChunkIndexes.size(), |