aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore/blockstore.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-05-07 10:23:42 +0200
committerGitHub Enterprise <[email protected]>2025-05-07 10:23:42 +0200
commit68938614c95635045a394ff0a52786b82f01ffc4 (patch)
tree6193f85d4fe3ec154e78bb494b979a91733f9ae6 /src/zenstore/blockstore.cpp
parentadded logic to handle empty directories correctly (#383) (diff)
downloadzen-68938614c95635045a394ff0a52786b82f01ffc4.tar.xz
zen-68938614c95635045a394ff0a52786b82f01ffc4.zip
optimize block store CompactBlocks (#384)
- Improvement: Optimize block compact reducing memcpy operations - Improvement: Handle padding of block store blocks when compacting to avoid excessive flusing of write buffer - Improvement: Handle padding when writing oplog index snapshot to avoid unnecessary flushing of write buffer
Diffstat (limited to 'src/zenstore/blockstore.cpp')
-rw-r--r--src/zenstore/blockstore.cpp33
1 files changed, 19 insertions, 14 deletions
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index c58080e6a..e0f371061 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -1062,11 +1062,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState,
std::sort(SortedChunkIndexes.begin(), SortedChunkIndexes.end(), [&ChunkLocations](size_t Lhs, size_t Rhs) {
return ChunkLocations[Lhs].Offset < ChunkLocations[Rhs].Offset;
});
- BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(65536u, OldBlockSize));
+ BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(256u * 1024u, OldBlockSize));
- uint64_t WrittenBytesToBlock = 0;
- uint64_t MovedFromBlock = 0;
- std::vector<uint8_t> Chunk;
+ uint64_t MovedFromBlock = 0;
+ std::vector<uint8_t> ChunkBuffer;
for (const size_t& ChunkIndex : SortedChunkIndexes)
{
const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
@@ -1084,10 +1083,15 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState,
continue;
}
- Chunk.resize(ChunkLocation.Size);
- SourceFileBuffer.Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
+ MemoryView ChunkView = SourceFileBuffer.MakeView(ChunkLocation.Size, ChunkLocation.Offset);
+ if (ChunkView.GetSize() != ChunkLocation.Size)
+ {
+ ChunkBuffer.resize(ChunkLocation.Size);
+ SourceFileBuffer.Read(ChunkBuffer.data(), ChunkLocation.Size, ChunkLocation.Offset);
+ ChunkView = MemoryView(ChunkBuffer.data(), ChunkLocation.Size);
+ }
- if ((WriteOffset + Chunk.size()) > m_MaxBlockSize)
+ if ((WriteOffset + ChunkView.GetSize()) > m_MaxBlockSize)
{
TargetFileBuffer.reset();
if (NewBlockFile)
@@ -1174,20 +1178,21 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState,
NewBlockFile->Create(m_MaxBlockSize);
NewBlockIndex = NextBlockIndex;
WriteOffset = 0;
- AddedSize += WrittenBytesToBlock;
- WrittenBytesToBlock = 0;
- TargetFileBuffer = std::make_unique<BasicFileWriter>(NewBlockFile->GetBasicFile(), Min(65536u, m_MaxBlockSize));
+ AddedSize += WriteOffset;
+ WriteOffset = 0;
+ TargetFileBuffer = std::make_unique<BasicFileWriter>(NewBlockFile->GetBasicFile(), Min(256u * 1024u, m_MaxBlockSize));
}
- TargetFileBuffer->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
+ WriteOffset = TargetFileBuffer->AlignTo(PayloadAlignment);
+
+ TargetFileBuffer->Write(ChunkView.GetData(), ChunkLocation.Size, WriteOffset);
MovedChunks.push_back(
{ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
- WrittenBytesToBlock = WriteOffset + ChunkLocation.Size;
+ WriteOffset += ChunkLocation.Size;
MovedFromBlock += RoundUp(ChunkLocation.Offset + ChunkLocation.Size, PayloadAlignment) - ChunkLocation.Offset;
- WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
}
- AddedSize += WrittenBytesToBlock;
+ AddedSize += WriteOffset;
ZEN_INFO("{}moved {} chunks ({}) from '{}' to new block, freeing {}",
LogPrefix,
KeepChunkIndexes.size(),