diff options
| author | Dan Engelbrecht <[email protected]> | 2022-12-07 11:21:41 +0100 |
|---|---|---|
| committer | GitHub <[email protected]> | 2022-12-07 02:21:41 -0800 |
| commit | 100c8f966b1c5b2fb190748f0177600562d1c5fe (patch) | |
| tree | fc85e350dea47330149a1d42eb7a6c7ae0a06111 /zencore/compositebuffer.cpp | |
| parent | Cache request record/replay (#198) (diff) | |
| download | zen-100c8f966b1c5b2fb190748f0177600562d1c5fe.tar.xz zen-100c8f966b1c5b2fb190748f0177600562d1c5fe.zip | |
optimizations (#200)
* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog
Diffstat (limited to 'zencore/compositebuffer.cpp')
| -rw-r--r-- | zencore/compositebuffer.cpp | 32 |
1 files changed, 21 insertions, 11 deletions
diff --git a/zencore/compositebuffer.cpp b/zencore/compositebuffer.cpp index e4ca93cc2..735020451 100644 --- a/zencore/compositebuffer.cpp +++ b/zencore/compositebuffer.cpp @@ -145,6 +145,12 @@ CompositeBuffer::GetIterator(uint64_t Offset) const MemoryView CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const { + // We use a sub range IoBuffer when we want to copy data from a segment. + // This means we will only materialize that range of the segment when doing + // GetView() rather than the full segment. + // A hot path for this code is when we call CompressedBuffer::FromCompressed which + // is only interested in reading the header (first 64 bytes or so) and then throws + // away the materialized data. MutableMemoryView WriteView; size_t SegmentCount = m_Segments.size(); ZEN_ASSERT(It.SegmentIndex < SegmentCount); @@ -155,9 +161,8 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy size_t SegmentSize = Segment.GetSize(); if (Size == SizeLeft && Size <= (SegmentSize - It.OffsetInSegment)) { - MemoryView View = Segment.GetView(); - View.RightChopInline(It.OffsetInSegment); - View.LeftInline(SizeLeft); + IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, SizeLeft); + MemoryView View = SubSegment.GetView(); It.OffsetInSegment += SizeLeft; ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); if (It.OffsetInSegment == SegmentSize) @@ -176,10 +181,9 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy WriteView = CopyBuffer.GetMutableView(); } size_t CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft); - MemoryView ReadView = Segment.GetView(); - ReadView.RightChopInline(It.OffsetInSegment); - ReadView.LeftInline(CopySize); - WriteView = WriteView.CopyFrom(ReadView); + IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize); + MemoryView ReadView = SubSegment.GetView(); + WriteView = WriteView.CopyFrom(ReadView); It.OffsetInSegment += CopySize; ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); if (It.OffsetInSegment == SegmentSize) @@ -195,6 +199,13 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy void CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const { + // We use a sub range IoBuffer when we want to copy data from a segment. + // This means we will only materialize that range of the segment when doing + // GetView() rather than the full segment. + // A hot path for this code is when we call CompressedBuffer::FromCompressed which + // is only interested in reading the header (first 64 bytes or so) and then throws + // away the materialized data. + size_t SizeLeft = WriteView.GetSize(); size_t SegmentCount = m_Segments.size(); ZEN_ASSERT(It.SegmentIndex < SegmentCount); @@ -203,10 +214,9 @@ CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const const SharedBuffer& Segment = m_Segments[It.SegmentIndex]; size_t SegmentSize = Segment.GetSize(); size_t CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft); - MemoryView ReadView = Segment.GetView(); - ReadView.RightChopInline(It.OffsetInSegment); - ReadView.LeftInline(CopySize); - WriteView = WriteView.CopyFrom(ReadView); + IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize); + MemoryView ReadView = SubSegment.GetView(); + WriteView = WriteView.CopyFrom(ReadView); It.OffsetInSegment += CopySize; ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); if (It.OffsetInSegment == SegmentSize) |