From 100c8f966b1c5b2fb190748f0177600562d1c5fe Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 7 Dec 2022 11:21:41 +0100 Subject: optimizations (#200) * Use direct file read and direct buffer allocation for small IoBuffer materalization * Reduce range of materialized data in CompositeBuffer reading CompressedBuffer header reading often only need a small part and not the whole file * reduce lock contention in IoBuffer::Materialize * Reduce parsing of compressed headers Validate header type at decompression * faster CreateDirectories - start from leaf going up and recurse back * optimized BufferHeader::IsValid * Add ValidateCompressedHeader to use when we don't need the actual compressed data Validate that we always get compressed data in CidStore::AddChunk * changelog --- zencore/compositebuffer.cpp | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) (limited to 'zencore/compositebuffer.cpp') diff --git a/zencore/compositebuffer.cpp b/zencore/compositebuffer.cpp index e4ca93cc2..735020451 100644 --- a/zencore/compositebuffer.cpp +++ b/zencore/compositebuffer.cpp @@ -145,6 +145,12 @@ CompositeBuffer::GetIterator(uint64_t Offset) const MemoryView CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const { + // We use a sub range IoBuffer when we want to copy data from a segment. + // This means we will only materialize that range of the segment when doing + // GetView() rather than the full segment. + // A hot path for this code is when we call CompressedBuffer::FromCompressed which + // is only interested in reading the header (first 64 bytes or so) and then throws + // away the materialized data. MutableMemoryView WriteView; size_t SegmentCount = m_Segments.size(); ZEN_ASSERT(It.SegmentIndex < SegmentCount); @@ -155,9 +161,8 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy size_t SegmentSize = Segment.GetSize(); if (Size == SizeLeft && Size <= (SegmentSize - It.OffsetInSegment)) { - MemoryView View = Segment.GetView(); - View.RightChopInline(It.OffsetInSegment); - View.LeftInline(SizeLeft); + IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, SizeLeft); + MemoryView View = SubSegment.GetView(); It.OffsetInSegment += SizeLeft; ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); if (It.OffsetInSegment == SegmentSize) @@ -176,10 +181,9 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy WriteView = CopyBuffer.GetMutableView(); } size_t CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft); - MemoryView ReadView = Segment.GetView(); - ReadView.RightChopInline(It.OffsetInSegment); - ReadView.LeftInline(CopySize); - WriteView = WriteView.CopyFrom(ReadView); + IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize); + MemoryView ReadView = SubSegment.GetView(); + WriteView = WriteView.CopyFrom(ReadView); It.OffsetInSegment += CopySize; ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); if (It.OffsetInSegment == SegmentSize) @@ -195,6 +199,13 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy void CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const { + // We use a sub range IoBuffer when we want to copy data from a segment. + // This means we will only materialize that range of the segment when doing + // GetView() rather than the full segment. + // A hot path for this code is when we call CompressedBuffer::FromCompressed which + // is only interested in reading the header (first 64 bytes or so) and then throws + // away the materialized data. + size_t SizeLeft = WriteView.GetSize(); size_t SegmentCount = m_Segments.size(); ZEN_ASSERT(It.SegmentIndex < SegmentCount); @@ -203,10 +214,9 @@ CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const const SharedBuffer& Segment = m_Segments[It.SegmentIndex]; size_t SegmentSize = Segment.GetSize(); size_t CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft); - MemoryView ReadView = Segment.GetView(); - ReadView.RightChopInline(It.OffsetInSegment); - ReadView.LeftInline(CopySize); - WriteView = WriteView.CopyFrom(ReadView); + IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize); + MemoryView ReadView = SubSegment.GetView(); + WriteView = WriteView.CopyFrom(ReadView); It.OffsetInSegment += CopySize; ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); if (It.OffsetInSegment == SegmentSize) -- cgit v1.2.3