Faster reading of compressed buffer headers by not materializing entire source buffer (#28)

1 files changed, 52 insertions, 22 deletions

diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp
index d2b6d97f9..49870a304 100644
--- a/src/zencore/compositebuffer.cpp
+++ b/src/zencore/compositebuffer.cpp
@@ -133,24 +133,28 @@ CompositeBuffer::ViewOrCopyRange(uint64_t									Offset,
 								 std::function<UniqueBuffer(uint64_t Size)> Allocator) const
 {
 	MemoryView View;
-	IterateRange(Offset, Size, [Size, &View, &CopyBuffer, &Allocator, WriteView = MutableMemoryView()](MemoryView Segment) mutable {
-		if (Size == Segment.GetSize())
-		{
-			View = Segment;
-		}
-		else
-		{
-			if (WriteView.IsEmpty())
+	IterateRange(
+		Offset,
+		Size,
+		[Size, &View, &CopyBuffer, &Allocator, WriteView = MutableMemoryView()](MemoryView Segment, const SharedBuffer& ViewOuter) mutable {
+			if (Segment.GetSize() == ViewOuter.GetSize())
+			{
+				// We assume that the segment of the buffer is kept in memory
+				View = Segment;
+			}
+			else
 			{
-				if (CopyBuffer.GetSize() < Size)
+				if (WriteView.IsEmpty())
 				{
-					CopyBuffer = Allocator(Size);
+					if (CopyBuffer.GetSize() < Size)
+					{
+						CopyBuffer = Allocator(Size);
+					}
+					View = WriteView = CopyBuffer.GetMutableView().Left(Size);
 				}
-				View = WriteView = CopyBuffer.GetMutableView().Left(Size);
+				WriteView = WriteView.CopyFrom(Segment);
 			}
-			WriteView = WriteView.CopyFrom(Segment);
-		}
-	});
+		});
 	return View;
 }
 
@@ -263,18 +267,44 @@ CompositeBuffer::IterateRange(uint64_t															  Offset,
 	ZEN_ASSERT(Offset + Size <= GetSize());
 	for (const SharedBuffer& Segment : m_Segments)
 	{
-		if (const uint64_t SegmentSize = Segment.GetSize(); Offset <= SegmentSize)
+		const uint64_t SegmentSize = Segment.GetSize();
+		if (Size == 0 && Offset == SegmentSize)
+		{
+			// Special case for getting the zero size end of a composite buffer
+			const MemoryView View = Segment.GetView().Mid(Offset, 0);
+			Visitor(View, Segment);
+			break;
+		}
+		if (Offset < SegmentSize)
 		{
-			const MemoryView View = Segment.GetView().Mid(Offset, Size);
-			Offset				  = 0;
-			if (Size == 0 || !View.IsEmpty())
+			if (Offset == 0 && Size >= SegmentSize)
 			{
-				Visitor(View, Segment);
+				const MemoryView View = Segment.GetView();
+				if (!View.IsEmpty())
+				{
+					Visitor(View, Segment);
+				}
+				Size -= View.GetSize();
+				if (Size == 0)
+				{
+					break;
+				}
 			}
-			Size -= View.GetSize();
-			if (Size == 0)
+			else
 			{
-				break;
+				// If we only want a section of the segment, do a subrange so we don't have to materialize the entire iobuffer
+				IoBuffer		 SubRange(Segment.AsIoBuffer(), Offset, Min(Size, SegmentSize - Offset));
+				const MemoryView View = SubRange.GetView();
+				if (!View.IsEmpty())
+				{
+					Visitor(View, Segment);
+				}
+				Size -= View.GetSize();
+				if (Size == 0)
+				{
+					break;
+				}
+				Offset = 0;
 			}
 		}
 		else