hashing fixes (#657)

* move structuredcachestore tests to zenstore-test
* Don't materialize entire files when hashing if it is a large files
* rewrite CompositeBuffer::Mid to never materialize buffers

1 files changed, 29 insertions, 3 deletions

diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp
index 583ef19c6..d2b6d97f9 100644
--- a/src/zencore/compositebuffer.cpp
+++ b/src/zencore/compositebuffer.cpp
@@ -93,10 +93,36 @@ CompositeBuffer::Mid(uint64_t Offset, uint64_t Size) const
 	const uint64_t BufferSize = GetSize();
 	Offset					  = Min(Offset, BufferSize);
 	Size					  = Min(Size, BufferSize - Offset);
+
 	CompositeBuffer Buffer;
-	IterateRange(Offset, Size, [&Buffer](MemoryView View, const SharedBuffer& ViewOuter) {
-		Buffer.m_Segments.push_back(SharedBuffer::MakeView(View, ViewOuter));
-	});
+	{
+		for (const SharedBuffer& Segment : m_Segments)
+		{
+			if (const uint64_t SegmentSize = Segment.GetSize(); Offset <= SegmentSize)
+			{
+				size_t PartSize = Min(Size, SegmentSize - Offset);
+				if (PartSize == SegmentSize)
+				{
+					Buffer.m_Segments.push_back(Segment);
+				}
+				else if (PartSize > 0 || Size == 0)
+				{
+					// We need to add the segment even if PartSize is zero if we are picking up zero bytes.
+					Buffer.m_Segments.push_back(SharedBuffer(IoBuffer(Segment.AsIoBuffer(), Offset, PartSize)));
+				}
+				Offset = 0;
+				Size -= PartSize;
+				if (Size == 0)
+				{
+					break;
+				}
+			}
+			else
+			{
+				Offset -= SegmentSize;
+			}
+		}
+	}
 	return Buffer;
 }