hashing fixes (#657)

* move structuredcachestore tests to zenstore-test
* Don't materialize entire files when hashing if it is a large files
* rewrite CompositeBuffer::Mid to never materialize buffers

1 files changed, 17 insertions, 1 deletions

diff --git a/src/zencore/iohash.cpp b/src/zencore/iohash.cpp
index 77076c133..cedee913a 100644
--- a/src/zencore/iohash.cpp
+++ b/src/zencore/iohash.cpp
@@ -31,7 +31,23 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer)
 
 	for (const SharedBuffer& Segment : Buffer.GetSegments())
 	{
-		Hasher.Append(Segment.GetData(), Segment.GetSize());
+		size_t SegmentSize = Segment.GetSize();
+		if (SegmentSize >= (65536 + 32768) && Segment.IsFileReference())
+		{
+			const IoBuffer SegmentBuffer = Segment.AsIoBuffer();
+			size_t		   Offset		 = 0;
+			while (Offset < SegmentSize)
+			{
+				size_t	 ChunkSize = Min<size_t>(SegmentSize - Offset, 65536u);
+				IoBuffer SubRange(SegmentBuffer, Offset, ChunkSize);
+				Hasher.Append(SubRange.GetData(), ChunkSize);
+				Offset += ChunkSize;
+			}
+		}
+		else
+		{
+			Hasher.Append(Segment.GetData(), SegmentSize);
+		}
 	}
 
 	return Hasher.GetHash();