hashing fixes (#657)

* move structuredcachestore tests to zenstore-test
* Don't materialize entire files when hashing if it is a large files
* rewrite CompositeBuffer::Mid to never materialize buffers

1 files changed, 17 insertions, 1 deletions

diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp
index 89826ae5d..bdbc8fb3e 100644
--- a/src/zencore/blake3.cpp
+++ b/src/zencore/blake3.cpp
@@ -45,7 +45,23 @@ BLAKE3::HashBuffer(const CompositeBuffer& Buffer)
 
 	for (const SharedBuffer& Segment : Buffer.GetSegments())
 	{
-		blake3_hasher_update(&Hasher, Segment.GetData(), Segment.GetSize());
+		size_t SegmentSize = Segment.GetSize();
+		if (SegmentSize >= (65536 + 32768) && Segment.IsFileReference())
+		{
+			const IoBuffer SegmentBuffer = Segment.AsIoBuffer();
+			size_t		   Offset		 = 0;
+			while (Offset < SegmentSize)
+			{
+				size_t	 ChunkSize = Min<size_t>(SegmentSize - Offset, 65536u);
+				IoBuffer SubRange(SegmentBuffer, Offset, ChunkSize);
+				blake3_hasher_update(&Hasher, SubRange.GetData(), ChunkSize);
+				Offset += ChunkSize;
+			}
+		}
+		else
+		{
+			blake3_hasher_update(&Hasher, Segment.GetData(), SegmentSize);
+		}
 	}
 
 	blake3_hasher_finalize(&Hasher, Hash.Hash, sizeof Hash.Hash);