From 8672d2235e73545abde15f075934f68495adeaf3 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan@engelbrecht.com>
Date: Mon, 26 Feb 2024 19:08:27 +0100
Subject: hashing fixes (#657)

* move structuredcachestore tests to zenstore-test
* Don't materialize entire files when hashing if it is a large files
* rewrite CompositeBuffer::Mid to never materialize buffers
---
 src/zencore/iobuffer.cpp | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

(limited to 'src/zencore/iobuffer.cpp')

diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index 54f2aaee8..c8bc4a629 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -704,8 +704,25 @@ IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName)
 IoHash
 HashBuffer(IoBuffer& Buffer)
 {
-	// TODO: handle disk buffers with special path
-	return IoHash::HashBuffer(Buffer.Data(), Buffer.Size());
+	size_t BufferSize = Buffer.Size();
+	if (BufferSize >= (65536 + 32768))
+	{
+		IoBufferFileReference _;
+		if (Buffer.GetFileReference(/* out */ _))
+		{
+			size_t		 Offset = 0;
+			IoHashStream HashStream;
+			while (Offset < BufferSize)
+			{
+				size_t	 ChunkSize = Min<size_t>(BufferSize - Offset, 65536u);
+				IoBuffer SubRange(Buffer, Offset, ChunkSize);
+				HashStream.Append(SubRange.GetData(), SubRange.GetSize());
+				Offset += ChunkSize;
+			}
+			return HashStream.GetHash();
+		}
+	}
+	return IoHash::HashBuffer(Buffer.Data(), BufferSize);
 }
 
 //////////////////////////////////////////////////////////////////////////
-- 
cgit v1.2.3