fixed size chunking for encrypted files (#410)

- Improvement: Use fixed size block chunking for know encrypted/compressed file types
- Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed
- Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80%

1 files changed, 22 insertions, 0 deletions

diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp
index 4a77aa49a..054f0d3a0 100644
--- a/src/zencore/blake3.cpp
+++ b/src/zencore/blake3.cpp
@@ -151,6 +151,28 @@ BLAKE3Stream::Append(const void* data, size_t byteCount)
 	return *this;
 }
 
+BLAKE3Stream&
+BLAKE3Stream::Append(const IoBuffer& Buffer)
+{
+	blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState);
+
+	size_t				  BufferSize	= Buffer.GetSize();
+	static const uint64_t BufferingSize = 256u * 1024u;
+	IoBufferFileReference FileRef;
+	if (BufferSize >= (BufferingSize + BufferingSize / 2) && Buffer.GetFileReference(FileRef))
+	{
+		ScanFile(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, [&b3h](const void* Data, size_t Size) {
+			blake3_hasher_update(b3h, Data, Size);
+		});
+	}
+	else
+	{
+		blake3_hasher_update(b3h, Buffer.GetData(), BufferSize);
+	}
+
+	return *this;
+}
+
 BLAKE3
 BLAKE3Stream::GetHash()
 {