From a0b10b046095d57ffbdb46c83084601a832f4562 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Jun 2025 16:21:01 +0200 Subject: fixed size chunking for encrypted files (#410) - Improvement: Use fixed size block chunking for know encrypted/compressed file types - Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed - Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80% --- src/zencore/blake3.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'src/zencore/blake3.cpp') diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp index 4a77aa49a..054f0d3a0 100644 --- a/src/zencore/blake3.cpp +++ b/src/zencore/blake3.cpp @@ -151,6 +151,28 @@ BLAKE3Stream::Append(const void* data, size_t byteCount) return *this; } +BLAKE3Stream& +BLAKE3Stream::Append(const IoBuffer& Buffer) +{ + blake3_hasher* b3h = reinterpret_cast(m_HashState); + + size_t BufferSize = Buffer.GetSize(); + static const uint64_t BufferingSize = 256u * 1024u; + IoBufferFileReference FileRef; + if (BufferSize >= (BufferingSize + BufferingSize / 2) && Buffer.GetFileReference(FileRef)) + { + ScanFile(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, [&b3h](const void* Data, size_t Size) { + blake3_hasher_update(b3h, Data, Size); + }); + } + else + { + blake3_hasher_update(b3h, Buffer.GetData(), BufferSize); + } + + return *this; +} + BLAKE3 BLAKE3Stream::GetHash() { -- cgit v1.2.3