fixed size chunking for encrypted files (#410)

- Improvement: Use fixed size block chunking for know encrypted/compressed file types
- Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed
- Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80%

1 files changed, 36 insertions, 14 deletions

diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 62b64bc9d..d9f381811 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -216,23 +216,45 @@ public:
 		std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
 		uint64_t /* BlockSize */) const final
 	{
-		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData));
-		Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize())));
+		const uint64_t HeaderSize = CompressedBuffer::GetHeaderSizeForNoneEncoder();
 
-		IoBufferFileReference FileRef = {nullptr, 0, 0};
-		if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
+		uint64_t	 RawOffset = 0;
+		BLAKE3Stream HashStream;
+
+		for (const SharedBuffer& Segment : RawData.GetSegments())
 		{
-			ZEN_ASSERT(FileRef.FileHandle != nullptr);
-			uint64_t CallbackOffset = 0;
-			ScanFile(FileRef.FileHandle, 0, RawData.GetSize(), 512u * 1024u, [&](const void* Data, size_t Size) {
-				CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size)));
-				Callback(CallbackOffset, Size, HeaderData.GetSize() + CallbackOffset, Tmp);
-				CallbackOffset += Size;
-			});
-			return true;
+			IoBufferFileReference FileRef		= {nullptr, 0, 0};
+			IoBuffer			  SegmentBuffer = Segment.AsIoBuffer();
+			if (SegmentBuffer.GetFileReference(FileRef))
+			{
+				ZEN_ASSERT(FileRef.FileHandle != nullptr);
+
+				ScanFile(FileRef.FileHandle,
+						 FileRef.FileChunkOffset,
+						 FileRef.FileChunkSize,
+						 512u * 1024u,
+						 [&](const void* Data, size_t Size) {
+							 HashStream.Append(Data, Size);
+							 CompositeBuffer Tmp(SharedBuffer::MakeView(Data, Size));
+							 Callback(RawOffset, Size, HeaderSize + RawOffset, Tmp);
+							 RawOffset += Size;
+						 });
+			}
+			else
+			{
+				const uint64_t Size = SegmentBuffer.GetSize();
+				HashStream.Append(SegmentBuffer);
+				Callback(RawOffset, Size, HeaderSize + RawOffset, CompositeBuffer(Segment));
+				RawOffset += Size;
+			}
 		}
 
-		Callback(0, RawData.GetSize(), HeaderData.GetSize(), RawData);
+		ZEN_ASSERT(RawOffset == RawData.GetSize());
+
+		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), HashStream.GetHash());
+		ZEN_ASSERT(HeaderData.GetSize() == HeaderSize);
+		Callback(0, 0, 0, CompositeBuffer(HeaderData.MoveToShared()));
+
 		return true;
 	}
 };
@@ -323,7 +345,7 @@ public:
 				ScanFile(FileRef.FileHandle, sizeof(BufferHeader) + RawOffset, RawSize, 512u * 1024u, [&](const void* Data, size_t Size) {
 					if (Result)
 					{
-						CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size)));
+						CompositeBuffer Tmp(SharedBuffer::MakeView(Data, Size));
 						Result = Callback(sizeof(BufferHeader) + RawOffset + CallbackOffset, Size, CallbackOffset, Tmp);
 					}
 					CallbackOffset += Size;