special treatment large oplog attachments v2 (#5)

- Bugfix: Install Ctrl+C handler earlier when doing `zen oplog-export` and `zen oplog-export` to properly cancel jobs
- Improvement: Add ability to block a set of CAS entries from GC in project store
- Improvement: Large attachments and loose files are now split into smaller chunks and stored in blocks during oplog export

5 files changed, 39 insertions, 22 deletions

diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp
index bdbc8fb3e..e4edff227 100644
--- a/src/zencore/blake3.cpp
+++ b/src/zencore/blake3.cpp
@@ -45,14 +45,15 @@ BLAKE3::HashBuffer(const CompositeBuffer& Buffer)
 
 	for (const SharedBuffer& Segment : Buffer.GetSegments())
 	{
-		size_t SegmentSize = Segment.GetSize();
-		if (SegmentSize >= (65536 + 32768) && Segment.IsFileReference())
+		size_t				SegmentSize	  = Segment.GetSize();
+		static const size_t BufferingSize = 512 * 1024;
+		if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.IsFileReference())
 		{
 			const IoBuffer SegmentBuffer = Segment.AsIoBuffer();
 			size_t		   Offset		 = 0;
 			while (Offset < SegmentSize)
 			{
-				size_t	 ChunkSize = Min<size_t>(SegmentSize - Offset, 65536u);
+				size_t	 ChunkSize = Min<size_t>(SegmentSize - Offset, BufferingSize);
 				IoBuffer SubRange(SegmentBuffer, Offset, ChunkSize);
 				blake3_hasher_update(&Hasher, SubRange.GetData(), ChunkSize);
 				Offset += ChunkSize;
diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index c41bdac42..a8e8a79f4 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -193,15 +193,7 @@ class NoneEncoder final : public BaseEncoder
 public:
 	[[nodiscard]] CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final
 	{
-		BufferHeader Header;
-		Header.Method			   = CompressionMethod::None;
-		Header.BlockCount		   = 1;
-		Header.TotalRawSize		   = RawData.GetSize();
-		Header.TotalCompressedSize = Header.TotalRawSize + sizeof(BufferHeader);
-		Header.RawHash			   = BLAKE3::HashBuffer(RawData);
-
-		UniqueBuffer HeaderData = UniqueBuffer::Alloc(sizeof(BufferHeader));
-		Header.Write(HeaderData);
+		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData));
 		return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned());
 	}
 };
@@ -1301,6 +1293,26 @@ CompressedBuffer::ValidateCompressedHeader(const IoBuffer& CompressedData, IoHas
 	return detail::BufferHeader::IsValid(SharedBuffer(CompressedData), OutRawHash, OutRawSize);
 }
 
+size_t
+CompressedBuffer::GetHeaderSizeForNoneEncoder()
+{
+	return sizeof(detail::BufferHeader);
+}
+
+UniqueBuffer
+CompressedBuffer::CreateHeaderForNoneEncoder(uint64_t RawSize, const BLAKE3& RawHash)
+{
+	detail::BufferHeader Header;
+	Header.Method			   = detail::CompressionMethod::None;
+	Header.BlockCount		   = 1;
+	Header.TotalRawSize		   = RawSize;
+	Header.TotalCompressedSize = Header.TotalRawSize + sizeof(detail::BufferHeader);
+	Header.RawHash			   = RawHash;
+	UniqueBuffer HeaderData	   = UniqueBuffer::Alloc(sizeof(detail::BufferHeader));
+	Header.Write(HeaderData);
+	return HeaderData;
+}
+
 uint64_t
 CompressedBuffer::DecodeRawSize() const
 {
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
index 44431f299..c51b5407f 100644
--- a/src/zencore/include/zencore/compress.h
+++ b/src/zencore/include/zencore/compress.h
@@ -94,10 +94,12 @@ public:
 																	 uint64_t&		OutRawSize);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
-	[[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
-	[[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const IoBuffer& CompressedData,
-																   IoHash&		   OutRawHash,
-																   uint64_t&	   OutRawSize);
+	[[nodiscard]] ZENCORE_API static bool	ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
+	[[nodiscard]] ZENCORE_API static bool	ValidateCompressedHeader(const IoBuffer& CompressedData,
+																	 IoHash&		 OutRawHash,
+																	 uint64_t&		 OutRawSize);
+	[[nodiscard]] ZENCORE_API static size_t GetHeaderSizeForNoneEncoder();
+	[[nodiscard]] ZENCORE_API static UniqueBuffer CreateHeaderForNoneEncoder(uint64_t RawSize, const BLAKE3& RawHash);
 
 	/** Reset this to null. */
 	inline void Reset() { CompressedData.Reset(); }
diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index c8bc4a629..96a893082 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -704,8 +704,9 @@ IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName)
 IoHash
 HashBuffer(IoBuffer& Buffer)
 {
-	size_t BufferSize = Buffer.Size();
-	if (BufferSize >= (65536 + 32768))
+	size_t				BufferSize	  = Buffer.Size();
+	static const size_t BufferingSize = 512 * 1024;
+	if (BufferSize >= (BufferingSize + BufferingSize / 2))
 	{
 		IoBufferFileReference _;
 		if (Buffer.GetFileReference(/* out */ _))
@@ -714,7 +715,7 @@ HashBuffer(IoBuffer& Buffer)
 			IoHashStream HashStream;
 			while (Offset < BufferSize)
 			{
-				size_t	 ChunkSize = Min<size_t>(BufferSize - Offset, 65536u);
+				size_t	 ChunkSize = Min<size_t>(BufferSize - Offset, BufferingSize);
 				IoBuffer SubRange(Buffer, Offset, ChunkSize);
 				HashStream.Append(SubRange.GetData(), SubRange.GetSize());
 				Offset += ChunkSize;
diff --git a/src/zencore/iohash.cpp b/src/zencore/iohash.cpp
index cedee913a..a6bf25f6c 100644
--- a/src/zencore/iohash.cpp
+++ b/src/zencore/iohash.cpp
@@ -31,14 +31,15 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer)
 
 	for (const SharedBuffer& Segment : Buffer.GetSegments())
 	{
-		size_t SegmentSize = Segment.GetSize();
-		if (SegmentSize >= (65536 + 32768) && Segment.IsFileReference())
+		size_t				SegmentSize	  = Segment.GetSize();
+		static const size_t BufferingSize = 512 * 1024;
+		if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.IsFileReference())
 		{
 			const IoBuffer SegmentBuffer = Segment.AsIoBuffer();
 			size_t		   Offset		 = 0;
 			while (Offset < SegmentSize)
 			{
-				size_t	 ChunkSize = Min<size_t>(SegmentSize - Offset, 65536u);
+				size_t	 ChunkSize = Min<size_t>(SegmentSize - Offset, BufferingSize);
 				IoBuffer SubRange(SegmentBuffer, Offset, ChunkSize);
 				Hasher.Append(SubRange.GetData(), ChunkSize);
 				Offset += ChunkSize;