optimizations (#200)

* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog

1 files changed, 19 insertions, 7 deletions

diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index 19675b9cf..a4fa38a1d 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -135,10 +135,12 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 		if (BinaryView.GetSize() > 0)
 		{
 			// Is a compressed binary blob
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
 			CompressedBuffer Compressed =
-				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned();
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned();
 			Value.emplace<CompressedBuffer>(Compressed);
-			Hash = IoHash::FromBLAKE3(Compressed.GetRawHash());
+			Hash = RawHash;
 			++Fields;
 		}
 		else
@@ -191,8 +193,10 @@ TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Fie
 		if (Buffer.GetSize() > 0)
 		{
 			// Is a compressed binary blob
-			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer));
-			TargetAttachment			= CbAttachment(Compressed, IoHash::FromBLAKE3(Compressed.GetRawHash()));
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
+			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize);
+			TargetAttachment			= CbAttachment(Compressed, RawHash);
 		}
 		else
 		{
@@ -715,9 +719,11 @@ namespace legacy {
 					{
 						return false;
 					}
-					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer))
+					IoHash	 RawHash;
+					uint64_t RawSize;
+					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize))
 					{
-						if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash)
+						if (RawHash != Hash)
 						{
 							return false;
 						}
@@ -747,8 +753,14 @@ namespace legacy {
 				ZEN_ASSERT(Mapper);
 				if (SharedBuffer AttachmentData = (*Mapper)(Hash))
 				{
-					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData))
+					IoHash	 RawHash;
+					uint64_t RawSize;
+					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize))
 					{
+						if (RawHash != Hash)
+						{
+							return false;
+						}
 						Package.AddAttachment(CbAttachment(Compressed, Hash));
 					}
 					else