optimizations (#200)

* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog

1 files changed, 17 insertions, 12 deletions

diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp
index e2f061a87..568585639 100644
--- a/zenhttp/httpshared.cpp
+++ b/zenhttp/httpshared.cpp
@@ -344,7 +344,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 			{
 				if (i == 0)
 				{
-					CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
+					CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
 					if (!CompBuf)
 					{
 						throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for CbObject", i));
@@ -359,18 +359,11 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 			}
 			else
 			{
-				// Make a copy of the buffer so we attachements don't reference the entire payload
-				IoBuffer AttachmentBufferCopy = CreateBuffer(Entry.AttachmentHash, AttachmentSize);
-				ZEN_ASSERT(AttachmentBufferCopy);
-				ZEN_ASSERT(AttachmentBufferCopy.Size() == AttachmentSize);
-				AttachmentBufferCopy.GetMutableView().CopyFrom(AttachmentBuffer.GetView());
-
-				CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBufferCopy)));
+				CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
 				if (!CompBuf)
 				{
 					throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for attachment", i));
 				}
-
 				Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash));
 			}
 		}
@@ -531,7 +524,13 @@ CbPackageReader::Finalize()
 				}
 				else if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
 				{
-					m_RootObject = LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
+					IoHash			 RawHash;
+					uint64_t		 RawSize;
+					CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize);
+					if (RawHash == Entry.AttachmentHash)
+					{
+						m_RootObject = LoadCompactBinaryObject(Compressed);
+					}
 				}
 				else
 				{
@@ -549,13 +548,19 @@ CbPackageReader::Finalize()
 
 			if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
 			{
-				m_Attachments.push_back(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference)), Entry.AttachmentHash));
+				IoHash			 RawHash;
+				uint64_t		 RawSize;
+				CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference), RawHash, RawSize);
+				if (RawHash == Entry.AttachmentHash)
+				{
+					m_Attachments.push_back(CbAttachment(Compressed, Entry.AttachmentHash));
+				}
 			}
 			else
 			{
 				CompressedBuffer Compressed =
 					CompressedBuffer::Compress(SharedBuffer(ChunkReference), OodleCompressor::NotSet, OodleCompressionLevel::None);
-				m_Attachments.push_back(CbAttachment(std::move(Compressed), IoHash::FromBLAKE3(Compressed.GetRawHash())));
+				m_Attachments.push_back(CbAttachment(std::move(Compressed), IoHash::FromBLAKE3(Compressed.DecodeRawHash())));
 			}
 		}