From 100c8f966b1c5b2fb190748f0177600562d1c5fe Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 7 Dec 2022 11:21:41 +0100 Subject: optimizations (#200) * Use direct file read and direct buffer allocation for small IoBuffer materalization * Reduce range of materialized data in CompositeBuffer reading CompressedBuffer header reading often only need a small part and not the whole file * reduce lock contention in IoBuffer::Materialize * Reduce parsing of compressed headers Validate header type at decompression * faster CreateDirectories - start from leaf going up and recurse back * optimized BufferHeader::IsValid * Add ValidateCompressedHeader to use when we don't need the actual compressed data Validate that we always get compressed data in CidStore::AddChunk * changelog --- zencore/compactbinarypackage.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'zencore/compactbinarypackage.cpp') diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp index 19675b9cf..a4fa38a1d 100644 --- a/zencore/compactbinarypackage.cpp +++ b/zencore/compactbinarypackage.cpp @@ -135,10 +135,12 @@ CbAttachment::TryLoad(CbFieldIterator& Fields) if (BinaryView.GetSize() > 0) { // Is a compressed binary blob + IoHash RawHash; + uint64_t RawSize; CompressedBuffer Compressed = - CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned(); + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned(); Value.emplace(Compressed); - Hash = IoHash::FromBLAKE3(Compressed.GetRawHash()); + Hash = RawHash; ++Fields; } else @@ -191,8 +193,10 @@ TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Fie if (Buffer.GetSize() > 0) { // Is a compressed binary blob - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer)); - TargetAttachment = CbAttachment(Compressed, IoHash::FromBLAKE3(Compressed.GetRawHash())); + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize); + TargetAttachment = CbAttachment(Compressed, RawHash); } else { @@ -715,9 +719,11 @@ namespace legacy { { return false; } - if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer)) + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize)) { - if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash) + if (RawHash != Hash) { return false; } @@ -747,8 +753,14 @@ namespace legacy { ZEN_ASSERT(Mapper); if (SharedBuffer AttachmentData = (*Mapper)(Hash)) { - if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData)) + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize)) { + if (RawHash != Hash) + { + return false; + } Package.AddAttachment(CbAttachment(Compressed, Hash)); } else -- cgit v1.2.3