optimizations (#200)

* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog

2 files changed, 19 insertions, 7 deletions

diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
index 92dc1fb76..5bf6d3001 100644
--- a/zencore/include/zencore/compress.h
+++ b/zencore/include/zencore/compress.h
@@ -71,12 +71,24 @@ public:
 	 *
 	 * @return A compressed buffer, or null on error, such as an invalid format or corrupt header.
 	 */
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData,
+																	 IoHash&				OutRawHash,
+																	 uint64_t&				OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData,
+																	 IoHash&		   OutRawHash,
+																	 uint64_t&		   OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData,
+																	 IoHash&			 OutRawHash,
+																	 uint64_t&			 OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData,
+																	 IoHash&		OutRawHash,
+																	 uint64_t&		OutRawSize);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
+	[[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const IoBuffer& CompressedData,
+																   IoHash&		   OutRawHash,
+																   uint64_t&	   OutRawSize);
 
 	/** Reset this to null. */
 	inline void Reset() { CompressedData.Reset(); }
@@ -102,10 +114,10 @@ public:
 	[[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); }
 
 	/** Returns the size of the raw data. Zero on error or if this is empty or null. */
-	[[nodiscard]] ZENCORE_API uint64_t GetRawSize() const;
+	[[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const;
 
 	/** Returns the hash of the raw data. Zero on error or if this is null. */
-	[[nodiscard]] ZENCORE_API BLAKE3 GetRawHash() const;
+	[[nodiscard]] ZENCORE_API BLAKE3 DecodeRawHash() const;
 
 	[[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const;
 
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 7f107cc0f..a39dbf6d6 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -254,7 +254,7 @@ protected:
 		kContentTypeBit3 = 1 << (24 + 3),  // bits are reserved
 	};
 
-	void AllocateBuffer(size_t InSize, size_t Alignment);
+	void AllocateBuffer(size_t InSize, size_t Alignment) const;
 	void FreeBuffer();
 };