optimizations (#200)

* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog

8 files changed, 234 insertions, 93 deletions

diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index 19675b9cf..a4fa38a1d 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -135,10 +135,12 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 		if (BinaryView.GetSize() > 0)
 		{
 			// Is a compressed binary blob
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
 			CompressedBuffer Compressed =
-				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned();
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned();
 			Value.emplace<CompressedBuffer>(Compressed);
-			Hash = IoHash::FromBLAKE3(Compressed.GetRawHash());
+			Hash = RawHash;
 			++Fields;
 		}
 		else
@@ -191,8 +193,10 @@ TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Fie
 		if (Buffer.GetSize() > 0)
 		{
 			// Is a compressed binary blob
-			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer));
-			TargetAttachment			= CbAttachment(Compressed, IoHash::FromBLAKE3(Compressed.GetRawHash()));
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
+			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize);
+			TargetAttachment			= CbAttachment(Compressed, RawHash);
 		}
 		else
 		{
@@ -715,9 +719,11 @@ namespace legacy {
 					{
 						return false;
 					}
-					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer))
+					IoHash	 RawHash;
+					uint64_t RawSize;
+					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize))
 					{
-						if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash)
+						if (RawHash != Hash)
 						{
 							return false;
 						}
@@ -747,8 +753,14 @@ namespace legacy {
 				ZEN_ASSERT(Mapper);
 				if (SharedBuffer AttachmentData = (*Mapper)(Hash))
 				{
-					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData))
+					IoHash	 RawHash;
+					uint64_t RawSize;
+					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize))
 					{
+						if (RawHash != Hash)
+						{
+							return false;
+						}
 						Package.AddAttachment(CbAttachment(Compressed, Hash));
 					}
 					else
diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp
index a787e88ab..02148d96a 100644
--- a/zencore/compactbinaryvalidation.cpp
+++ b/zencore/compactbinaryvalidation.cpp
@@ -463,17 +463,18 @@ ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode
 	{
 		if (BinaryView.GetSize() > 0)
 		{
-			CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView));
+			IoHash			 DecodedHash;
+			uint64_t		 DecodedRawSize;
+			CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView), DecodedHash, DecodedRawSize);
 			if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && Buffer.IsNull())
 			{
 				AddError(Error, CbValidateError::NullPackageAttachment);
 			}
-			if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) &&
-				(IoHash::FromBLAKE3(Buffer.GetRawHash()) != IoHash::HashBuffer(Buffer.DecompressToComposite())))
+			if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (DecodedHash != IoHash::HashBuffer(Buffer.DecompressToComposite())))
 			{
 				AddError(Error, CbValidateError::InvalidPackageHash);
 			}
-			return IoHash::FromBLAKE3(Buffer.GetRawHash());
+			return DecodedHash;
 		}
 		else
 		{
diff --git a/zencore/compositebuffer.cpp b/zencore/compositebuffer.cpp
index e4ca93cc2..735020451 100644
--- a/zencore/compositebuffer.cpp
+++ b/zencore/compositebuffer.cpp
@@ -145,6 +145,12 @@ CompositeBuffer::GetIterator(uint64_t Offset) const
 MemoryView
 CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const
 {
+	// We use a sub range IoBuffer when we want to copy data from a segment.
+	// This means we will only materialize that range of the segment when doing
+	// GetView() rather than the full segment.
+	// A hot path for this code is when we call CompressedBuffer::FromCompressed which
+	// is only interested in reading the header (first 64 bytes or so) and then throws
+	// away the materialized data.
 	MutableMemoryView WriteView;
 	size_t			  SegmentCount = m_Segments.size();
 	ZEN_ASSERT(It.SegmentIndex < SegmentCount);
@@ -155,9 +161,8 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy
 		size_t				SegmentSize = Segment.GetSize();
 		if (Size == SizeLeft && Size <= (SegmentSize - It.OffsetInSegment))
 		{
-			MemoryView View = Segment.GetView();
-			View.RightChopInline(It.OffsetInSegment);
-			View.LeftInline(SizeLeft);
+			IoBuffer   SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, SizeLeft);
+			MemoryView View = SubSegment.GetView();
 			It.OffsetInSegment += SizeLeft;
 			ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize);
 			if (It.OffsetInSegment == SegmentSize)
@@ -176,10 +181,9 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy
 			WriteView = CopyBuffer.GetMutableView();
 		}
 		size_t	   CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft);
-		MemoryView ReadView = Segment.GetView();
-		ReadView.RightChopInline(It.OffsetInSegment);
-		ReadView.LeftInline(CopySize);
-		WriteView = WriteView.CopyFrom(ReadView);
+		IoBuffer   SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize);
+		MemoryView ReadView = SubSegment.GetView();
+		WriteView			= WriteView.CopyFrom(ReadView);
 		It.OffsetInSegment += CopySize;
 		ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize);
 		if (It.OffsetInSegment == SegmentSize)
@@ -195,6 +199,13 @@ CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& Copy
 void
 CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const
 {
+	// We use a sub range IoBuffer when we want to copy data from a segment.
+	// This means we will only materialize that range of the segment when doing
+	// GetView() rather than the full segment.
+	// A hot path for this code is when we call CompressedBuffer::FromCompressed which
+	// is only interested in reading the header (first 64 bytes or so) and then throws
+	// away the materialized data.
+
 	size_t SizeLeft		= WriteView.GetSize();
 	size_t SegmentCount = m_Segments.size();
 	ZEN_ASSERT(It.SegmentIndex < SegmentCount);
@@ -203,10 +214,9 @@ CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const
 		const SharedBuffer& Segment		= m_Segments[It.SegmentIndex];
 		size_t				SegmentSize = Segment.GetSize();
 		size_t				CopySize	= zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft);
-		MemoryView			ReadView	= Segment.GetView();
-		ReadView.RightChopInline(It.OffsetInSegment);
-		ReadView.LeftInline(CopySize);
-		WriteView = WriteView.CopyFrom(ReadView);
+		IoBuffer			SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize);
+		MemoryView			ReadView = SubSegment.GetView();
+		WriteView					 = WriteView.CopyFrom(ReadView);
 		It.OffsetInSegment += CopySize;
 		ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize);
 		if (It.OffsetInSegment == SegmentSize)
diff --git a/zencore/compress.cpp b/zencore/compress.cpp
index 15cc5f6a7..6e06739ea 100644
--- a/zencore/compress.cpp
+++ b/zencore/compress.cpp
@@ -6,6 +6,7 @@
 #include <zencore/compositebuffer.h>
 #include <zencore/crc32.h>
 #include <zencore/endian.h>
+#include <zencore/iohash.h>
 #include <zencore/testing.h>
 
 #include "../thirdparty/Oodle/include/oodle2.h"
@@ -56,8 +57,11 @@ struct BufferHeader
 	BLAKE3	 RawHash;				   // The hash of the uncompressed data
 
 	/** Checks validity of the buffer based on the magic number, method, and CRC-32. */
-	static bool IsValid(const CompositeBuffer& CompressedData);
-	static bool IsValid(const SharedBuffer& CompressedData) { return IsValid(CompositeBuffer(CompressedData)); }
+	static bool IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
+	static bool IsValid(const SharedBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+	{
+		return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize);
+	}
 
 	/** Read a header from a buffer that is at least sizeof(BufferHeader) without any validation. */
 	static BufferHeader Read(const CompositeBuffer& CompressedData)
@@ -65,6 +69,10 @@ struct BufferHeader
 		BufferHeader Header;
 		if (sizeof(BufferHeader) <= CompressedData.GetSize())
 		{
+			//			if (CompressedData.GetSegments()[0].AsIoBuffer().IsWholeFile())
+			//			{
+			//				ZEN_ASSERT(true);
+			//			}
 			CompositeBuffer::Iterator It;
 			CompressedData.CopyTo(MakeMutableMemoryView(&Header, &Header + 1), It);
 			Header.ByteSwap();
@@ -664,36 +672,68 @@ GetDecoder(CompressionMethod Method)
 //////////////////////////////////////////////////////////////////////////
 
 bool
-BufferHeader::IsValid(const CompositeBuffer& CompressedData)
+BufferHeader::IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
-	if (sizeof(BufferHeader) <= CompressedData.GetSize())
+	uint64_t Size = CompressedData.GetSize();
+	if (Size < sizeof(BufferHeader))
 	{
-		const BufferHeader Header = Read(CompressedData);
-		if (Header.Magic == BufferHeader::ExpectedMagic)
+		return false;
+	}
+	const size_t  StackBufferSize = 256;
+	uint8_t		  StackBuffer[StackBufferSize];
+	uint64_t	  ReadSize = Min(Size, StackBufferSize);
+	BufferHeader* Header   = reinterpret_cast<BufferHeader*>(StackBuffer);
+	{
+		CompositeBuffer::Iterator It;
+		CompressedData.CopyTo(MutableMemoryView(StackBuffer, StackBuffer + StackBufferSize), It);
+	}
+	Header->ByteSwap();
+	if (Header->Magic != BufferHeader::ExpectedMagic)
+	{
+		return false;
+	}
+	const BaseDecoder* const Decoder = GetDecoder(Header->Method);
+	if (!Decoder)
+	{
+		return false;
+	}
+	uint32_t Crc32		= Header->Crc32;
+	OutRawHash			= IoHash::FromBLAKE3(Header->RawHash);
+	OutRawSize			= Header->TotalRawSize;
+	uint64_t HeaderSize = Decoder->GetHeaderSize(*Header);
+	Header->ByteSwap();
+
+	if (HeaderSize > ReadSize)
+	{
+		// 0.004% of cases on a Fortnite hot cache cook
+		UniqueBuffer			  HeaderCopy = UniqueBuffer::Alloc(HeaderSize);
+		CompositeBuffer::Iterator It;
+		CompressedData.CopyTo(HeaderCopy.GetMutableView(), It);
+		const MemoryView HeaderView = HeaderCopy.GetView();
+		if (Crc32 != BufferHeader::CalculateCrc32(HeaderView))
 		{
-			if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
-			{
-				UniqueBuffer			  HeaderCopy = UniqueBuffer::Alloc(Decoder->GetHeaderSize(Header));
-				CompositeBuffer::Iterator It;
-				CompressedData.CopyTo(HeaderCopy.GetMutableView(), It);
-				const MemoryView HeaderView = HeaderCopy.GetView();
-				if (Header.Crc32 == BufferHeader::CalculateCrc32(HeaderView))
-				{
-					return true;
-				}
-			}
+			return false;
 		}
 	}
-	return false;
+	else
+	{
+		MemoryView FullHeaderView(StackBuffer, StackBuffer + HeaderSize);
+		if (Crc32 != BufferHeader::CalculateCrc32(FullHeaderView))
+		{
+			return false;
+		}
+	}
+	return true;
 }
 
 //////////////////////////////////////////////////////////////////////////
 
 template<typename BufferType>
 inline CompositeBuffer
-ValidBufferOrEmpty(BufferType&& CompressedData)
+ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
-	return BufferHeader::IsValid(CompressedData) ? CompositeBuffer(std::forward<BufferType>(CompressedData)) : CompositeBuffer();
+	return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize) ? CompositeBuffer(std::forward<BufferType>(CompressedData))
+																		 : CompositeBuffer();
 }
 
 CompositeBuffer
@@ -826,34 +866,34 @@ CompressedBuffer::Compress(const SharedBuffer&	 RawData,
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData)
+CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
 	return Local;
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData)
+CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData));
+	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
 	return Local;
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData)
+CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
 	return Local;
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData)
+CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData));
+	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
 	return Local;
 }
 
@@ -881,14 +921,26 @@ CompressedBuffer::FromCompressedNoValidate(CompositeBuffer&& InCompressedData)
 	return Local;
 }
 
+bool
+CompressedBuffer::ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+{
+	return detail::BufferHeader::IsValid(SharedBuffer(std::move(CompressedData)), OutRawHash, OutRawSize);
+}
+
+bool
+CompressedBuffer::ValidateCompressedHeader(const IoBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+{
+	return detail::BufferHeader::IsValid(SharedBuffer(CompressedData), OutRawHash, OutRawSize);
+}
+
 uint64_t
-CompressedBuffer::GetRawSize() const
+CompressedBuffer::DecodeRawSize() const
 {
 	return CompressedData ? detail::BufferHeader::Read(CompressedData).TotalRawSize : 0;
 }
 
 BLAKE3
-CompressedBuffer::GetRawHash() const
+CompressedBuffer::DecodeRawHash() const
 {
 	return CompressedData ? detail::BufferHeader::Read(CompressedData).RawHash : BLAKE3();
 }
@@ -913,9 +965,12 @@ CompressedBuffer::TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset)
 	if (CompressedData)
 	{
 		const BufferHeader Header = BufferHeader::Read(CompressedData);
-		if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
+		if (Header.Magic == BufferHeader::ExpectedMagic)
 		{
-			return Decoder->TryDecompressTo(Header, CompressedData, RawView, RawOffset);
+			if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
+			{
+				return Decoder->TryDecompressTo(Header, CompressedData, RawView, RawOffset);
+			}
 		}
 	}
 	return false;
@@ -928,13 +983,16 @@ CompressedBuffer::Decompress(uint64_t RawOffset, uint64_t RawSize) const
 	if (CompressedData && RawSize > 0)
 	{
 		const BufferHeader Header = BufferHeader::Read(CompressedData);
-		if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
+		if (Header.Magic == BufferHeader::ExpectedMagic)
 		{
-			const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset;
-			UniqueBuffer   RawData		= UniqueBuffer::Alloc(TotalRawSize);
-			if (Decoder->TryDecompressTo(Header, CompressedData, RawData, RawOffset))
+			if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
 			{
-				return RawData.MoveToShared();
+				const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset;
+				UniqueBuffer   RawData		= UniqueBuffer::Alloc(TotalRawSize);
+				if (Decoder->TryDecompressTo(Header, CompressedData, RawData, RawOffset))
+				{
+					return RawData.MoveToShared();
+				}
 			}
 		}
 	}
@@ -948,9 +1006,12 @@ CompressedBuffer::DecompressToComposite() const
 	if (CompressedData)
 	{
 		const BufferHeader Header = BufferHeader::Read(CompressedData);
-		if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
+		if (Header.Magic == BufferHeader::ExpectedMagic)
 		{
-			return Decoder->Decompress(Header, CompressedData);
+			if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
+			{
+				return Decoder->Decompress(Header, CompressedData);
+			}
 		}
 	}
 	return CompositeBuffer();
@@ -1005,18 +1066,20 @@ TEST_CASE("CompressedBuffer")
 															 OodleCompressor::NotSet,
 															 OodleCompressionLevel::None);
 
-		CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
+		CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes));
 		CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(detail::BufferHeader)));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	{
@@ -1025,53 +1088,59 @@ TEST_CASE("CompressedBuffer")
 			OodleCompressor::NotSet,
 			OodleCompressionLevel::None);
 
-		CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
+		CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
 		CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(Ones) + sizeof(detail::BufferHeader)));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	{
 		CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))));
 
-		CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
+		CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes));
 		CHECK(Buffer.GetCompressedSize() < sizeof(Zeroes));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	{
 		CompressedBuffer Buffer = CompressedBuffer::Compress(
 			CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))));
 
-		CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
+		CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
 		CHECK(Buffer.GetCompressedSize() < (sizeof(Zeroes) + sizeof(Ones)));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	auto GenerateData = [](uint64_t N) -> std::vector<uint64_t> {
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 0aa478404..1e4a52638 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -188,13 +188,21 @@ CleanDirectory(const wchar_t* DirPath)
 bool
 CreateDirectories(const std::filesystem::path& Dir)
 {
-	std::error_code ErrorCode;
-	bool			WasCreated = std::filesystem::create_directories(Dir, ErrorCode);
-	if (ErrorCode)
+	while (!std::filesystem::is_directory(Dir))
 	{
-		throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string()));
+		if (Dir.has_parent_path())
+		{
+			CreateDirectories(Dir.parent_path());
+		}
+		std::error_code ErrorCode;
+		std::filesystem::create_directory(Dir, ErrorCode);
+		if (ErrorCode)
+		{
+			throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string()));
+		}
+		return true;
 	}
-	return WasCreated;
+	return false;
 }
 
 bool
diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
index 92dc1fb76..5bf6d3001 100644
--- a/zencore/include/zencore/compress.h
+++ b/zencore/include/zencore/compress.h
@@ -71,12 +71,24 @@ public:
 	 *
 	 * @return A compressed buffer, or null on error, such as an invalid format or corrupt header.
 	 */
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData,
+																	 IoHash&				OutRawHash,
+																	 uint64_t&				OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData,
+																	 IoHash&		   OutRawHash,
+																	 uint64_t&		   OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData,
+																	 IoHash&			 OutRawHash,
+																	 uint64_t&			 OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData,
+																	 IoHash&		OutRawHash,
+																	 uint64_t&		OutRawSize);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
+	[[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const IoBuffer& CompressedData,
+																   IoHash&		   OutRawHash,
+																   uint64_t&	   OutRawSize);
 
 	/** Reset this to null. */
 	inline void Reset() { CompressedData.Reset(); }
@@ -102,10 +114,10 @@ public:
 	[[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); }
 
 	/** Returns the size of the raw data. Zero on error or if this is empty or null. */
-	[[nodiscard]] ZENCORE_API uint64_t GetRawSize() const;
+	[[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const;
 
 	/** Returns the hash of the raw data. Zero on error or if this is null. */
-	[[nodiscard]] ZENCORE_API BLAKE3 GetRawHash() const;
+	[[nodiscard]] ZENCORE_API BLAKE3 DecodeRawHash() const;
 
 	[[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const;
 
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 7f107cc0f..a39dbf6d6 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -254,7 +254,7 @@ protected:
 		kContentTypeBit3 = 1 << (24 + 3),  // bits are reserved
 	};
 
-	void AllocateBuffer(size_t InSize, size_t Alignment);
+	void AllocateBuffer(size_t InSize, size_t Alignment) const;
 	void FreeBuffer();
 };
 
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index 16dd22a58..2522daf35 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -34,7 +34,7 @@ namespace zen {
 //////////////////////////////////////////////////////////////////////////
 
 void
-IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment)
+IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) const
 {
 #if ZEN_PLATFORM_WINDOWS
 	if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000))
@@ -238,7 +238,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
 	m_DataPtr = nullptr;
 }
 
-static constexpr size_t MappingLockCount = 64;
+static constexpr size_t MappingLockCount = 128;
 static_assert(IsPow2(MappingLockCount), "MappingLockCount must be power of two");
 
 static RwLock g_MappingLocks[MappingLockCount];
@@ -247,7 +247,7 @@ static RwLock&
 MappingLockForInstance(const IoBufferExtendedCore* instance)
 {
 	intptr_t base		= (intptr_t)instance;
-	size_t	 lock_index = ((base >> 8) ^ (base >> 16)) & (MappingLockCount - 1u);
+	size_t	 lock_index = ((base >> 5) ^ (base >> 13)) & (MappingLockCount - 1u);
 	return g_MappingLocks[lock_index];
 }
 
@@ -271,9 +271,38 @@ IoBufferExtendedCore::Materialize() const
 
 	if (m_DataBytes == 0)
 	{
-		m_Flags.fetch_or(NewFlags, std::memory_order_release);
 		// Fake a "valid" pointer, nobody should read this as size is zero
 		m_DataPtr = reinterpret_cast<uint8_t*>(&m_MmapHandle);
+		m_Flags.fetch_or(NewFlags, std::memory_order_release);
+		return;
+	}
+
+	const size_t DisableMMapSizeLimit = 0x1000ull;
+
+	if (m_DataBytes < DisableMMapSizeLimit)
+	{
+		AllocateBuffer(m_DataBytes, sizeof(void*));
+		NewFlags |= kIsOwnedByThis;
+
+#if ZEN_PLATFORM_WINDOWS
+		OVERLAPPED Ovl{};
+
+		Ovl.Offset	   = DWORD(m_FileOffset & 0xffff'ffffu);
+		Ovl.OffsetHigh = DWORD(m_FileOffset >> 32);
+
+		DWORD dwNumberOfBytesRead = 0;
+		BOOL  Success			  = ::ReadFile(m_FileHandle, (void*)m_DataPtr, DWORD(m_DataBytes), &dwNumberOfBytesRead, &Ovl);
+
+		ZEN_ASSERT(Success);
+		ZEN_ASSERT(dwNumberOfBytesRead == m_DataBytes);
+#else
+		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+		int	 Fd		   = int(uintptr_t(m_FileHandle));
+		int	 BytesRead = pread(Fd, (void*)m_DataPtr, m_DataBytes, m_FileOffset);
+		bool Success   = (BytesRead > 0);
+#endif	// ZEN_PLATFORM_WINDOWS
+
+		m_Flags.fetch_or(NewFlags, std::memory_order_release);
 		return;
 	}