reduce parsing of compressed headers

4 files changed, 88 insertions, 48 deletions

diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index 19675b9cf..a4fa38a1d 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -135,10 +135,12 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 		if (BinaryView.GetSize() > 0)
 		{
 			// Is a compressed binary blob
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
 			CompressedBuffer Compressed =
-				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned();
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned();
 			Value.emplace<CompressedBuffer>(Compressed);
-			Hash = IoHash::FromBLAKE3(Compressed.GetRawHash());
+			Hash = RawHash;
 			++Fields;
 		}
 		else
@@ -191,8 +193,10 @@ TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Fie
 		if (Buffer.GetSize() > 0)
 		{
 			// Is a compressed binary blob
-			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer));
-			TargetAttachment			= CbAttachment(Compressed, IoHash::FromBLAKE3(Compressed.GetRawHash()));
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
+			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize);
+			TargetAttachment			= CbAttachment(Compressed, RawHash);
 		}
 		else
 		{
@@ -715,9 +719,11 @@ namespace legacy {
 					{
 						return false;
 					}
-					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer))
+					IoHash	 RawHash;
+					uint64_t RawSize;
+					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize))
 					{
-						if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash)
+						if (RawHash != Hash)
 						{
 							return false;
 						}
@@ -747,8 +753,14 @@ namespace legacy {
 				ZEN_ASSERT(Mapper);
 				if (SharedBuffer AttachmentData = (*Mapper)(Hash))
 				{
-					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData))
+					IoHash	 RawHash;
+					uint64_t RawSize;
+					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize))
 					{
+						if (RawHash != Hash)
+						{
+							return false;
+						}
 						Package.AddAttachment(CbAttachment(Compressed, Hash));
 					}
 					else
diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp
index a787e88ab..02148d96a 100644
--- a/zencore/compactbinaryvalidation.cpp
+++ b/zencore/compactbinaryvalidation.cpp
@@ -463,17 +463,18 @@ ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode
 	{
 		if (BinaryView.GetSize() > 0)
 		{
-			CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView));
+			IoHash			 DecodedHash;
+			uint64_t		 DecodedRawSize;
+			CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView), DecodedHash, DecodedRawSize);
 			if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && Buffer.IsNull())
 			{
 				AddError(Error, CbValidateError::NullPackageAttachment);
 			}
-			if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) &&
-				(IoHash::FromBLAKE3(Buffer.GetRawHash()) != IoHash::HashBuffer(Buffer.DecompressToComposite())))
+			if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (DecodedHash != IoHash::HashBuffer(Buffer.DecompressToComposite())))
 			{
 				AddError(Error, CbValidateError::InvalidPackageHash);
 			}
-			return IoHash::FromBLAKE3(Buffer.GetRawHash());
+			return DecodedHash;
 		}
 		else
 		{
diff --git a/zencore/compress.cpp b/zencore/compress.cpp
index 15cc5f6a7..89d35b2fb 100644
--- a/zencore/compress.cpp
+++ b/zencore/compress.cpp
@@ -6,6 +6,7 @@
 #include <zencore/compositebuffer.h>
 #include <zencore/crc32.h>
 #include <zencore/endian.h>
+#include <zencore/iohash.h>
 #include <zencore/testing.h>
 
 #include "../thirdparty/Oodle/include/oodle2.h"
@@ -56,8 +57,11 @@ struct BufferHeader
 	BLAKE3	 RawHash;				   // The hash of the uncompressed data
 
 	/** Checks validity of the buffer based on the magic number, method, and CRC-32. */
-	static bool IsValid(const CompositeBuffer& CompressedData);
-	static bool IsValid(const SharedBuffer& CompressedData) { return IsValid(CompositeBuffer(CompressedData)); }
+	static bool IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
+	static bool IsValid(const SharedBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+	{
+		return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize);
+	}
 
 	/** Read a header from a buffer that is at least sizeof(BufferHeader) without any validation. */
 	static BufferHeader Read(const CompositeBuffer& CompressedData)
@@ -65,6 +69,10 @@ struct BufferHeader
 		BufferHeader Header;
 		if (sizeof(BufferHeader) <= CompressedData.GetSize())
 		{
+			//			if (CompressedData.GetSegments()[0].AsIoBuffer().IsWholeFile())
+			//			{
+			//				ZEN_ASSERT(true);
+			//			}
 			CompositeBuffer::Iterator It;
 			CompressedData.CopyTo(MakeMutableMemoryView(&Header, &Header + 1), It);
 			Header.ByteSwap();
@@ -664,7 +672,7 @@ GetDecoder(CompressionMethod Method)
 //////////////////////////////////////////////////////////////////////////
 
 bool
-BufferHeader::IsValid(const CompositeBuffer& CompressedData)
+BufferHeader::IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	if (sizeof(BufferHeader) <= CompressedData.GetSize())
 	{
@@ -679,6 +687,8 @@ BufferHeader::IsValid(const CompositeBuffer& CompressedData)
 				const MemoryView HeaderView = HeaderCopy.GetView();
 				if (Header.Crc32 == BufferHeader::CalculateCrc32(HeaderView))
 				{
+					OutRawHash = IoHash::FromBLAKE3(Header.RawHash);
+					OutRawSize = Header.TotalRawSize;
 					return true;
 				}
 			}
@@ -691,9 +701,10 @@ BufferHeader::IsValid(const CompositeBuffer& CompressedData)
 
 template<typename BufferType>
 inline CompositeBuffer
-ValidBufferOrEmpty(BufferType&& CompressedData)
+ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
-	return BufferHeader::IsValid(CompressedData) ? CompositeBuffer(std::forward<BufferType>(CompressedData)) : CompositeBuffer();
+	return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize) ? CompositeBuffer(std::forward<BufferType>(CompressedData))
+																		 : CompositeBuffer();
 }
 
 CompositeBuffer
@@ -826,34 +837,34 @@ CompressedBuffer::Compress(const SharedBuffer&	 RawData,
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData)
+CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
 	return Local;
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData)
+CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData));
+	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
 	return Local;
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData)
+CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
 	return Local;
 }
 
 CompressedBuffer
-CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData)
+CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData));
+	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
 	return Local;
 }
 
@@ -882,13 +893,13 @@ CompressedBuffer::FromCompressedNoValidate(CompositeBuffer&& InCompressedData)
 }
 
 uint64_t
-CompressedBuffer::GetRawSize() const
+CompressedBuffer::DecodeRawSize() const
 {
 	return CompressedData ? detail::BufferHeader::Read(CompressedData).TotalRawSize : 0;
 }
 
 BLAKE3
-CompressedBuffer::GetRawHash() const
+CompressedBuffer::DecodeRawHash() const
 {
 	return CompressedData ? detail::BufferHeader::Read(CompressedData).RawHash : BLAKE3();
 }
@@ -1005,18 +1016,20 @@ TEST_CASE("CompressedBuffer")
 															 OodleCompressor::NotSet,
 															 OodleCompressionLevel::None);
 
-		CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
+		CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes));
 		CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(detail::BufferHeader)));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	{
@@ -1025,53 +1038,59 @@ TEST_CASE("CompressedBuffer")
 			OodleCompressor::NotSet,
 			OodleCompressionLevel::None);
 
-		CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
+		CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
 		CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(Ones) + sizeof(detail::BufferHeader)));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	{
 		CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))));
 
-		CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
+		CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes));
 		CHECK(Buffer.GetCompressedSize() < sizeof(Zeroes));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	{
 		CompressedBuffer Buffer = CompressedBuffer::Compress(
 			CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))));
 
-		CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
+		CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
 		CHECK(Buffer.GetCompressedSize() < (sizeof(Zeroes) + sizeof(Ones)));
 
 		CompositeBuffer	 Compressed = Buffer.GetCompressed();
-		CompressedBuffer BufferD	= CompressedBuffer::FromCompressed(Compressed);
+		IoHash			 DecodedHash;
+		uint64_t		 DecodedRawSize;
+		CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
 
 		CHECK(BufferD.IsNull() == false);
 
 		CompositeBuffer Decomp = BufferD.DecompressToComposite();
 
-		CHECK(Decomp.GetSize() == Buffer.GetRawSize());
-		CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+		CHECK(Decomp.GetSize() == DecodedRawSize);
+		CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
 	}
 
 	auto GenerateData = [](uint64_t N) -> std::vector<uint64_t> {
diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
index 92dc1fb76..8d1e4841f 100644
--- a/zencore/include/zencore/compress.h
+++ b/zencore/include/zencore/compress.h
@@ -71,10 +71,18 @@ public:
 	 *
 	 * @return A compressed buffer, or null on error, such as an invalid format or corrupt header.
 	 */
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData);
-	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData,
+																	 IoHash&				OutRawHash,
+																	 uint64_t&				OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData,
+																	 IoHash&		   OutRawHash,
+																	 uint64_t&		   OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData,
+																	 IoHash&			 OutRawHash,
+																	 uint64_t&			 OutRawSize);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData,
+																	 IoHash&		OutRawHash,
+																	 uint64_t&		OutRawSize);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
 
@@ -102,10 +110,10 @@ public:
 	[[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); }
 
 	/** Returns the size of the raw data. Zero on error or if this is empty or null. */
-	[[nodiscard]] ZENCORE_API uint64_t GetRawSize() const;
+	[[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const;
 
 	/** Returns the hash of the raw data. Zero on error or if this is null. */
-	[[nodiscard]] ZENCORE_API BLAKE3 GetRawHash() const;
+	[[nodiscard]] ZENCORE_API BLAKE3 DecodeRawHash() const;
 
 	[[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const;