hashing fixes (#657)

* move structuredcachestore tests to zenstore-test
* Don't materialize entire files when hashing if it is a large files
* rewrite CompositeBuffer::Mid to never materialize buffers

1 files changed, 3 insertions, 87 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 4d6b9f89e..615f8640f 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -22,21 +22,6 @@
 
 namespace zen {
 
-bool
-IsKnownBadBucketName(std::string_view Bucket)
-{
-	if (Bucket.size() == 32)
-	{
-		uint8_t BucketHex[16];
-		if (ParseHexBytes(Bucket, BucketHex))
-		{
-			return true;
-		}
-	}
-
-	return false;
-}
-
 namespace {
 
 #pragma pack(push)
@@ -1577,75 +1562,6 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 	}
 }
 
-IoHash
-HashBuffer(const CompositeBuffer& Buffer)
-{
-	IoHashStream Hasher;
-
-	for (const SharedBuffer& Segment : Buffer.GetSegments())
-	{
-		Hasher.Append(Segment.GetView());
-	}
-
-	return Hasher.GetHash();
-}
-
-bool
-ValidateCacheBucketEntryValue(ZenContentType ContentType, IoBuffer Buffer)
-{
-	ZEN_ASSERT_SLOW(Buffer.GetContentType() == ContentType);
-
-	if (ContentType == ZenContentType::kCbObject)
-	{
-		CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
-
-		if (Error == CbValidateError::None)
-		{
-			return true;
-		}
-
-		ZEN_SCOPED_ERROR("compact binary validation failed: '{}'", ToString(Error));
-
-		return false;
-	}
-	else if (ContentType == ZenContentType::kCompressedBinary)
-	{
-		IoBuffer MemoryBuffer = IoBufferBuilder::ReadFromFileMaybe(Buffer);
-
-		IoHash	 HeaderRawHash;
-		uint64_t RawSize = 0;
-		if (!CompressedBuffer::ValidateCompressedHeader(MemoryBuffer, /* out */ HeaderRawHash, /* out */ RawSize))
-		{
-			ZEN_SCOPED_ERROR("compressed buffer header validation failed");
-
-			return false;
-		}
-
-		CompressedBuffer Compressed =
-			CompressedBuffer::FromCompressed(SharedBuffer(MemoryBuffer), /* out */ HeaderRawHash, /* out */ RawSize);
-		CompositeBuffer Decompressed	 = Compressed.DecompressToComposite();
-		IoHash			DecompressedHash = HashBuffer(Decompressed);
-
-		if (HeaderRawHash != DecompressedHash)
-		{
-			ZEN_SCOPED_ERROR("decompressed hash {} differs from header hash {}", DecompressedHash, HeaderRawHash);
-
-			return false;
-		}
-	}
-	else
-	{
-		// No way to verify this kind of content (what is it exactly?)
-
-		static int Once = [&] {
-			ZEN_WARN("ValidateCacheBucketEntryValue called with unknown content type ({})", ToString(ContentType));
-			return 42;
-		}();
-	}
-
-	return true;
-};
-
 void
 ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 {
@@ -1729,7 +1645,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 						ReportBadKey(HashKey);
 						continue;
 					}
-					if (!ValidateCacheBucketEntryValue(Loc.GetContentType(), Buffer))
+					if (!ValidateIoBuffer(Loc.GetContentType(), Buffer))
 					{
 						ReportBadKey(HashKey);
 						continue;
@@ -1768,7 +1684,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 			const BucketPayload& Payload	 = m_Payloads[m_Index.at(Hash)];
 			ZenContentType		 ContentType = Payload.Location.GetContentType();
 			Buffer.SetContentType(ContentType);
-			if (!ValidateCacheBucketEntryValue(ContentType, Buffer))
+			if (!ValidateIoBuffer(ContentType, Buffer))
 			{
 				ReportBadKey(Hash);
 				return;
@@ -1790,7 +1706,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 			const BucketPayload& Payload	 = m_Payloads[m_Index.at(Hash)];
 			ZenContentType		 ContentType = Payload.Location.GetContentType();
 			Buffer.SetContentType(ContentType);
-			if (!ValidateCacheBucketEntryValue(ContentType, Buffer))
+			if (!ValidateIoBuffer(ContentType, Buffer))
 			{
 				ReportBadKey(Hash);
 				return;