hashing fixes (#657)

* move structuredcachestore tests to zenstore-test
* Don't materialize entire files when hashing if it is a large files
* rewrite CompositeBuffer::Mid to never materialize buffers

2 files changed, 75 insertions, 88 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 4d6b9f89e..615f8640f 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -22,21 +22,6 @@
 
 namespace zen {
 
-bool
-IsKnownBadBucketName(std::string_view Bucket)
-{
-	if (Bucket.size() == 32)
-	{
-		uint8_t BucketHex[16];
-		if (ParseHexBytes(Bucket, BucketHex))
-		{
-			return true;
-		}
-	}
-
-	return false;
-}
-
 namespace {
 
 #pragma pack(push)
@@ -1577,75 +1562,6 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 	}
 }
 
-IoHash
-HashBuffer(const CompositeBuffer& Buffer)
-{
-	IoHashStream Hasher;
-
-	for (const SharedBuffer& Segment : Buffer.GetSegments())
-	{
-		Hasher.Append(Segment.GetView());
-	}
-
-	return Hasher.GetHash();
-}
-
-bool
-ValidateCacheBucketEntryValue(ZenContentType ContentType, IoBuffer Buffer)
-{
-	ZEN_ASSERT_SLOW(Buffer.GetContentType() == ContentType);
-
-	if (ContentType == ZenContentType::kCbObject)
-	{
-		CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
-
-		if (Error == CbValidateError::None)
-		{
-			return true;
-		}
-
-		ZEN_SCOPED_ERROR("compact binary validation failed: '{}'", ToString(Error));
-
-		return false;
-	}
-	else if (ContentType == ZenContentType::kCompressedBinary)
-	{
-		IoBuffer MemoryBuffer = IoBufferBuilder::ReadFromFileMaybe(Buffer);
-
-		IoHash	 HeaderRawHash;
-		uint64_t RawSize = 0;
-		if (!CompressedBuffer::ValidateCompressedHeader(MemoryBuffer, /* out */ HeaderRawHash, /* out */ RawSize))
-		{
-			ZEN_SCOPED_ERROR("compressed buffer header validation failed");
-
-			return false;
-		}
-
-		CompressedBuffer Compressed =
-			CompressedBuffer::FromCompressed(SharedBuffer(MemoryBuffer), /* out */ HeaderRawHash, /* out */ RawSize);
-		CompositeBuffer Decompressed	 = Compressed.DecompressToComposite();
-		IoHash			DecompressedHash = HashBuffer(Decompressed);
-
-		if (HeaderRawHash != DecompressedHash)
-		{
-			ZEN_SCOPED_ERROR("decompressed hash {} differs from header hash {}", DecompressedHash, HeaderRawHash);
-
-			return false;
-		}
-	}
-	else
-	{
-		// No way to verify this kind of content (what is it exactly?)
-
-		static int Once = [&] {
-			ZEN_WARN("ValidateCacheBucketEntryValue called with unknown content type ({})", ToString(ContentType));
-			return 42;
-		}();
-	}
-
-	return true;
-};
-
 void
 ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 {
@@ -1729,7 +1645,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 						ReportBadKey(HashKey);
 						continue;
 					}
-					if (!ValidateCacheBucketEntryValue(Loc.GetContentType(), Buffer))
+					if (!ValidateIoBuffer(Loc.GetContentType(), Buffer))
 					{
 						ReportBadKey(HashKey);
 						continue;
@@ -1768,7 +1684,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 			const BucketPayload& Payload	 = m_Payloads[m_Index.at(Hash)];
 			ZenContentType		 ContentType = Payload.Location.GetContentType();
 			Buffer.SetContentType(ContentType);
-			if (!ValidateCacheBucketEntryValue(ContentType, Buffer))
+			if (!ValidateIoBuffer(ContentType, Buffer))
 			{
 				ReportBadKey(Hash);
 				return;
@@ -1790,7 +1706,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 			const BucketPayload& Payload	 = m_Payloads[m_Index.at(Hash)];
 			ZenContentType		 ContentType = Payload.Location.GetContentType();
 			Buffer.SetContentType(ContentType);
-			if (!ValidateCacheBucketEntryValue(ContentType, Buffer))
+			if (!ValidateIoBuffer(ContentType, Buffer))
 			{
 				ReportBadKey(Hash);
 				return;
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index fd04af2a3..49183600d 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -44,6 +44,77 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
+bool
+IsKnownBadBucketName(std::string_view Bucket)
+{
+	if (Bucket.size() == 32)
+	{
+		uint8_t BucketHex[16];
+		if (ParseHexBytes(Bucket, BucketHex))
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool
+ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer)
+{
+	ZEN_ASSERT_SLOW(Buffer.GetContentType() == ContentType);
+
+	if (ContentType == ZenContentType::kCbObject)
+	{
+		CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
+
+		if (Error == CbValidateError::None)
+		{
+			return true;
+		}
+
+		ZEN_SCOPED_ERROR("compact binary validation failed: '{}'", ToString(Error));
+
+		return false;
+	}
+	else if (ContentType == ZenContentType::kCompressedBinary)
+	{
+		IoBuffer MemoryBuffer = IoBufferBuilder::ReadFromFileMaybe(Buffer);
+
+		IoHash	 HeaderRawHash;
+		uint64_t RawSize = 0;
+		if (!CompressedBuffer::ValidateCompressedHeader(MemoryBuffer, /* out */ HeaderRawHash, /* out */ RawSize))
+		{
+			ZEN_SCOPED_ERROR("compressed buffer header validation failed");
+
+			return false;
+		}
+
+		CompressedBuffer Compressed =
+			CompressedBuffer::FromCompressed(SharedBuffer(MemoryBuffer), /* out */ HeaderRawHash, /* out */ RawSize);
+		CompositeBuffer Decompressed	 = Compressed.DecompressToComposite();
+		IoHash			DecompressedHash = IoHash::HashBuffer(Decompressed);
+
+		if (HeaderRawHash != DecompressedHash)
+		{
+			ZEN_SCOPED_ERROR("decompressed hash {} differs from header hash {}", DecompressedHash, HeaderRawHash);
+
+			return false;
+		}
+	}
+	else
+	{
+		// No way to verify this kind of content (what is it exactly?)
+
+		static int Once = [&] {
+			ZEN_WARN("ValidateIoBuffer called with unknown content type ({})", ToString(ContentType));
+			return 42;
+		}();
+	}
+
+	return true;
+};
+
 ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config)
 : m_Gc(Gc)
 , m_JobQueue(JobQueue)
@@ -2433,7 +2504,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 #endif
 
 void
-z$_forcelink()
+structured_cachestore_forcelink()
 {
 }