merged from main

1 files changed, 116 insertions, 24 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 0ccd5d52a..4be33170c 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -1308,56 +1308,148 @@ ZenCacheDiskLayer::CacheBucket::SaveManifest()
 void
 ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 {
-	std::vector<IoHash> BadKeys;
+	std::vector<IoHash>				BadKeys;
+	std::vector<BlockStoreLocation> ChunkLocations;
+	std::vector<IoHash>				ChunkIndexToChunkHash;
+
+	auto ValidateEntry = [](ZenContentType ContentType, IoBuffer Buffer) {
+		if (ContentType == ZenContentType::kCbObject)
+		{
+			CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
+			return Error == CbValidateError::None;
+		}
+		if (ContentType == ZenContentType::kCompressedBinary)
+		{
+			if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer)); !Compressed)
+			{
+				return false;
+			}
+		}
+		return true;
+	};
+
+	RwLock::SharedLockScope _(m_IndexLock);
+
+	const size_t BlockChunkInitialCount = m_Index.size() / 4;
+	ChunkLocations.reserve(BlockChunkInitialCount);
+	ChunkIndexToChunkHash.reserve(BlockChunkInitialCount);
 
+	for (auto& Kv : m_Index)
 	{
-		RwLock::SharedLockScope _(m_IndexLock);
+		const IoHash&		HashKey = Kv.first;
+		const DiskLocation& Loc		= Kv.second.Location;
 
-		for (auto& Kv : m_Index)
+		if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 		{
-			const IoHash&		HashKey = Kv.first;
-			const DiskLocation& Loc		= Kv.second.Location;
+			if (Loc.GetContentType() == ZenContentType::kBinary)
+			{
+				ExtendablePathBuilder<256> DataFilePath;
+				BuildPath(DataFilePath, HashKey);
 
-			ZenCacheValue Value;
+				RwLock::SharedLockScope ValueLock(LockForHash(HashKey));
 
-			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
-			{
-				if (GetStandaloneCacheValue(Loc, HashKey, Value))
+				std::error_code Ec;
+				uintmax_t		size = std::filesystem::file_size(DataFilePath.ToPath(), Ec);
+				if (Ec)
 				{
-					// Note: we cannot currently validate contents since we don't
-					// have a content hash!
-					continue;
+					BadKeys.push_back(HashKey);
+				}
+				if (size != Loc.Size())
+				{
+					BadKeys.push_back(HashKey);
 				}
+				continue;
 			}
-			else if (GetInlineCacheValue(Loc, Value))
+			ZenCacheValue Value;
+			if (!GetStandaloneCacheValue(Loc, HashKey, Value))
 			{
-				// Validate contents
+				BadKeys.push_back(HashKey);
+				continue;
+			}
+			if (!ValidateEntry(Loc.GetContentType(), Value.Value))
+			{
+				BadKeys.push_back(HashKey);
 				continue;
 			}
-			// Value not found
-			BadKeys.push_back(HashKey);
+		}
+		else
+		{
+			ChunkLocations.emplace_back(Loc.GetBlockLocation(m_PayloadAlignment));
+			ChunkIndexToChunkHash.push_back(HashKey);
+			continue;
 		}
 	}
 
+	const auto ValidateSmallChunk = [&](size_t ChunkIndex, const void* Data, uint64_t Size) {
+		const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
+		if (!Data)
+		{
+			// ChunkLocation out of range of stored blocks
+			BadKeys.push_back(Hash);
+			return;
+		}
+		IoBuffer Buffer(IoBuffer::Wrap, Data, Size);
+		if (!Buffer)
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+		ZenContentType ContentType = m_Index.at(Hash).Location.GetContentType();
+		if (!ValidateEntry(ContentType, Buffer))
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+	};
+
+	const auto ValidateLargeChunk = [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+		const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
+		IoBuffer	  Buffer(IoBuffer::BorrowedFile, File.GetBasicFile().Handle(), Offset, Size);
+		if (!Buffer)
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+		ZenContentType ContentType = m_Index.at(Hash).Location.GetContentType();
+		if (!ValidateEntry(ContentType, Buffer))
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+	};
+
+	m_BlockStore.IterateChunks(ChunkLocations, ValidateSmallChunk, ValidateLargeChunk);
+
+	_.ReleaseNow();
+
 	if (BadKeys.empty())
 	{
 		return;
 	}
 
+	ZEN_ERROR("Scrubbing found #{} bad chunks in '{}'", BadKeys.size(), m_BucketDir / m_BucketName);
+
 	if (Ctx.RunRecovery())
 	{
-		RwLock::ExclusiveLockScope _(m_IndexLock);
+		// Deal with bad chunks by removing them from our lookup map
+
+		std::vector<DiskIndexEntry> LogEntries;
+		LogEntries.reserve(BadKeys.size());
 
-		for (const IoHash& BadKey : BadKeys)
 		{
-			// Log a tombstone and delete the in-memory index for the bad entry
+			RwLock::ExclusiveLockScope __(m_IndexLock);
+			for (const IoHash& BadKey : BadKeys)
+			{
+				// Log a tombstone and delete the in-memory index for the bad entry
 
-			const auto	 It		  = m_Index.find(BadKey);
-			DiskLocation Location = It->second.Location;
-			Location.Flags |= DiskLocation::kTombStone;
-			m_SlogFile.Append(DiskIndexEntry{.Key = BadKey, .Location = Location});
-			m_Index.erase(BadKey);
+				const auto	 It		  = m_Index.find(BadKey);
+				DiskLocation Location = It->second.Location;
+				Location.Flags |= DiskLocation::kTombStone;
+				LogEntries.push_back(DiskIndexEntry{.Key = BadKey, .Location = Location});
+				m_Index.erase(BadKey);
+			}
 		}
+		m_SlogFile.Append(LogEntries);
 	}
 
 	// Let whomever it concerns know about the bad chunks. This could