Content scrubbing (#271)

Added zen scrub command which may be triggered via the zen CLI helper. This traverses storage and validates contents either by content hash and/or by structure. If unexpected data is encountered it is invalidated.

1 files changed, 73 insertions, 66 deletions

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index a8a4dc102..e9037b16c 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -244,93 +244,100 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 	std::vector<BlockStoreLocation> ChunkLocations;
 	std::vector<IoHash>				ChunkIndexToChunkHash;
 
-	RwLock::SharedLockScope _(m_LocationMapLock);
-
-	uint64_t TotalChunkCount = m_LocationMap.size();
-	ChunkLocations.reserve(TotalChunkCount);
-	ChunkIndexToChunkHash.reserve(TotalChunkCount);
+	try
 	{
-		for (const auto& Entry : m_LocationMap)
+		RwLock::SharedLockScope _(m_LocationMapLock);
+
+		uint64_t TotalChunkCount = m_LocationMap.size();
+		ChunkLocations.reserve(TotalChunkCount);
+		ChunkIndexToChunkHash.reserve(TotalChunkCount);
 		{
-			const IoHash&				  ChunkHash	   = Entry.first;
-			const BlockStoreDiskLocation& DiskLocation = Entry.second;
-			BlockStoreLocation			  Location	   = DiskLocation.Get(m_PayloadAlignment);
+			for (const auto& Entry : m_LocationMap)
+			{
+				const IoHash&				  ChunkHash	   = Entry.first;
+				const BlockStoreDiskLocation& DiskLocation = Entry.second;
+				BlockStoreLocation			  Location	   = DiskLocation.Get(m_PayloadAlignment);
 
-			ChunkLocations.push_back(Location);
-			ChunkIndexToChunkHash.push_back(ChunkHash);
+				ChunkLocations.push_back(Location);
+				ChunkIndexToChunkHash.push_back(ChunkHash);
+			}
 		}
-	}
 
-	const auto ValidateSmallChunk = [&](size_t ChunkIndex, const void* Data, uint64_t Size) {
-		++ChunkCount;
-		ChunkBytes += Size;
+		const auto ValidateSmallChunk = [&](size_t ChunkIndex, const void* Data, uint64_t Size) {
+			++ChunkCount;
+			ChunkBytes += Size;
 
-		const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
-		if (!Data)
-		{
-			// ChunkLocation out of range of stored blocks
-			BadKeys.push_back(Hash);
-			return;
-		}
-
-		IoBuffer Buffer(IoBuffer::Wrap, Data, Size);
-		IoHash	 RawHash;
-		uint64_t RawSize;
-		if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
-		{
-			if (RawHash != Hash)
+			const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
+			if (!Data)
 			{
-				// Hash mismatch
+				// ChunkLocation out of range of stored blocks
 				BadKeys.push_back(Hash);
 				return;
 			}
-			return;
-		}
+
+			IoBuffer Buffer(IoBuffer::Wrap, Data, Size);
+			IoHash	 RawHash;
+			uint64_t RawSize;
+			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
+			{
+				if (RawHash != Hash)
+				{
+					// Hash mismatch
+					BadKeys.push_back(Hash);
+					return;
+				}
+				return;
+			}
 #if ZEN_WITH_TESTS
-		IoHash ComputedHash = IoHash::HashBuffer(Data, Size);
-		if (ComputedHash == Hash)
-		{
-			return;
-		}
+			IoHash ComputedHash = IoHash::HashBuffer(Data, Size);
+			if (ComputedHash == Hash)
+			{
+				return;
+			}
 #endif
-		BadKeys.push_back(Hash);
-	};
+			BadKeys.push_back(Hash);
+		};
 
-	const auto ValidateLargeChunk = [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
-		++ChunkCount;
-		ChunkBytes += Size;
+		const auto ValidateLargeChunk = [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+			Ctx.ThrowIfDeadlineExpired();
 
-		const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
-		IoBuffer	  Buffer(IoBuffer::BorrowedFile, File.GetBasicFile().Handle(), Offset, Size);
+			++ChunkCount;
+			ChunkBytes += Size;
 
-		IoHash	 RawHash;
-		uint64_t RawSize;
-		// TODO: Add API to verify compressed buffer without having to memorymap the whole file
-		if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
-		{
-			if (RawHash != Hash)
+			const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
+			IoBuffer	  Buffer(IoBuffer::BorrowedFile, File.GetBasicFile().Handle(), Offset, Size);
+
+			IoHash	 RawHash;
+			uint64_t RawSize;
+			// TODO: Add API to verify compressed buffer without having to memorymap the whole file
+			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
 			{
-				// Hash mismatch
-				BadKeys.push_back(Hash);
+				if (RawHash != Hash)
+				{
+					// Hash mismatch
+					BadKeys.push_back(Hash);
+					return;
+				}
 				return;
 			}
-			return;
-		}
 #if ZEN_WITH_TESTS
-		IoHashStream Hasher;
-		File.StreamByteRange(Offset, Size, [&](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
-		IoHash ComputedHash = Hasher.GetHash();
-		if (ComputedHash == Hash)
-		{
-			return;
-		}
+			IoHashStream Hasher;
+			File.StreamByteRange(Offset, Size, [&](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
+			IoHash ComputedHash = Hasher.GetHash();
+			if (ComputedHash == Hash)
+			{
+				return;
+			}
 #endif
-		BadKeys.push_back(Hash);
-	};
-
-	m_BlockStore.IterateChunks(ChunkLocations, ValidateSmallChunk, ValidateLargeChunk);
+			BadKeys.push_back(Hash);
+		};
 
-	_.ReleaseNow();
+		m_BlockStore.IterateChunks(ChunkLocations, ValidateSmallChunk, ValidateLargeChunk);
+	}
+	catch (ScrubDeadlineExpiredException&)
+	{
+		ZEN_INFO("Scrubbing deadline expired, operation incomplete");
+	}
 
 	Ctx.ReportScrubbed(ChunkCount, ChunkBytes);