improved scrubbing of oplogs and filecas (#596)

- Improvement: Scrub command now validates compressed buffer hashes in filecas storage (used for large chunks)
- Improvement: Added --dry, --no-gc and --no-cas options to zen scrub command
- Improvement: Implemented oplog scrubbing (previously was a no-op)
- Improvement: Implemented support for running scrubbint at startup with --scrub=<options>

1 files changed, 12 insertions, 26 deletions

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 96ab65a5f..b21f9f8d8 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -254,6 +254,12 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 {
 	ZEN_TRACE_CPU("CasContainer::ScrubStorage");
 
+	if (Ctx.IsSkipCas())
+	{
+		ZEN_INFO("SKIPPED scrubbing: '{}'", m_BlocksBasePath);
+		return;
+	}
+
 	ZEN_INFO("scrubbing '{}'", m_BlocksBasePath);
 
 	std::vector<IoHash>				BadKeys;
@@ -297,21 +303,12 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 			uint64_t RawSize;
 			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
 			{
-				if (RawHash != Hash)
+				if (RawHash == Hash)
 				{
-					// Hash mismatch
-					BadKeys.push_back(Hash);
+					// TODO: this should also hash the (decompressed) contents
 					return;
 				}
-				return;
-			}
-#if ZEN_WITH_TESTS
-			IoHash ComputedHash = IoHash::HashBuffer(Data, Size);
-			if (ComputedHash == Hash)
-			{
-				return;
 			}
-#endif
 			BadKeys.push_back(Hash);
 		};
 
@@ -326,26 +323,15 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 
 			IoHash	 RawHash;
 			uint64_t RawSize;
-			// TODO: Add API to verify compressed buffer without having to memorymap the whole file
+			// TODO: Add API to verify compressed buffer without having to memory-map the whole file
 			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
 			{
-				if (RawHash != Hash)
+				if (RawHash == Hash)
 				{
-					// Hash mismatch
-					BadKeys.push_back(Hash);
+					// TODO: this should also hash the (decompressed) contents
 					return;
 				}
-				return;
-			}
-#if ZEN_WITH_TESTS
-			IoHashStream Hasher;
-			File.StreamByteRange(Offset, Size, [&](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
-			IoHash ComputedHash = Hasher.GetHash();
-			if (ComputedHash == Hash)
-			{
-				return;
 			}
-#endif
 			BadKeys.push_back(Hash);
 		};
 
@@ -398,7 +384,7 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 		Ctx.ReportBadCidChunks(BadKeys);
 	}
 
-	ZEN_INFO("compact cas scrubbed: {} chunks ({})", ChunkCount, NiceBytes(ChunkBytes));
+	ZEN_INFO("scrubbed {} chunks ({}) in '{}'", ChunkCount, NiceBytes(ChunkBytes), m_RootDirectory / m_ContainerBaseName);
 }
 
 void