automatic scrub on startup (#667)

- Improvement: Deeper validation of data when scrub is activated (cas/cache/project)
- Improvement: Enabled more multi threading when running scrub operations
- Improvement: Added means to force a scrub operation at startup with a new release using ZEN_DATA_FORCE_SCRUB_VERSION variable in xmake.lua

1 files changed, 21 insertions, 29 deletions

diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index a164f66c3..c0b433c51 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -4,7 +4,9 @@
 
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryutil.h>
 #include <zencore/compactbinaryvalidation.h>
+#include <zencore/compositebuffer.h>
 #include <zencore/compress.h>
 #include <zencore/except.h>
 #include <zencore/filesystem.h>
@@ -71,48 +73,38 @@ IsKnownBadBucketName(std::string_view Bucket)
 }
 
 bool
-ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer)
+ValidateIoBuffer(ZenContentType ContentType, IoBuffer&& Buffer)
 {
 	ZEN_ASSERT_SLOW(Buffer.GetContentType() == ContentType);
 
 	if (ContentType == ZenContentType::kCbObject)
 	{
-		CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
+		uint64_t		BufferSize = Buffer.GetSize();
+		CbValidateError Error	   = CbValidateError::None;
+		CbObject		Object	   = ValidateAndReadCompactBinaryObject(std::move(Buffer), Error);
 
 		if (Error == CbValidateError::None)
 		{
-			return true;
-		}
-
-		ZEN_SCOPED_ERROR("compact binary validation failed: '{}'", ToString(Error));
-
-		return false;
-	}
-	else if (ContentType == ZenContentType::kCompressedBinary)
-	{
-		IoBuffer MemoryBuffer = IoBufferBuilder::ReadFromFileMaybe(Buffer);
-
-		IoHash	 HeaderRawHash;
-		uint64_t RawSize = 0;
-		if (!CompressedBuffer::ValidateCompressedHeader(MemoryBuffer, /* out */ HeaderRawHash, /* out */ RawSize))
-		{
-			ZEN_SCOPED_ERROR("compressed buffer header validation failed");
-
-			return false;
+			if (Object.GetSize() == BufferSize)
+			{
+				return true;
+			}
+			else
+			{
+				ZEN_SCOPED_WARN("compact binary object size {} does not match payload size {}", Object.GetSize(), BufferSize);
+				return false;
+			}
 		}
-
-		CompressedBuffer Compressed =
-			CompressedBuffer::FromCompressed(SharedBuffer(MemoryBuffer), /* out */ HeaderRawHash, /* out */ RawSize);
-		CompositeBuffer Decompressed	 = Compressed.DecompressToComposite();
-		IoHash			DecompressedHash = IoHash::HashBuffer(Decompressed);
-
-		if (HeaderRawHash != DecompressedHash)
+		else
 		{
-			ZEN_SCOPED_ERROR("decompressed hash {} differs from header hash {}", DecompressedHash, HeaderRawHash);
-
+			ZEN_SCOPED_WARN("compact binary validation failed: '{}'", ToString(Error));
 			return false;
 		}
 	}
+	else if (ContentType == ZenContentType::kCompressedBinary)
+	{
+		return ValidateCompressedBuffer(CompositeBuffer(SharedBuffer(std::move(Buffer))), /*OptionalExpectedHash*/ nullptr);
+	}
 	else
 	{
 		// No way to verify this kind of content (what is it exactly?)