From 4984e8cd5c38cf77c8cb978f75f808bce0577f2d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 27 Nov 2025 16:05:56 +0100 Subject: automatic scrub on startup (#667) - Improvement: Deeper validation of data when scrub is activated (cas/cache/project) - Improvement: Enabled more multi threading when running scrub operations - Improvement: Added means to force a scrub operation at startup with a new release using ZEN_DATA_FORCE_SCRUB_VERSION variable in xmake.lua --- src/zenstore/cache/structuredcachestore.cpp | 50 ++++++++++++----------------- 1 file changed, 21 insertions(+), 29 deletions(-) (limited to 'src/zenstore/cache/structuredcachestore.cpp') diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index a164f66c3..c0b433c51 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -4,7 +4,9 @@ #include #include +#include #include +#include #include #include #include @@ -71,48 +73,38 @@ IsKnownBadBucketName(std::string_view Bucket) } bool -ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer) +ValidateIoBuffer(ZenContentType ContentType, IoBuffer&& Buffer) { ZEN_ASSERT_SLOW(Buffer.GetContentType() == ContentType); if (ContentType == ZenContentType::kCbObject) { - CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All); + uint64_t BufferSize = Buffer.GetSize(); + CbValidateError Error = CbValidateError::None; + CbObject Object = ValidateAndReadCompactBinaryObject(std::move(Buffer), Error); if (Error == CbValidateError::None) { - return true; - } - - ZEN_SCOPED_ERROR("compact binary validation failed: '{}'", ToString(Error)); - - return false; - } - else if (ContentType == ZenContentType::kCompressedBinary) - { - IoBuffer MemoryBuffer = IoBufferBuilder::ReadFromFileMaybe(Buffer); - - IoHash HeaderRawHash; - uint64_t RawSize = 0; - if (!CompressedBuffer::ValidateCompressedHeader(MemoryBuffer, /* out */ HeaderRawHash, /* out */ RawSize)) - { - ZEN_SCOPED_ERROR("compressed buffer header validation failed"); - - return false; + if (Object.GetSize() == BufferSize) + { + return true; + } + else + { + ZEN_SCOPED_WARN("compact binary object size {} does not match payload size {}", Object.GetSize(), BufferSize); + return false; + } } - - CompressedBuffer Compressed = - CompressedBuffer::FromCompressed(SharedBuffer(MemoryBuffer), /* out */ HeaderRawHash, /* out */ RawSize); - CompositeBuffer Decompressed = Compressed.DecompressToComposite(); - IoHash DecompressedHash = IoHash::HashBuffer(Decompressed); - - if (HeaderRawHash != DecompressedHash) + else { - ZEN_SCOPED_ERROR("decompressed hash {} differs from header hash {}", DecompressedHash, HeaderRawHash); - + ZEN_SCOPED_WARN("compact binary validation failed: '{}'", ToString(Error)); return false; } } + else if (ContentType == ZenContentType::kCompressedBinary) + { + return ValidateCompressedBuffer(CompositeBuffer(SharedBuffer(std::move(Buffer))), /*OptionalExpectedHash*/ nullptr); + } else { // No way to verify this kind of content (what is it exactly?) -- cgit v1.2.3