automatic scrub on startup (#667)

- Improvement: Deeper validation of data when scrub is activated (cas/cache/project)
- Improvement: Enabled more multi threading when running scrub operations
- Improvement: Added means to force a scrub operation at startup with a new release using ZEN_DATA_FORCE_SCRUB_VERSION variable in xmake.lua

2 files changed, 8 insertions, 41 deletions

diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 76f36a921..ba82c5956 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -2926,7 +2926,10 @@ BuildsOperationUpdateFolder::FindDownloadedChunk(const IoHash& ChunkHash)
 		{
 			IoHash	 RawHash;
 			uint64_t RawSize;
-			if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize))
+			if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart,
+														   RawHash,
+														   RawSize,
+														   /*OutOptionalTotalCompressedSize*/ nullptr))
 			{
 				return CompressedChunkPath;
 			}
diff --git a/src/zenremotestore/builds/filebuildstorage.cpp b/src/zenremotestore/builds/filebuildstorage.cpp
index 96d81b281..153deaa9f 100644
--- a/src/zenremotestore/builds/filebuildstorage.cpp
+++ b/src/zenremotestore/builds/filebuildstorage.cpp
@@ -9,6 +9,7 @@
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
 #include <zencore/trace.h>
+#include <zenstore/scrubcontext.h>
 
 namespace zen {
 
@@ -278,7 +279,7 @@ public:
 		ZEN_UNUSED(BuildId);
 		ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
 
-		ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, Payload));
+		ZEN_ASSERT_SLOW(ValidateCompressedBuffer(Payload, &RawHash));
 
 		uint64_t ReceivedBytes = 0;
 		uint64_t SentBytes	   = Payload.GetSize();
@@ -374,7 +375,7 @@ public:
 					if (IsLastPart)
 					{
 						Workload->TempFile.Flush();
-						ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(Workload->TempFile.ReadAll())));
+						ZEN_ASSERT_SLOW(ValidateCompressedBuffer(CompositeBuffer(Workload->TempFile.ReadAll()), &RawHash));
 						Workload->TempFile.MoveTemporaryIntoPlace(BlockPath, Ec);
 						if (Ec)
 						{
@@ -423,7 +424,7 @@ public:
 			else
 			{
 				Payload = File.ReadAll();
-				ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload))));
+				ZEN_ASSERT_SLOW(ValidateCompressedBuffer(CompositeBuffer(SharedBuffer(Payload)), &RawHash));
 			}
 			Payload.SetContentType(ZenContentType::kCompressedBinary);
 			ReceivedBytes = Payload.GetSize();
@@ -734,43 +735,6 @@ protected:
 		return NeededAttachments;
 	}
 
-	bool ValidateCompressedBuffer(const IoHash& RawHash, const CompositeBuffer& Payload)
-	{
-		IoHash			 VerifyHash;
-		uint64_t		 VerifySize;
-		CompressedBuffer ValidateBuffer = CompressedBuffer::FromCompressed(Payload, VerifyHash, VerifySize);
-		if (!ValidateBuffer)
-		{
-			return false;
-		}
-		if (VerifyHash != RawHash)
-		{
-			return false;
-		}
-
-		IoHashStream Hash;
-		bool		 CouldDecompress = ValidateBuffer.DecompressToStream(
-			0,
-			(uint64_t)-1,
-			[&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
-				ZEN_UNUSED(SourceOffset, SourceSize, Offset);
-				for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
-				{
-					Hash.Append(Segment.GetView());
-				}
-				return true;
-			});
-		if (!CouldDecompress)
-		{
-			return false;
-		}
-		if (Hash.GetHash() != VerifyHash)
-		{
-			return false;
-		}
-		return true;
-	}
-
 private:
 	void AddStatistic(Stopwatch& ExecutionTimer, uint64_t UploadedBytes, uint64_t DownloadedBytes)
 	{