automatic scrub on startup (#667)

- Improvement: Deeper validation of data when scrub is activated (cas/cache/project)
- Improvement: Enabled more multi threading when running scrub operations
- Improvement: Added means to force a scrub operation at startup with a new release using ZEN_DATA_FORCE_SCRUB_VERSION variable in xmake.lua

1 files changed, 17 insertions, 4 deletions

diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp
index 6c489c5d3..938a1a011 100644
--- a/src/zenserver/storage/upstream/upstreamcache.cpp
+++ b/src/zenserver/storage/upstream/upstreamcache.cpp
@@ -208,7 +208,10 @@ namespace detail {
 
 								IoHash	 RawHash;
 								uint64_t RawSize;
-								if (CompressedBuffer::ValidateCompressedHeader(AttachmentResult.Response, RawHash, RawSize))
+								if (CompressedBuffer::ValidateCompressedHeader(AttachmentResult.Response,
+																			   RawHash,
+																			   RawSize,
+																			   /*OutOptionalTotalCompressedSize*/ nullptr))
 								{
 									Result.Response = AttachmentResult.Response;
 									++NumAttachments;
@@ -425,7 +428,10 @@ namespace detail {
 					m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason);
 					if (Payload && IsCompressedBinary(Payload.GetContentType()))
 					{
-						IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize);
+						IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload,
+																				  RawHash,
+																				  RawSize,
+																				  /*OutOptionalTotalCompressedSize*/ nullptr);
 					}
 				}
 
@@ -481,7 +487,11 @@ namespace detail {
 					{
 						if (IsCompressedBinary(Payload.GetContentType()))
 						{
-							IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize) && RawHash != PayloadHash;
+							IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload,
+																					  RawHash,
+																					  RawSize,
+																					  /*OutOptionalTotalCompressedSize*/ nullptr) &&
+										   RawHash != PayloadHash;
 						}
 						else
 						{
@@ -559,7 +569,10 @@ namespace detail {
 				{
 					IoHash	 RawHash;
 					uint64_t RawSize;
-					if (!CompressedBuffer::ValidateCompressedHeader(RecordValue, RawHash, RawSize))
+					if (!CompressedBuffer::ValidateCompressedHeader(RecordValue,
+																	RawHash,
+																	RawSize,
+																	/*OutOptionalTotalCompressedSize*/ nullptr))
 					{
 						return {.Reason = std::string("Invalid compressed value buffer"), .Success = false};
 					}