automatic scrub on startup (#667)

- Improvement: Deeper validation of data when scrub is activated (cas/cache/project)
- Improvement: Enabled more multi threading when running scrub operations
- Improvement: Added means to force a scrub operation at startup with a new release using ZEN_DATA_FORCE_SCRUB_VERSION variable in xmake.lua

1 files changed, 46 insertions, 15 deletions

diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 0c53a8000..1c623db4d 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -59,10 +59,16 @@ struct BufferHeader
 	BLAKE3	 RawHash;				   // The hash of the uncompressed data
 
 	/** Checks validity of the buffer based on the magic number, method, and CRC-32. */
-	static bool IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
-	static bool IsValid(const SharedBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+	static bool IsValid(const CompositeBuffer& CompressedData,
+						IoHash&				   OutRawHash,
+						uint64_t&			   OutRawSize,
+						uint64_t*			   OutOptionalTotalCompressedSize);
+	static bool IsValid(const SharedBuffer& CompressedData,
+						IoHash&				OutRawHash,
+						uint64_t&			OutRawSize,
+						uint64_t*			OutOptionalTotalCompressedSize)
 	{
-		return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize);
+		return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
 	}
 
 	/** Read a header from a buffer that is at least sizeof(BufferHeader) without any validation. */
@@ -1467,13 +1473,20 @@ ReadHeader(const CompositeBuffer& CompressedData, BufferHeader& OutHeader, Uniqu
 }
 
 bool
-BufferHeader::IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+BufferHeader::IsValid(const CompositeBuffer& CompressedData,
+					  IoHash&				 OutRawHash,
+					  uint64_t&				 OutRawSize,
+					  uint64_t*				 OutOptionalTotalCompressedSize)
 {
 	detail::BufferHeader Header;
 	if (ReadHeader(CompressedData, Header, nullptr))
 	{
 		OutRawHash = IoHash::FromBLAKE3(Header.RawHash);
 		OutRawSize = Header.TotalRawSize;
+		if (OutOptionalTotalCompressedSize)
+		{
+			*OutOptionalTotalCompressedSize = Header.TotalCompressedSize;
+		}
 		return true;
 	}
 	return false;
@@ -1643,10 +1656,11 @@ private:
 
 template<typename BufferType>
 inline CompositeBuffer
-ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize, uint64_t* OutOptionalTotalCompressedSize)
 {
-	return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize) ? CompositeBuffer(std::forward<BufferType>(CompressedData))
-																		 : CompositeBuffer();
+	return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize, OutOptionalTotalCompressedSize)
+			   ? CompositeBuffer(std::forward<BufferType>(CompressedData))
+			   : CompositeBuffer();
 }
 
 CompositeBuffer
@@ -1893,7 +1907,7 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1901,7 +1915,8 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
+	Local.CompressedData =
+		detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1909,7 +1924,7 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1917,7 +1932,8 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
+	Local.CompressedData =
+		detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1946,15 +1962,30 @@ CompressedBuffer::FromCompressedNoValidate(CompositeBuffer&& InCompressedData)
 }
 
 bool
-CompressedBuffer::ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+CompressedBuffer::ValidateCompressedHeader(IoBuffer&& CompressedData,
+										   IoHash&	  OutRawHash,
+										   uint64_t&  OutRawSize,
+										   uint64_t*  OutOptionalTotalCompressedSize)
+{
+	return detail::BufferHeader::IsValid(SharedBuffer(std::move(CompressedData)), OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
+}
+
+bool
+CompressedBuffer::ValidateCompressedHeader(const IoBuffer& CompressedData,
+										   IoHash&		   OutRawHash,
+										   uint64_t&	   OutRawSize,
+										   uint64_t*	   OutOptionalTotalCompressedSize)
 {
-	return detail::BufferHeader::IsValid(SharedBuffer(std::move(CompressedData)), OutRawHash, OutRawSize);
+	return detail::BufferHeader::IsValid(SharedBuffer(CompressedData), OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
 }
 
 bool
-CompressedBuffer::ValidateCompressedHeader(const IoBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+CompressedBuffer::ValidateCompressedHeader(const CompositeBuffer& CompressedData,
+										   IoHash&				  OutRawHash,
+										   uint64_t&			  OutRawSize,
+										   uint64_t*			  OutOptionalTotalCompressedSize)
 {
-	return detail::BufferHeader::IsValid(SharedBuffer(CompressedData), OutRawHash, OutRawSize);
+	return detail::BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
 }
 
 size_t