automatic scrub on startup (#667)

- Improvement: Deeper validation of data when scrub is activated (cas/cache/project)
- Improvement: Enabled more multi threading when running scrub operations
- Improvement: Added means to force a scrub operation at startup with a new release using ZEN_DATA_FORCE_SCRUB_VERSION variable in xmake.lua

3 files changed, 61 insertions, 21 deletions

diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 0c53a8000..1c623db4d 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -59,10 +59,16 @@ struct BufferHeader
 	BLAKE3	 RawHash;				   // The hash of the uncompressed data
 
 	/** Checks validity of the buffer based on the magic number, method, and CRC-32. */
-	static bool IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
-	static bool IsValid(const SharedBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+	static bool IsValid(const CompositeBuffer& CompressedData,
+						IoHash&				   OutRawHash,
+						uint64_t&			   OutRawSize,
+						uint64_t*			   OutOptionalTotalCompressedSize);
+	static bool IsValid(const SharedBuffer& CompressedData,
+						IoHash&				OutRawHash,
+						uint64_t&			OutRawSize,
+						uint64_t*			OutOptionalTotalCompressedSize)
 	{
-		return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize);
+		return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
 	}
 
 	/** Read a header from a buffer that is at least sizeof(BufferHeader) without any validation. */
@@ -1467,13 +1473,20 @@ ReadHeader(const CompositeBuffer& CompressedData, BufferHeader& OutHeader, Uniqu
 }
 
 bool
-BufferHeader::IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+BufferHeader::IsValid(const CompositeBuffer& CompressedData,
+					  IoHash&				 OutRawHash,
+					  uint64_t&				 OutRawSize,
+					  uint64_t*				 OutOptionalTotalCompressedSize)
 {
 	detail::BufferHeader Header;
 	if (ReadHeader(CompressedData, Header, nullptr))
 	{
 		OutRawHash = IoHash::FromBLAKE3(Header.RawHash);
 		OutRawSize = Header.TotalRawSize;
+		if (OutOptionalTotalCompressedSize)
+		{
+			*OutOptionalTotalCompressedSize = Header.TotalCompressedSize;
+		}
 		return true;
 	}
 	return false;
@@ -1643,10 +1656,11 @@ private:
 
 template<typename BufferType>
 inline CompositeBuffer
-ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize, uint64_t* OutOptionalTotalCompressedSize)
 {
-	return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize) ? CompositeBuffer(std::forward<BufferType>(CompressedData))
-																		 : CompositeBuffer();
+	return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize, OutOptionalTotalCompressedSize)
+			   ? CompositeBuffer(std::forward<BufferType>(CompressedData))
+			   : CompositeBuffer();
 }
 
 CompositeBuffer
@@ -1893,7 +1907,7 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1901,7 +1915,8 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
+	Local.CompressedData =
+		detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1909,7 +1924,7 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
+	Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1917,7 +1932,8 @@ CompressedBuffer
 CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
 	CompressedBuffer Local;
-	Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
+	Local.CompressedData =
+		detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize, /*OutOptionalTotalCompressedSize*/ nullptr);
 	return Local;
 }
 
@@ -1946,15 +1962,30 @@ CompressedBuffer::FromCompressedNoValidate(CompositeBuffer&& InCompressedData)
 }
 
 bool
-CompressedBuffer::ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+CompressedBuffer::ValidateCompressedHeader(IoBuffer&& CompressedData,
+										   IoHash&	  OutRawHash,
+										   uint64_t&  OutRawSize,
+										   uint64_t*  OutOptionalTotalCompressedSize)
+{
+	return detail::BufferHeader::IsValid(SharedBuffer(std::move(CompressedData)), OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
+}
+
+bool
+CompressedBuffer::ValidateCompressedHeader(const IoBuffer& CompressedData,
+										   IoHash&		   OutRawHash,
+										   uint64_t&	   OutRawSize,
+										   uint64_t*	   OutOptionalTotalCompressedSize)
 {
-	return detail::BufferHeader::IsValid(SharedBuffer(std::move(CompressedData)), OutRawHash, OutRawSize);
+	return detail::BufferHeader::IsValid(SharedBuffer(CompressedData), OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
 }
 
 bool
-CompressedBuffer::ValidateCompressedHeader(const IoBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+CompressedBuffer::ValidateCompressedHeader(const CompositeBuffer& CompressedData,
+										   IoHash&				  OutRawHash,
+										   uint64_t&			  OutRawSize,
+										   uint64_t*			  OutOptionalTotalCompressedSize)
 {
-	return detail::BufferHeader::IsValid(SharedBuffer(CompressedData), OutRawHash, OutRawSize);
+	return detail::BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize, OutOptionalTotalCompressedSize);
 }
 
 size_t
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
index 09fa6249d..3802a8c31 100644
--- a/src/zencore/include/zencore/compress.h
+++ b/src/zencore/include/zencore/compress.h
@@ -100,12 +100,20 @@ public:
 																	 uint64_t&		OutRawSize);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
-	[[nodiscard]] ZENCORE_API static bool	ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
-	[[nodiscard]] ZENCORE_API static bool	ValidateCompressedHeader(const IoBuffer& CompressedData,
-																	 IoHash&		 OutRawHash,
-																	 uint64_t&		 OutRawSize);
-	[[nodiscard]] ZENCORE_API static size_t GetHeaderSizeForNoneEncoder();
-	[[nodiscard]] ZENCORE_API static UniqueBuffer CreateHeaderForNoneEncoder(uint64_t RawSize, const BLAKE3& RawHash);
+	[[nodiscard]] ZENCORE_API static bool			  ValidateCompressedHeader(IoBuffer&& CompressedData,
+																			   IoHash&	  OutRawHash,
+																			   uint64_t&  OutRawSize,
+																			   uint64_t*  OutOptionalTotalCompressedSize);
+	[[nodiscard]] ZENCORE_API static bool			  ValidateCompressedHeader(const IoBuffer& CompressedData,
+																			   IoHash&		   OutRawHash,
+																			   uint64_t&	   OutRawSize,
+																			   uint64_t*	   OutOptionalTotalCompressedSize);
+	[[nodiscard]] ZENCORE_API static bool			  ValidateCompressedHeader(const CompositeBuffer& CompressedData,
+																			   IoHash&				  OutRawHash,
+																			   uint64_t&			  OutRawSize,
+																			   uint64_t*			  OutOptionalTotalCompressedSize);
+	[[nodiscard]] ZENCORE_API static size_t			  GetHeaderSizeForNoneEncoder();
+	[[nodiscard]] ZENCORE_API static UniqueBuffer	  CreateHeaderForNoneEncoder(uint64_t RawSize, const BLAKE3& RawHash);
 
 	/** Reset this to null. */
 	inline void Reset() { CompressedData.Reset(); }
diff --git a/src/zencore/include/zencore/config.h.in b/src/zencore/include/zencore/config.h.in
index 3372eca2a..93256756d 100644
--- a/src/zencore/include/zencore/config.h.in
+++ b/src/zencore/include/zencore/config.h.in
@@ -14,3 +14,4 @@
 #define ZEN_CFG_VERSION_BUILD_STRING "${VERSION}-${plat}-${arch}-${mode}"
 #define ZEN_CFG_VERSION_BUILD_STRING_FULL "${VERSION}-${VERSION_BUILD}-${plat}-${arch}-${mode}-${GIT_COMMIT}"
 #define ZEN_CFG_SCHEMA_VERSION ${ZEN_SCHEMA_VERSION}
+#define ZEN_CFG_DATA_FORCE_SCRUB_VERSION ${ZEN_DATA_FORCE_SCRUB_VERSION}