From c1af02eeb2badfbd2c01125730c6b85bbed8be9e Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Mon, 20 Oct 2025 12:09:46 +0200
Subject: updated chunking strategy (#589)

- Improvement: `zen builds`now split large files that are compress only into 64 MB chunks to avoiding very large files in Cloud Storage
- Improvement: `zen builds` now treats `.msixvc` files as non-compressable

Moved and cleaned up compactbinary_helpers functions
Tweaked fixed chunking implementation for better performance
Refactored so we have one list of "non-compressable" extensions
Implemented new `StandardChunkingStrategy` and move the two existing to hidden legacy namespace
Added `FilteredDownloadedBytesPerSecond.Start();` call that got lost during previous refactoring
---
 src/zencore/include/zencore/compactbinaryutil.h | 132 ++++++++++++++++++++++++
 1 file changed, 132 insertions(+)

(limited to 'src/zencore/include')

diff --git a/src/zencore/include/zencore/compactbinaryutil.h b/src/zencore/include/zencore/compactbinaryutil.h
index d750c6492..eecc3344b 100644
--- a/src/zencore/include/zencore/compactbinaryutil.h
+++ b/src/zencore/include/zencore/compactbinaryutil.h
@@ -52,4 +52,136 @@ ValidateAndReadCompactBinaryObject(const IoBuffer&& Payload, CbValidateError& Ou
 }
 CbObject ValidateAndReadCompactBinaryObject(const CompressedBuffer&& Payload, CbValidateError& OutError);
 
+namespace compactbinary_helpers {
+	template<typename Type>
+	inline void WriteArray(std::span<const Type> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const Type Value : Values)
+		{
+			Output << Value;
+		}
+		Output.EndArray();
+	}
+
+	template<typename Type>
+	inline void WriteArray(const std::vector<Type>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const Type>(Values), ArrayName, Output);
+	}
+
+	template<>
+	inline void WriteArray(std::span<const std::filesystem::path> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const std::filesystem::path& Path : Values)
+		{
+			Output.AddString((const char*)Path.generic_u8string().c_str());
+		}
+		Output.EndArray();
+	}
+
+	template<>
+	inline void WriteArray(const std::vector<std::filesystem::path>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const std::filesystem::path>(Values), ArrayName, Output);
+	}
+
+	inline void WriteBinaryAttachmentArray(std::span<const IoHash> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const IoHash& Hash : Values)
+		{
+			Output.AddBinaryAttachment(Hash);
+		}
+		Output.EndArray();
+	}
+
+	inline void WriteBinaryAttachmentArray(const std::vector<IoHash>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const IoHash>(Values), ArrayName, Output);
+	}
+
+	template<typename Type>
+	std::vector<Type> ReadArray(std::string_view ArrayName, CbObjectView Input);
+
+	template<>
+	inline std::vector<uint32_t> ReadArray(std::string_view ArrayName, CbObjectView Input)
+	{
+		std::vector<uint32_t> Result;
+		CbArrayView			  Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsUInt32());
+		}
+		return Result;
+	}
+
+	template<>
+	inline std::vector<uint64_t> ReadArray(std::string_view ArrayName, CbObjectView Input)
+	{
+		std::vector<uint64_t> Result;
+		CbArrayView			  Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsUInt64());
+		}
+		return Result;
+	}
+
+	template<>
+	inline std::vector<std::filesystem::path> ReadArray(std::string_view ArrayName, CbObjectView Input)
+	{
+		std::vector<std::filesystem::path> Result;
+		CbArrayView						   Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(std::filesystem::path(ItemView.AsU8String()));
+		}
+		return Result;
+	}
+
+	template<>
+	inline std::vector<std::string> ReadArray(std::string_view ArrayName, CbObjectView Input)
+	{
+		std::vector<std::string> Result;
+		CbArrayView				 Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(std::string(ItemView.AsString()));
+		}
+		return Result;
+	}
+
+	template<>
+	inline std::vector<IoHash> ReadArray(std::string_view ArrayName, CbObjectView Input)
+	{
+		std::vector<IoHash> Result;
+		CbArrayView			Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsHash());
+		}
+		return Result;
+	}
+
+	inline std::vector<IoHash> ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input)
+	{
+		std::vector<IoHash> Result;
+		CbArrayView			Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsBinaryAttachment());
+		}
+		return Result;
+	}
+
+}  // namespace compactbinary_helpers
+
 }  // namespace zen
-- 
cgit v1.2.3