updated chunking strategy (#589)

- Improvement: `zen builds`now split large files that are compress only into 64 MB chunks to avoiding very large files in Cloud Storage
- Improvement: `zen builds` now treats `.msixvc` files as non-compressable

Moved and cleaned up compactbinary_helpers functions
Tweaked fixed chunking implementation for better performance
Refactored so we have one list of "non-compressable" extensions
Implemented new `StandardChunkingStrategy` and move the two existing to hidden legacy namespace
Added `FilteredDownloadedBytesPerSecond.Start();` call that got lost during previous refactoring

1 files changed, 16 insertions, 7 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 27f050a44..9c6fd17ab 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -550,7 +550,8 @@ namespace {
 													 .TempDir					= TempDir,
 													 .ExcludeFolders			= DefaultExcludeFolders,
 													 .ExcludeExtensions			= DefaultExcludeExtensions,
-													 .ZenExcludeManifestName	= ZenExcludeManifestName});
+													 .ZenExcludeManifestName	= ZenExcludeManifestName,
+													 .NonCompressableExtensions = DefaultSplitOnlyExtensions});
 			UploadOp.Execute();
 			if (AbortFlag)
 			{
@@ -1282,6 +1283,11 @@ namespace {
 			// TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them
 
 			{
+				if (!IsQuiet)
+				{
+					ZEN_CONSOLE("Fetching metadata for {} blocks", BlockRawHashes.size());
+				}
+
 				Stopwatch GetBlockMetadataTimer;
 
 				std::vector<ChunkBlockDescription>			   UnorderedList;
@@ -2045,7 +2051,7 @@ namespace {
 				if (!ChunkController && !IsQuiet)
 				{
 					ZEN_CONSOLE_WARN("Unspecified chunking algorith, using default");
-					ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
+					ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
 				}
 
 				LocalContent = GetLocalContent(LocalFolderScanStats,
@@ -2348,14 +2354,17 @@ namespace {
 		ChunkedFolderContent CompareFolderContent;
 
 		{
-			std::unique_ptr<ChunkingController> ChunkController =
-				CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
-			std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
+			StandardChunkingControllerSettings	ChunkingSettings;
+			std::unique_ptr<ChunkingController> ChunkController	  = CreateStandardChunkingController(ChunkingSettings);
+			std::vector<std::string>			ExcludeExtensions = DefaultExcludeExtensions;
 			if (OnlyChunked)
 			{
 				ExcludeExtensions.insert(ExcludeExtensions.end(),
-										 DefaultChunkingExcludeExtensions.begin(),
-										 DefaultChunkingExcludeExtensions.end());
+										 ChunkingSettings.SplitOnlyExtensions.begin(),
+										 ChunkingSettings.SplitOnlyExtensions.end());
+				ExcludeExtensions.insert(ExcludeExtensions.end(),
+										 ChunkingSettings.SplitAndCompressExtensions.begin(),
+										 ChunkingSettings.SplitAndCompressExtensions.end());
 			}
 
 			auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool {