updated chunking strategy (#589)

- Improvement: `zen builds`now split large files that are compress only into 64 MB chunks to avoiding very large files in Cloud Storage
- Improvement: `zen builds` now treats `.msixvc` files as non-compressable

Moved and cleaned up compactbinary_helpers functions
Tweaked fixed chunking implementation for better performance
Refactored so we have one list of "non-compressable" extensions
Implemented new `StandardChunkingStrategy` and move the two existing to hidden legacy namespace
Added `FilteredDownloadedBytesPerSecond.Start();` call that got lost during previous refactoring

1 files changed, 18 insertions, 17 deletions

diff --git a/src/zenremotestore/chunking/chunkedcontent.cpp b/src/zenremotestore/chunking/chunkedcontent.cpp
index eb0e8bdc9..ea67e3d94 100644
--- a/src/zenremotestore/chunking/chunkedcontent.cpp
+++ b/src/zenremotestore/chunking/chunkedcontent.cpp
@@ -2,6 +2,7 @@
 
 #include <zenremotestore/chunking/chunkedcontent.h>
 
+#include <zencore/compactbinaryutil.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -362,11 +363,11 @@ LoadFolderContentToCompactBinary(CbObjectView Input)
 {
 	ZEN_TRACE_CPU("LoadFolderContentToCompactBinary");
 	FolderContent Content;
-	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
-	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
-	compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes);
-	compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes);
-	compactbinary_helpers::ReadArray("modificationTimes"sv, Input, Content.ModificationTicks);
+	Content.Platform		  = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
+	Content.Paths			  = compactbinary_helpers::ReadArray<std::filesystem::path>("paths"sv, Input);
+	Content.RawSizes		  = compactbinary_helpers::ReadArray<uint64_t>("rawSizes"sv, Input);
+	Content.Attributes		  = compactbinary_helpers::ReadArray<uint32_t>("attributes"sv, Input);
+	Content.ModificationTicks = compactbinary_helpers::ReadArray<uint64_t>("modificationTimes"sv, Input);
 	return Content;
 }
 
@@ -534,18 +535,18 @@ LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
 {
 	ZEN_TRACE_CPU("LoadChunkedFolderContentToCompactBinary");
 	ChunkedFolderContent Content;
-	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
-	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
-	compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes);
-	compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes);
-	compactbinary_helpers::ReadArray("rawHashes"sv, Input, Content.RawHashes);
-
-	CbObjectView ChunkedContentView = Input["chunkedContent"sv].AsObjectView();
-	compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkedContentView, Content.ChunkedContent.SequenceRawHashes);
-	compactbinary_helpers::ReadArray("chunkCounts"sv, ChunkedContentView, Content.ChunkedContent.ChunkCounts);
-	compactbinary_helpers::ReadArray("chunkOrders"sv, ChunkedContentView, Content.ChunkedContent.ChunkOrders);
-	compactbinary_helpers::ReadArray("chunkHashes"sv, ChunkedContentView, Content.ChunkedContent.ChunkHashes);
-	compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkedContentView, Content.ChunkedContent.ChunkRawSizes);
+	Content.Platform   = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
+	Content.Paths	   = compactbinary_helpers::ReadArray<std::filesystem::path>("paths"sv, Input);
+	Content.RawSizes   = compactbinary_helpers::ReadArray<uint64_t>("rawSizes"sv, Input);
+	Content.Attributes = compactbinary_helpers::ReadArray<uint32_t>("attributes"sv, Input);
+	Content.RawHashes  = compactbinary_helpers::ReadArray<IoHash>("rawHashes"sv, Input);
+
+	CbObjectView ChunkedContentView			 = Input["chunkedContent"sv].AsObjectView();
+	Content.ChunkedContent.SequenceRawHashes = compactbinary_helpers::ReadArray<IoHash>("sequenceRawHashes"sv, ChunkedContentView);
+	Content.ChunkedContent.ChunkCounts		 = compactbinary_helpers::ReadArray<uint32_t>("chunkCounts"sv, ChunkedContentView);
+	Content.ChunkedContent.ChunkOrders		 = compactbinary_helpers::ReadArray<uint32_t>("chunkOrders"sv, ChunkedContentView);
+	Content.ChunkedContent.ChunkHashes		 = compactbinary_helpers::ReadArray<IoHash>("chunkHashes"sv, ChunkedContentView);
+	Content.ChunkedContent.ChunkRawSizes	 = compactbinary_helpers::ReadArray<uint64_t>("chunkRawSizes"sv, ChunkedContentView);
 	return Content;
 }