limit number of chunks per block (#508)

- Improvement: Chunk block generation for `zen oplog-export` and `zen builds upload` command are now limited to max 4000 chunks per block

zen builds list-block testing function to check results for recently used blocks in a context

1 files changed, 66 insertions, 47 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 78e014346..09cbe1aa3 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -174,13 +174,14 @@ namespace {
 			bool			 ForceDisableBlocks	   = File["disableblocks"sv].AsBool(false);
 			bool			 ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false);
 
-			FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-											  FolderPath,
-											  std::string(Name),
-											  std::string(OptionalBaseName),
-											  ForceDisableBlocks,
-											  ForceEnableTempBlocks};
-			RemoteStore					   = CreateFileRemoteStore(Options);
+			FileRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				FolderPath,
+				std::string(Name),
+				std::string(OptionalBaseName),
+				ForceDisableBlocks,
+				ForceEnableTempBlocks};
+			RemoteStore = CreateFileRemoteStore(Options);
 		}
 
 		if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -255,20 +256,21 @@ namespace {
 			bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
 			bool AssumeHttp2			= Cloud["assumehttp2"sv].AsBool(false);
 
-			JupiterRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-												 Url,
-												 std::string(Namespace),
-												 std::string(Bucket),
-												 Key,
-												 BaseKey,
-												 std::string(OpenIdProvider),
-												 AccessToken,
-												 AuthManager,
-												 OidcExePath,
-												 ForceDisableBlocks,
-												 ForceDisableTempBlocks,
-												 AssumeHttp2};
-			RemoteStore						  = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
+			JupiterRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				Url,
+				std::string(Namespace),
+				std::string(Bucket),
+				Key,
+				BaseKey,
+				std::string(OpenIdProvider),
+				AccessToken,
+				AuthManager,
+				OidcExePath,
+				ForceDisableBlocks,
+				ForceDisableTempBlocks,
+				AssumeHttp2};
+			RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
 		}
 
 		if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -284,11 +286,12 @@ namespace {
 			{
 				return {nullptr, "Missing oplog"};
 			}
-			ZenRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-											 std::string(Url),
-											 std::string(Project),
-											 std::string(Oplog)};
-			RemoteStore					  = CreateZenRemoteStore(Options, TempFilePath);
+			ZenRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				std::string(Url),
+				std::string(Project),
+				std::string(Oplog)};
+			RemoteStore = CreateZenRemoteStore(Options, TempFilePath);
 		}
 
 		if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
@@ -352,20 +355,21 @@ namespace {
 			MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
 			IoBuffer   MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
 
-			BuildsRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-												Url,
-												std::string(Namespace),
-												std::string(Bucket),
-												BuildId,
-												std::string(OpenIdProvider),
-												AccessToken,
-												AuthManager,
-												OidcExePath,
-												ForceDisableBlocks,
-												ForceDisableTempBlocks,
-												AssumeHttp2,
-												MetaData};
-			RemoteStore						 = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
+			BuildsRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				Url,
+				std::string(Namespace),
+				std::string(Bucket),
+				BuildId,
+				std::string(OpenIdProvider),
+				AccessToken,
+				AuthManager,
+				OidcExePath,
+				ForceDisableBlocks,
+				ForceDisableTempBlocks,
+				AssumeHttp2,
+				MetaData};
+			RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
 		}
 
 		if (!RemoteStore)
@@ -5732,6 +5736,14 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 			MaxChunkEmbedSize = Value.value();
 		}
 	}
+	size_t MaxChunksPerBlock = RemoteStoreOptions::DefaultMaxChunksPerBlock;
+	if (auto Param = Params.GetValue("maxchunksperblock"); Param.empty() == false)
+	{
+		if (auto Value = ParseInt<size_t>(Param))
+		{
+			MaxChunksPerBlock = Value.value();
+		}
+	}
 
 	size_t ChunkFileSizeLimit = RemoteStoreOptions::DefaultChunkFileSizeLimit;
 	if (auto Param = Params.GetValue("chunkfilesizelimit"); Param.empty() == false)
@@ -5750,6 +5762,7 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 		*Oplog,
 		MaxBlockSize,
 		MaxChunkEmbedSize,
+		MaxChunksPerBlock,
 		ChunkFileSizeLimit,
 		/* BuildBlocks */ false,
 		/* IgnoreMissingAttachments */ false,
@@ -6149,6 +6162,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 
 	size_t MaxBlockSize				= Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize);
 	size_t MaxChunkEmbedSize		= Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize);
+	size_t MaxChunksPerBlock		= Params["maxchunksperblock"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunksPerBlock);
 	size_t ChunkFileSizeLimit		= Params["chunkfilesizelimit"sv].AsUInt64(RemoteStoreOptions::DefaultChunkFileSizeLimit);
 	bool   Force					= Params["force"sv].AsBool(false);
 	bool   IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false);
@@ -6170,6 +6184,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 		 Project,
 		 OplogPtr = &Oplog,
 		 MaxBlockSize,
+		 MaxChunksPerBlock,
 		 MaxChunkEmbedSize,
 		 ChunkFileSizeLimit,
 		 EmbedLooseFile,
@@ -6187,6 +6202,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 															*Project.Get(),
 															*OplogPtr,
 															MaxBlockSize,
+															MaxChunksPerBlock,
 															MaxChunkEmbedSize,
 															ChunkFileSizeLimit,
 															EmbedLooseFile,
@@ -7529,13 +7545,15 @@ TEST_CASE_TEMPLATE("project.store.export",
 		Oid::NewOid(),
 		CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None)));
 
-	FileRemoteStoreOptions Options = {
-		RemoteStoreOptions{.MaxBlockSize = 64u * 1024, .MaxChunkEmbedSize = 32 * 1024u, .ChunkFileSizeLimit = 64u * 1024u},
-		/*.FolderPath = */ ExportDir.Path(),
-		/*.Name = */ std::string("oplog1"),
-		/*OptionalBaseName = */ std::string(),
-		/*.ForceDisableBlocks = */ Settings::ForceDisableBlocks,
-		/*.ForceEnableTempBlocks = */ Settings::ForceEnableTempBlocks};
+	FileRemoteStoreOptions				Options		= {RemoteStoreOptions{.MaxBlockSize		  = 64u * 1024,
+																		  .MaxChunksPerBlock  = 1000,
+																		  .MaxChunkEmbedSize  = 32 * 1024u,
+																		  .ChunkFileSizeLimit = 64u * 1024u},
+									   /*.FolderPath = */ ExportDir.Path(),
+									   /*.Name = */ std::string("oplog1"),
+									   /*OptionalBaseName = */ std::string(),
+									   /*.ForceDisableBlocks = */ Settings::ForceDisableBlocks,
+									   /*.ForceEnableTempBlocks = */ Settings::ForceEnableTempBlocks};
 	std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options);
 	RemoteProjectStore::RemoteStoreInfo StoreInfo	= RemoteStore->GetInfo();
 
@@ -7544,6 +7562,7 @@ TEST_CASE_TEMPLATE("project.store.export",
 														*Project.Get(),
 														*Oplog,
 														Options.MaxBlockSize,
+														Options.MaxChunksPerBlock,
 														Options.MaxChunkEmbedSize,
 														Options.ChunkFileSizeLimit,
 														true,