limit number of chunks per block (#508)

- Improvement: Chunk block generation for `zen oplog-export` and `zen builds upload` command are now limited to max 4000 chunks per block

zen builds list-block testing function to check results for recently used blocks in a context

3 files changed, 77 insertions, 49 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 78e014346..09cbe1aa3 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -174,13 +174,14 @@ namespace {
 			bool			 ForceDisableBlocks	   = File["disableblocks"sv].AsBool(false);
 			bool			 ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false);
 
-			FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-											  FolderPath,
-											  std::string(Name),
-											  std::string(OptionalBaseName),
-											  ForceDisableBlocks,
-											  ForceEnableTempBlocks};
-			RemoteStore					   = CreateFileRemoteStore(Options);
+			FileRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				FolderPath,
+				std::string(Name),
+				std::string(OptionalBaseName),
+				ForceDisableBlocks,
+				ForceEnableTempBlocks};
+			RemoteStore = CreateFileRemoteStore(Options);
 		}
 
 		if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -255,20 +256,21 @@ namespace {
 			bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
 			bool AssumeHttp2			= Cloud["assumehttp2"sv].AsBool(false);
 
-			JupiterRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-												 Url,
-												 std::string(Namespace),
-												 std::string(Bucket),
-												 Key,
-												 BaseKey,
-												 std::string(OpenIdProvider),
-												 AccessToken,
-												 AuthManager,
-												 OidcExePath,
-												 ForceDisableBlocks,
-												 ForceDisableTempBlocks,
-												 AssumeHttp2};
-			RemoteStore						  = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
+			JupiterRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				Url,
+				std::string(Namespace),
+				std::string(Bucket),
+				Key,
+				BaseKey,
+				std::string(OpenIdProvider),
+				AccessToken,
+				AuthManager,
+				OidcExePath,
+				ForceDisableBlocks,
+				ForceDisableTempBlocks,
+				AssumeHttp2};
+			RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
 		}
 
 		if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -284,11 +286,12 @@ namespace {
 			{
 				return {nullptr, "Missing oplog"};
 			}
-			ZenRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-											 std::string(Url),
-											 std::string(Project),
-											 std::string(Oplog)};
-			RemoteStore					  = CreateZenRemoteStore(Options, TempFilePath);
+			ZenRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				std::string(Url),
+				std::string(Project),
+				std::string(Oplog)};
+			RemoteStore = CreateZenRemoteStore(Options, TempFilePath);
 		}
 
 		if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
@@ -352,20 +355,21 @@ namespace {
 			MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
 			IoBuffer   MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
 
-			BuildsRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-												Url,
-												std::string(Namespace),
-												std::string(Bucket),
-												BuildId,
-												std::string(OpenIdProvider),
-												AccessToken,
-												AuthManager,
-												OidcExePath,
-												ForceDisableBlocks,
-												ForceDisableTempBlocks,
-												AssumeHttp2,
-												MetaData};
-			RemoteStore						 = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
+			BuildsRemoteStoreOptions Options = {
+				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+				Url,
+				std::string(Namespace),
+				std::string(Bucket),
+				BuildId,
+				std::string(OpenIdProvider),
+				AccessToken,
+				AuthManager,
+				OidcExePath,
+				ForceDisableBlocks,
+				ForceDisableTempBlocks,
+				AssumeHttp2,
+				MetaData};
+			RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false);
 		}
 
 		if (!RemoteStore)
@@ -5732,6 +5736,14 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 			MaxChunkEmbedSize = Value.value();
 		}
 	}
+	size_t MaxChunksPerBlock = RemoteStoreOptions::DefaultMaxChunksPerBlock;
+	if (auto Param = Params.GetValue("maxchunksperblock"); Param.empty() == false)
+	{
+		if (auto Value = ParseInt<size_t>(Param))
+		{
+			MaxChunksPerBlock = Value.value();
+		}
+	}
 
 	size_t ChunkFileSizeLimit = RemoteStoreOptions::DefaultChunkFileSizeLimit;
 	if (auto Param = Params.GetValue("chunkfilesizelimit"); Param.empty() == false)
@@ -5750,6 +5762,7 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 		*Oplog,
 		MaxBlockSize,
 		MaxChunkEmbedSize,
+		MaxChunksPerBlock,
 		ChunkFileSizeLimit,
 		/* BuildBlocks */ false,
 		/* IgnoreMissingAttachments */ false,
@@ -6149,6 +6162,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 
 	size_t MaxBlockSize				= Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize);
 	size_t MaxChunkEmbedSize		= Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize);
+	size_t MaxChunksPerBlock		= Params["maxchunksperblock"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunksPerBlock);
 	size_t ChunkFileSizeLimit		= Params["chunkfilesizelimit"sv].AsUInt64(RemoteStoreOptions::DefaultChunkFileSizeLimit);
 	bool   Force					= Params["force"sv].AsBool(false);
 	bool   IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false);
@@ -6170,6 +6184,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 		 Project,
 		 OplogPtr = &Oplog,
 		 MaxBlockSize,
+		 MaxChunksPerBlock,
 		 MaxChunkEmbedSize,
 		 ChunkFileSizeLimit,
 		 EmbedLooseFile,
@@ -6187,6 +6202,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 															*Project.Get(),
 															*OplogPtr,
 															MaxBlockSize,
+															MaxChunksPerBlock,
 															MaxChunkEmbedSize,
 															ChunkFileSizeLimit,
 															EmbedLooseFile,
@@ -7529,13 +7545,15 @@ TEST_CASE_TEMPLATE("project.store.export",
 		Oid::NewOid(),
 		CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None)));
 
-	FileRemoteStoreOptions Options = {
-		RemoteStoreOptions{.MaxBlockSize = 64u * 1024, .MaxChunkEmbedSize = 32 * 1024u, .ChunkFileSizeLimit = 64u * 1024u},
-		/*.FolderPath = */ ExportDir.Path(),
-		/*.Name = */ std::string("oplog1"),
-		/*OptionalBaseName = */ std::string(),
-		/*.ForceDisableBlocks = */ Settings::ForceDisableBlocks,
-		/*.ForceEnableTempBlocks = */ Settings::ForceEnableTempBlocks};
+	FileRemoteStoreOptions				Options		= {RemoteStoreOptions{.MaxBlockSize		  = 64u * 1024,
+																		  .MaxChunksPerBlock  = 1000,
+																		  .MaxChunkEmbedSize  = 32 * 1024u,
+																		  .ChunkFileSizeLimit = 64u * 1024u},
+									   /*.FolderPath = */ ExportDir.Path(),
+									   /*.Name = */ std::string("oplog1"),
+									   /*OptionalBaseName = */ std::string(),
+									   /*.ForceDisableBlocks = */ Settings::ForceDisableBlocks,
+									   /*.ForceEnableTempBlocks = */ Settings::ForceEnableTempBlocks};
 	std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options);
 	RemoteProjectStore::RemoteStoreInfo StoreInfo	= RemoteStore->GetInfo();
 
@@ -7544,6 +7562,7 @@ TEST_CASE_TEMPLATE("project.store.export",
 														*Project.Get(),
 														*Oplog,
 														Options.MaxBlockSize,
+														Options.MaxChunksPerBlock,
 														Options.MaxChunkEmbedSize,
 														Options.ChunkFileSizeLimit,
 														true,
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index e61ccd917..407a284c1 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -1167,6 +1167,7 @@ BuildContainer(CidStore&																	ChunkStore,
 			   ProjectStore::Project&														Project,
 			   ProjectStore::Oplog&															Oplog,
 			   size_t																		MaxBlockSize,
+			   size_t																		MaxChunksPerBlock,
 			   size_t																		MaxChunkEmbedSize,
 			   size_t																		ChunkFileSizeLimit,
 			   bool																			BuildBlocks,
@@ -2020,7 +2021,7 @@ BuildContainer(CidStore&																	ChunkStore,
 					}
 					BlockSize += PayloadSize;
 
-					if (BlockSize >= MaxBlockSize && (CurrentOpKey != LastOpKey))
+					if ((BlockSize >= MaxBlockSize || ChunksInBlock.size() > MaxChunksPerBlock) && (CurrentOpKey != LastOpKey))
 					{
 						NewBlock();
 					}
@@ -2079,7 +2080,7 @@ BuildContainer(CidStore&																	ChunkStore,
 								BlockSize += CompressedBuffer::GetHeaderSizeForNoneEncoder() + Source.Size;
 								if (BuildBlocks)
 								{
-									if (BlockSize >= MaxBlockSize)
+									if (BlockSize >= MaxBlockSize || ChunksInBlock.size() > MaxChunksPerBlock)
 									{
 										NewBlock();
 									}
@@ -2286,6 +2287,7 @@ BuildContainer(CidStore&																	ChunkStore,
 			   ProjectStore::Project&														Project,
 			   ProjectStore::Oplog&															Oplog,
 			   size_t																		MaxBlockSize,
+			   size_t																		MaxChunksPerBlock,
 			   size_t																		MaxChunkEmbedSize,
 			   size_t																		ChunkFileSizeLimit,
 			   bool																			BuildBlocks,
@@ -2303,6 +2305,7 @@ BuildContainer(CidStore&																	ChunkStore,
 												 Project,
 												 Oplog,
 												 MaxBlockSize,
+												 MaxChunksPerBlock,
 												 MaxChunkEmbedSize,
 												 ChunkFileSizeLimit,
 												 BuildBlocks,
@@ -2325,6 +2328,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		  ProjectStore::Project& Project,
 		  ProjectStore::Oplog&	 Oplog,
 		  size_t				 MaxBlockSize,
+		  size_t				 MaxChunksPerBlock,
 		  size_t				 MaxChunkEmbedSize,
 		  size_t				 ChunkFileSizeLimit,
 		  bool					 EmbedLooseFiles,
@@ -2474,6 +2478,7 @@ SaveOplog(CidStore&				 ChunkStore,
 												   Project,
 												   Oplog,
 												   MaxBlockSize,
+												   MaxChunksPerBlock,
 												   MaxChunkEmbedSize,
 												   ChunkFileSizeLimit,
 												   RemoteStoreInfo.CreateBlocks,
diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h
index 1210afc7c..8e65bbd58 100644
--- a/src/zenserver/projectstore/remoteprojectstore.h
+++ b/src/zenserver/projectstore/remoteprojectstore.h
@@ -110,10 +110,12 @@ public:
 struct RemoteStoreOptions
 {
 	static const size_t DefaultMaxBlockSize		  = 64u * 1024u * 1024u;
+	static const size_t DefaultMaxChunksPerBlock  = 4u * 1000u;
 	static const size_t DefaultMaxChunkEmbedSize  = 3u * 512u * 1024u;
 	static const size_t DefaultChunkFileSizeLimit = 256u * 1024u * 1024u;
 
 	size_t MaxBlockSize		  = DefaultMaxBlockSize;
+	size_t MaxChunksPerBlock  = DefaultMaxChunksPerBlock;
 	size_t MaxChunkEmbedSize  = DefaultMaxChunkEmbedSize;
 	size_t ChunkFileSizeLimit = DefaultChunkFileSizeLimit;
 };
@@ -125,6 +127,7 @@ RemoteProjectStore::LoadContainerResult BuildContainer(
 	ProjectStore::Project&														 Project,
 	ProjectStore::Oplog&														 Oplog,
 	size_t																		 MaxBlockSize,
+	size_t																		 MaxChunksPerBlock,
 	size_t																		 MaxChunkEmbedSize,
 	size_t																		 ChunkFileSizeLimit,
 	bool																		 BuildBlocks,
@@ -151,6 +154,7 @@ RemoteProjectStore::Result SaveOplog(CidStore&				ChunkStore,
 									 ProjectStore::Project& Project,
 									 ProjectStore::Oplog&	Oplog,
 									 size_t					MaxBlockSize,
+									 size_t					MaxChunksPerBlock,
 									 size_t					MaxChunkEmbedSize,
 									 size_t					ChunkFileSizeLimit,
 									 bool					EmbedLooseFiles,