limit number of chunks per block (#508)

- Improvement: Chunk block generation for `zen oplog-export` and `zen builds upload` command are now limited to max 4000 chunks per block

zen builds list-block testing function to check results for recently used blocks in a context

1 files changed, 109 insertions, 2 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index d9f57959e..fee2adaee 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -263,16 +263,19 @@ namespace {
 	}
 
 	static const size_t DefaultMaxBlockSize		 = 64u * 1024u * 1024u;
+	static const size_t DefaultMaxChunksPerBlock = 4u * 1000u;
 	static const size_t DefaultMaxChunkEmbedSize = 3u * 512u * 1024u;
 
 	struct ChunksBlockParameters
 	{
 		size_t MaxBlockSize		 = DefaultMaxBlockSize;
+		size_t MaxChunksPerBlock = DefaultMaxChunksPerBlock;
 		size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize;
 	};
 
 	const ChunksBlockParameters DefaultChunksBlockParams{
 		.MaxBlockSize	   = 32u * 1024u * 1024u,
+		.MaxChunksPerBlock = DefaultMaxChunksPerBlock,
 		.MaxChunkEmbedSize = 2u * 1024u * 1024u	 // DefaultChunkedParams.MaxSize
 	};
 	const uint64_t DefaultPreferredMultipartChunkSize = 32u * 1024u * 1024u;
@@ -2614,6 +2617,7 @@ namespace {
 	void ArrangeChunksIntoBlocks(const ChunkedFolderContent&		 Content,
 								 const ChunkedContentLookup&		 Lookup,
 								 uint64_t							 MaxBlockSize,
+								 uint64_t							 MaxChunksPerBlock,
 								 std::vector<uint32_t>&				 ChunkIndexes,
 								 std::vector<std::vector<uint32_t>>& OutBlocks)
 	{
@@ -2642,7 +2646,7 @@ namespace {
 			const uint32_t ChunkIndex = ChunkIndexes[ChunkIndexOffset];
 			const uint64_t ChunkSize  = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
 
-			if ((BlockSize + ChunkSize) > MaxBlockSize)
+			if (((BlockSize + ChunkSize) > MaxBlockSize) || (ChunkIndexOffset - ChunkIndexStart) > MaxChunksPerBlock)
 			{
 				// Within the span of MaxBlockSizeLowThreshold and MaxBlockSize, see if there is a break
 				// between source paths for chunks. Break the block at the last such break if any.
@@ -4091,7 +4095,12 @@ namespace {
 		}
 
 		std::vector<std::vector<uint32_t>> NewBlockChunks;
-		ArrangeChunksIntoBlocks(LocalContent, LocalLookup, DefaultChunksBlockParams.MaxBlockSize, NewBlockChunkIndexes, NewBlockChunks);
+		ArrangeChunksIntoBlocks(LocalContent,
+								LocalLookup,
+								DefaultChunksBlockParams.MaxBlockSize,
+								DefaultChunksBlockParams.MaxChunksPerBlock,
+								NewBlockChunkIndexes,
+								NewBlockChunks);
 
 		FindBlocksStats.NewBlocksCount = NewBlockChunks.size();
 		for (uint32_t ChunkIndex : NewBlockChunkIndexes)
@@ -10352,6 +10361,25 @@ BuildsCommand::BuildsCommand()
 	m_ListOptions.parse_positional({"query-path", "result-path"});
 	m_ListOptions.positional_help("query-path result-path");
 
+	// list-blocks
+	AddSystemOptions(m_ListBlocksOptions);
+	AddCloudOptions(m_ListBlocksOptions);
+	AddZenFolderOptions(m_ListBlocksOptions);
+	m_ListBlocksOptions.add_options()("h,help", "Print help");
+	m_ListBlocksOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+	m_ListBlocksOptions.add_option("",
+								   "",
+								   "result-path",
+								   "Path to json or compactbinary to write query result to",
+								   cxxopts::value(m_ListResultPath),
+								   "<result-path>");
+
+	m_ListBlocksOptions
+		.add_option("", "", "max-count", "Maximum number of blocks to list", cxxopts::value(m_ListBlocksMaxCount), "<maxcount>");
+
+	m_ListBlocksOptions.parse_positional({"build-id"});
+	m_ListBlocksOptions.positional_help("build-id");
+
 	// upload
 	AddSystemOptions(m_UploadOptions);
 	AddCloudOptions(m_UploadOptions);
@@ -11304,6 +11332,85 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		return;
 	}
 
+	if (SubOption == &m_ListBlocksOptions)
+	{
+		MakeSafeAbsolutePathÍnPlace(m_ListResultPath);
+
+		if (!m_ListResultPath.empty())
+		{
+			if (!IsQuiet)
+			{
+				ZEN_CONSOLE("Running {}: {} (pid {})",
+							GetRunningExecutablePath(),
+							ZEN_CFG_VERSION_BUILD_STRING_FULL,
+							GetCurrentProcessId());
+			}
+		}
+
+		if (m_ListBlocksMaxCount == 0)
+		{
+			throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_ListBlocksMaxCount), SubOption->help());
+		}
+
+		BuildStorage::Statistics	  StorageStats;
+		BuildStorageCache::Statistics StorageCacheStats;
+
+		if (m_ZenFolderPath.empty())
+		{
+			m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+		}
+		MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+		CreateDirectories(m_ZenFolderPath);
+		auto _ = MakeGuard([this]() {
+			if (CleanDirectory(m_ZenFolderPath, {}))
+			{
+				std::error_code DummyEc;
+				RemoveDir(m_ZenFolderPath, DummyEc);
+			}
+		});
+
+		StorageInstance Storage = CreateBuildStorage(StorageStats,
+													 StorageCacheStats,
+													 ZenTempFolderPath(m_ZenFolderPath),
+													 /*RequriesNamespace*/ true,
+													 /*RequireBucket*/ true);
+
+		const Oid BuildId = ParseBuildId();
+
+		CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_ListBlocksMaxCount);
+		ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
+
+		std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response);
+		if (!IsQuiet)
+		{
+			ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size());
+		}
+		if (m_ListResultPath.empty())
+		{
+			ExtendableStringBuilder<1024> SB;
+			CompactBinaryToJson(Response.GetView(), SB);
+			ForEachStrTok(SB.ToView(), '\n', [](std::string_view Row) {
+				ZEN_CONSOLE("{}", Row);
+				return true;
+			});
+		}
+		else
+		{
+			if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
+			{
+				MemoryView ResponseView = Response.GetView();
+				WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
+			}
+			else
+			{
+				ExtendableStringBuilder<1024> SB;
+				CompactBinaryToJson(Response.GetView(), SB);
+				WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+			}
+		}
+		return;
+	}
 	if (SubOption == &m_UploadOptions)
 	{
 		if (!IsQuiet)