use partial blocks for oplog import (#780)

Feature: Add --allow-partial-block-requests to zen oplog-import
Improvement: zen oplog-import now uses partial block requests to reduce download size
Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests

1 files changed, 27 insertions, 1 deletions

diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 4de6ad25c..bedab3cfd 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -1469,6 +1469,20 @@ ImportOplogCommand::ImportOplogCommand()
 						 "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
 						 cxxopts::value(m_BoostWorkers),
 						 "<boostworkermemory>");
+	m_Options.add_option(
+		"",
+		"",
+		"allow-partial-block-requests",
+		"Allow request for partial chunk blocks.\n"
+		"  false        = only full block requests allowed\n"
+		"  mixed        = multiple partial block ranges requests per block allowed to zen cache, single partial block range "
+		"request per block to host\n"
+		"  zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests "
+		"allowed to host\n"
+		"  true         = multiple partial block ranges requests per block allowed to zen cache and host\n"
+		"Defaults to 'mixed'.",
+		cxxopts::value(m_AllowPartialBlockRequests),
+		"<allowpartialblockrequests>");
 
 	m_Options.parse_positional({"project", "oplog", "gcpath"});
 	m_Options.positional_help("[<projectid> <oplogid> [<gcpath>]]");
@@ -1513,6 +1527,13 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 		throw OptionParseException("'--oplog' is required", m_Options.help());
 	}
 
+	EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
+	if (Mode == EPartialBlockRequestMode::Invalid)
+	{
+		throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
+								   m_Options.help());
+	}
+
 	HttpClient Http(m_HostName);
 	m_ProjectName = ResolveProject(Http, m_ProjectName);
 	if (m_ProjectName.empty())
@@ -1649,6 +1670,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 			{
 				Writer.AddBool("boostworkermemory"sv, true);
 			}
+
+			Writer.AddString("partialblockrequestmode", m_AllowPartialBlockRequests);
+
 			if (!m_FileDirectoryPath.empty())
 			{
 				Writer.BeginObject("file"sv);
@@ -2571,6 +2595,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 	ClientSettings.AssumeHttp2				   = ResolveRes.HostAssumeHttp2;
 	ClientSettings.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u;
 	Storage.BuildStorageHttp				   = std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings);
+	Storage.BuildStorageLatencySec			   = ResolveRes.HostLatencySec;
 
 	BuildStorageCache::Statistics StorageCacheStats;
 
@@ -2589,7 +2614,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 				.RetryCount					 = 0,
 				.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u},
 			[&AbortFlag]() { return AbortFlag.load(); });
-		Storage.CacheName = ResolveRes.CacheName;
+		Storage.CacheName		= ResolveRes.CacheName;
+		Storage.CacheLatencySec = ResolveRes.CacheLatencySec;
 	}
 
 	if (!m_Quiet)