use partial blocks for oplog import (#780)

Feature: Add --allow-partial-block-requests to zen oplog-import
Improvement: zen oplog-import now uses partial block requests to reduce download size
Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests

2 files changed, 47 insertions, 0 deletions

diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index d3b59df2b..078e27b34 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -21,6 +21,8 @@
 
 #include "clients/httpclientcommon.h"
 
+#include <numeric>
+
 #if ZEN_WITH_TESTS
 #	include <zencore/scopeguard.h>
 #	include <zencore/testing.h>
@@ -340,6 +342,42 @@ HttpClient::Authenticate()
 	return m_Inner->Authenticate();
 }
 
+LatencyTestResult
+MeasureLatency(HttpClient& Client, std::string_view Url)
+{
+	std::vector<double> MeasurementTimes;
+	std::string			ErrorMessage;
+
+	for (uint32_t AttemptCount = 0; AttemptCount < 20 && MeasurementTimes.size() < 5; AttemptCount++)
+	{
+		HttpClient::Response MeasureResponse = Client.Get(Url);
+		if (MeasureResponse.IsSuccess())
+		{
+			MeasurementTimes.push_back(MeasureResponse.ElapsedSeconds);
+			Sleep(5);
+		}
+		else
+		{
+			ErrorMessage = MeasureResponse.ErrorMessage(fmt::format("Unable to measure latency using {}", Url));
+		}
+	}
+
+	if (MeasurementTimes.empty())
+	{
+		return {.Success = false, .FailureReason = ErrorMessage};
+	}
+
+	if (MeasurementTimes.size() > 2)
+	{
+		std::sort(MeasurementTimes.begin(), MeasurementTimes.end());
+		MeasurementTimes.pop_back();  // Remove the worst time
+	}
+
+	double AverageLatency = std::accumulate(MeasurementTimes.begin(), MeasurementTimes.end(), 0.0) / MeasurementTimes.size();
+
+	return {.Success = true, .LatencySeconds = AverageLatency};
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_WITH_TESTS
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index 9a9b74d72..7a129a98c 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -260,6 +260,15 @@ private:
 	const HttpClientSettings m_ConnectionSettings;
 };
 
+struct LatencyTestResult
+{
+	bool		Success = false;
+	std::string FailureReason;
+	double		LatencySeconds = -1.0;
+};
+
+LatencyTestResult MeasureLatency(HttpClient& Client, std::string_view Url);
+
 void httpclient_forcelink();  // internal
 
 }  // namespace zen