use multi range requests (#800)

- Improvement: `zen builds download` now uses multi-range requests for blocks to reduce download size
- Improvement: `zen oplog-import` now uses partial block with multi-range requests for blocks to reduce download size
- Improvement: Improved feedback in log/console during `zen oplog-import`
- Improvement: `--allow-partial-block-requests` now defaults to `true` for `zen builds download` and `zen oplog-import` (was `mixed`)
- Improvement: Improved range merging analysis when downloading partial blocks

1 files changed, 65 insertions, 0 deletions

diff --git a/src/zenremotestore/jupiter/jupitersession.cpp b/src/zenremotestore/jupiter/jupitersession.cpp
index 1bc6564ce..52f9eb678 100644
--- a/src/zenremotestore/jupiter/jupitersession.cpp
+++ b/src/zenremotestore/jupiter/jupitersession.cpp
@@ -852,6 +852,71 @@ JupiterSession::GetBuildBlob(std::string_view	   Namespace,
 	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
 }
 
+BuildBlobRangesResult
+JupiterSession::GetBuildBlob(std::string_view								Namespace,
+							 std::string_view								BucketId,
+							 const Oid&										BuildId,
+							 const IoHash&									Hash,
+							 std::filesystem::path							TempFolderPath,
+							 std::span<const std::pair<uint64_t, uint64_t>> Ranges)
+{
+	HttpClient::KeyValueMap Headers;
+	if (!Ranges.empty())
+	{
+		ExtendableStringBuilder<512> SB;
+		for (const std::pair<uint64_t, uint64_t>& R : Ranges)
+		{
+			if (SB.Size() > 0)
+			{
+				SB << ", ";
+			}
+			SB << R.first << "-" << R.first + R.second - 1;
+		}
+		Headers.Entries.insert({"Range", fmt::format("bytes={}", SB.ToView())});
+	}
+	std::string Url = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}",
+								  Namespace,
+								  BucketId,
+								  BuildId,
+								  Hash.ToHexString(),
+								  m_AllowRedirect ? "true"sv : "false"sv);
+
+	HttpClient::Response Response = m_HttpClient.Download(Url, TempFolderPath, Headers);
+	if (Response.StatusCode == HttpResponseCode::RangeNotSatisfiable && Ranges.size() > 1)
+	{
+		// Requests to Jupiter that is not served via nginx (content not stored locally in the file system) can not serve multi-range
+		// requests (asp.net limitation) This rejection is not implemented as of 2026-03-02, it is in the backlog (@joakim.lindqvist)
+		// If we encounter this error we fall back to a single range which covers all the requested ranges
+		uint64_t RangeStart = Ranges.front().first;
+		uint64_t RangeEnd	= Ranges.back().first + Ranges.back().second - 1;
+		Headers.Entries.insert_or_assign("Range", fmt::format("bytes={}-{}", RangeStart, RangeEnd));
+		Response = m_HttpClient.Download(Url, TempFolderPath, Headers);
+	}
+	if (Response.IsSuccess())
+	{
+		// If we get a redirect to S3 or a non-Jupiter endpoint the content type will not be correct, validate it and set it
+		if (m_AllowRedirect && (Response.ResponsePayload.GetContentType() == HttpContentType::kBinary))
+		{
+			IoHash	 ValidateRawHash;
+			uint64_t ValidateRawSize = 0;
+			if (!Headers.Entries.contains("Range"))
+			{
+				ZEN_ASSERT_SLOW(CompressedBuffer::ValidateCompressedHeader(Response.ResponsePayload,
+																		   ValidateRawHash,
+																		   ValidateRawSize,
+																		   /*OutOptionalTotalCompressedSize*/ nullptr));
+				ZEN_ASSERT_SLOW(ValidateRawHash == Hash);
+				ZEN_ASSERT_SLOW(ValidateRawSize > 0);
+				ZEN_UNUSED(ValidateRawHash, ValidateRawSize);
+				Response.ResponsePayload.SetContentType(ZenContentType::kCompressedBinary);
+			}
+		}
+	}
+	BuildBlobRangesResult Result = {detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv)};
+	Result.Ranges				 = Response.GetRanges(Ranges);
+	return Result;
+}
+
 JupiterResult
 JupiterSession::PutBlockMetadata(std::string_view Namespace,
 								 std::string_view BucketId,