use multi range requests (#800)

- Improvement: `zen builds download` now uses multi-range requests for blocks to reduce download size
- Improvement: `zen oplog-import` now uses partial block with multi-range requests for blocks to reduce download size
- Improvement: Improved feedback in log/console during `zen oplog-import`
- Improvement: `--allow-partial-block-requests` now defaults to `true` for `zen builds download` and `zen oplog-import` (was `mixed`)
- Improvement: Improved range merging analysis when downloading partial blocks

1 files changed, 19 insertions, 5 deletions

diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index 6ada085a5..459e044eb 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -233,16 +233,21 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
 		{
 			const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0;
 			const uint64_t RangeSize   = Min(Range.second, MaxBlobSize);
-			if (Range.first + RangeSize <= BlobSize)
+			Writer.BeginObject();
 			{
-				RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize));
-				Writer.BeginObject();
+				if (Range.first + RangeSize <= BlobSize)
 				{
+					RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize));
 					Writer.AddInteger("offset"sv, Range.first);
 					Writer.AddInteger("length"sv, RangeSize);
 				}
-				Writer.EndObject();
+				else
+				{
+					Writer.AddInteger("offset"sv, Range.first);
+					Writer.AddInteger("length"sv, 0);
+				}
 			}
+			Writer.EndObject();
 		}
 		Writer.EndArray();
 
@@ -262,7 +267,16 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
 	}
 	else
 	{
-		ZEN_ASSERT(OffsetAndLengthPairs.size() == 1);
+		if (OffsetAndLengthPairs.size() != 1)
+		{
+			// Only a single http range is supported, we have limited support for http multirange responses
+			m_BuildStoreStats.BadRequestCount++;
+			return ServerRequest.WriteResponse(
+				HttpResponseCode::BadRequest,
+				HttpContentType::kText,
+				fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage)));
+		}
+
 		const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front();
 		const uint64_t						 BlobSize		 = Blob.GetSize();
 		const uint64_t						 MaxBlobSize	 = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0;