add multirange requests to blob store (#795)

* add multirange requests to blob store

1 files changed, 95 insertions, 19 deletions

diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index bf7afcc02..6ada085a5 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -71,7 +71,7 @@ HttpBuildStoreService::Initialize()
 	m_Router.RegisterRoute(
 		"{namespace}/{bucket}/{buildid}/blobs/{hash}",
 		[this](HttpRouterRequest& Req) { GetBlobRequest(Req); },
-		HttpVerb::kGet);
+		HttpVerb::kGet | HttpVerb::kPost);
 
 	m_Router.RegisterRoute(
 		"{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata",
@@ -161,14 +161,49 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
 										   HttpContentType::kText,
 										   fmt::format("Invalid blob hash '{}'", Hash));
 	}
-	zen::HttpRanges Ranges;
-	bool			HasRange = ServerRequest.TryGetRanges(Ranges);
-	if (Ranges.size() > 1)
+
+	std::vector<std::pair<uint64_t, uint64_t>> OffsetAndLengthPairs;
+	if (ServerRequest.RequestVerb() == HttpVerb::kPost)
 	{
-		// Only a single range is supported
-		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
-										   HttpContentType::kText,
-										   "Multiple ranges in blob request is not supported");
+		CbObject RangePayload = ServerRequest.ReadPayloadObject();
+		if (RangePayload)
+		{
+			CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView();
+			OffsetAndLengthPairs.reserve(RangesArray.Num());
+			for (CbFieldView FieldView : RangesArray)
+			{
+				CbObjectView RangeView	 = FieldView.AsObjectView();
+				uint64_t	 RangeOffset = RangeView["offset"sv].AsUInt64();
+				uint64_t	 RangeLength = RangeView["length"sv].AsUInt64();
+				OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength));
+			}
+		}
+		if (OffsetAndLengthPairs.empty())
+		{
+			m_BuildStoreStats.BadRequestCount++;
+			return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+											   HttpContentType::kText,
+											   "Fetching blob without ranges must be done with the GET verb");
+		}
+	}
+	else
+	{
+		HttpRanges Ranges;
+		bool	   HasRange = ServerRequest.TryGetRanges(Ranges);
+		if (HasRange)
+		{
+			if (Ranges.size() > 1)
+			{
+				// Only a single http range is supported, we have limited support for http multirange responses
+				m_BuildStoreStats.BadRequestCount++;
+				return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+												   HttpContentType::kText,
+												   fmt::format("Multiple ranges in blob request is only supported for {} accept type",
+															   ToString(HttpContentType::kCbPackage)));
+			}
+			const HttpRange& FirstRange = Ranges.front();
+			OffsetAndLengthPairs.push_back(std::make_pair<uint64_t, uint64_t>(FirstRange.Start, FirstRange.End - FirstRange.Start + 1));
+		}
 	}
 
 	m_BuildStoreStats.BlobReadCount++;
@@ -179,24 +214,65 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
 										   HttpContentType::kText,
 										   fmt::format("Blob with hash '{}' could not be found", Hash));
 	}
-	//	ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize());
 	m_BuildStoreStats.BlobHitCount++;
-	if (HasRange)
+
+	if (OffsetAndLengthPairs.empty())
 	{
-		const HttpRange& Range		 = Ranges.front();
-		const uint64_t	 BlobSize	 = Blob.GetSize();
-		const uint64_t	 MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0;
-		const uint64_t	 RangeSize	 = Min(Range.End - Range.Start + 1, MaxBlobSize);
-		if (Range.Start + RangeSize > BlobSize)
+		return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+	}
+
+	if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage)
+	{
+		const uint64_t BlobSize = Blob.GetSize();
+
+		CbPackage			  ResponsePackage;
+		std::vector<IoBuffer> RangeBuffers;
+		CbObjectWriter		  Writer;
+		Writer.BeginArray("ranges"sv);
+		for (const std::pair<uint64_t, uint64_t>& Range : OffsetAndLengthPairs)
 		{
-			return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+			const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0;
+			const uint64_t RangeSize   = Min(Range.second, MaxBlobSize);
+			if (Range.first + RangeSize <= BlobSize)
+			{
+				RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize));
+				Writer.BeginObject();
+				{
+					Writer.AddInteger("offset"sv, Range.first);
+					Writer.AddInteger("length"sv, RangeSize);
+				}
+				Writer.EndObject();
+			}
 		}
-		Blob = IoBuffer(Blob, Range.Start, RangeSize);
-		return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
+		Writer.EndArray();
+
+		CompositeBuffer Ranges(RangeBuffers);
+		CbAttachment	PayloadAttachment(std::move(Ranges), BlobHash);
+		Writer.AddAttachment("payload", PayloadAttachment);
+
+		CbObject HeaderObject = Writer.Save();
+
+		ResponsePackage.AddAttachment(PayloadAttachment);
+		ResponsePackage.SetObject(HeaderObject);
+
+		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
+		uint64_t		ResponseSize	  = RpcResponseBuffer.GetSize();
+		ZEN_UNUSED(ResponseSize);
+		return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
 	}
 	else
 	{
-		return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+		ZEN_ASSERT(OffsetAndLengthPairs.size() == 1);
+		const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front();
+		const uint64_t						 BlobSize		 = Blob.GetSize();
+		const uint64_t						 MaxBlobSize	 = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0;
+		const uint64_t						 RangeSize		 = Min(OffsetAndLength.second, MaxBlobSize);
+		if (OffsetAndLength.first + RangeSize > BlobSize)
+		{
+			return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+		}
+		Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize);
+		return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
 	}
 }