diff options
Diffstat (limited to 'src/zenserver/storage/buildstore/httpbuildstore.cpp')
| -rw-r--r-- | src/zenserver/storage/buildstore/httpbuildstore.cpp | 153 |
1 files changed, 131 insertions, 22 deletions
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index f5ba30616..de9589078 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -71,7 +71,7 @@ HttpBuildStoreService::Initialize() m_Router.RegisterRoute( "{namespace}/{bucket}/{buildid}/blobs/{hash}", [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, - HttpVerb::kGet); + HttpVerb::kGet | HttpVerb::kPost); m_Router.RegisterRoute( "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", @@ -161,14 +161,57 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) HttpContentType::kText, fmt::format("Invalid blob hash '{}'", Hash)); } - zen::HttpRanges Ranges; - bool HasRange = ServerRequest.TryGetRanges(Ranges); - if (Ranges.size() > 1) + + std::vector<std::pair<uint64_t, uint64_t>> OffsetAndLengthPairs; + if (ServerRequest.RequestVerb() == HttpVerb::kPost) { - // Only a single range is supported - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Multiple ranges in blob request is not supported"); + CbObject RangePayload = ServerRequest.ReadPayloadObject(); + if (RangePayload) + { + CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView(); + OffsetAndLengthPairs.reserve(RangesArray.Num()); + for (CbFieldView FieldView : RangesArray) + { + CbObjectView RangeView = FieldView.AsObjectView(); + uint64_t RangeOffset = RangeView["offset"sv].AsUInt64(); + uint64_t RangeLength = RangeView["length"sv].AsUInt64(); + OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength)); + } + if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}", + OffsetAndLengthPairs.size(), + MaxRangeCountPerRequestSupported)); + } + } + if (OffsetAndLengthPairs.empty()) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Fetching blob without ranges must be done with the GET verb"); + } + } + else + { + HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (HasRange) + { + if (Ranges.size() > 1) + { + // Only a single http range is supported, we have limited support for http multirange responses + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Multiple ranges in blob request is only supported for {} accept type", + ToString(HttpContentType::kCbPackage))); + } + const HttpRange& FirstRange = Ranges.front(); + OffsetAndLengthPairs.push_back(std::make_pair<uint64_t, uint64_t>(FirstRange.Start, FirstRange.End - FirstRange.Start + 1)); + } } m_BuildStoreStats.BlobReadCount++; @@ -179,24 +222,79 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) HttpContentType::kText, fmt::format("Blob with hash '{}' could not be found", Hash)); } - // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); m_BuildStoreStats.BlobHitCount++; - if (HasRange) + + if (OffsetAndLengthPairs.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } + + if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage) { - const HttpRange& Range = Ranges.front(); - const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; - const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); - if (Range.Start + RangeSize > BlobSize) + const uint64_t BlobSize = Blob.GetSize(); + + CbPackage ResponsePackage; + std::vector<IoBuffer> RangeBuffers; + CbObjectWriter Writer; + Writer.BeginArray("ranges"sv); + for (const std::pair<uint64_t, uint64_t>& Range : OffsetAndLengthPairs) { - return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0; + const uint64_t RangeSize = Min(Range.second, MaxBlobSize); + Writer.BeginObject(); + { + if (Range.first + RangeSize <= BlobSize) + { + RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize)); + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, RangeSize); + } + else + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, 0); + } + } + Writer.EndObject(); } - Blob = IoBuffer(Blob, Range.Start, RangeSize); - return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + Writer.EndArray(); + + CompositeBuffer Ranges(RangeBuffers); + CbAttachment PayloadAttachment(std::move(Ranges), BlobHash); + Writer.AddAttachment("payload", PayloadAttachment); + + CbObject HeaderObject = Writer.Save(); + + ResponsePackage.AddAttachment(PayloadAttachment); + ResponsePackage.SetObject(HeaderObject); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + uint64_t ResponseSize = RpcResponseBuffer.GetSize(); + ZEN_UNUSED(ResponseSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); } else { - return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + if (OffsetAndLengthPairs.size() != 1) + { + // Only a single http range is supported, we have limited support for http multirange responses + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage))); + } + + const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0; + const uint64_t RangeSize = Min(OffsetAndLength.second, MaxBlobSize); + if (OffsetAndLength.first + RangeSize > BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); } } @@ -507,8 +605,8 @@ HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); } -void -HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +CbObject +HttpBuildStoreService::CollectStats() { ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); @@ -562,7 +660,13 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) } Cbo.EndObject(); - return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + return Cbo.Save(); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + Request.WriteResponse(HttpResponseCode::OK, CollectStats()); } void @@ -571,6 +675,11 @@ HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) ZEN_TRACE_CPU("HttpBuildStoreService::Status"); CbObjectWriter Cbo; Cbo << "ok" << true; + Cbo.BeginObject("capabilities"); + { + Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported; + } + Cbo.EndObject(); // capabilities Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } |