aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2026-04-10 19:09:10 +0200
committerGitHub Enterprise <[email protected]>2026-04-10 19:09:10 +0200
commit1783a5c414c6ae2088bacf6183580432fddbfbe7 (patch)
treea4d3e40bbb7f1a111e50b41c1e85fc1e5bd565b7 /src
parentreduce test runtime (#933) (diff)
downloadzen-1783a5c414c6ae2088bacf6183580432fddbfbe7.tar.xz
zen-1783a5c414c6ae2088bacf6183580432fddbfbe7.zip
HTTP range responses (RFC 7233) - httpobjectstore (#928)
- Improvement: HTTP range responses (RFC 7233) are now fully compliant across the object store and build store - 206 Partial Content responses now include a `Content-Range` header; previously absent for single-range requests, which broke `HttpClient::GetRanges()` - 416 Range Not Satisfiable responses now include `Content-Range: bytes */N` as required by RFC 7233 - Out-of-bounds range requests return 416 Range Not Satisfiable (was 400 Bad Request) - Single-byte ranges (`bytes=N-N`) are now correctly accepted (were previously rejected) - Range byte positions widened from 32-bit to 64-bit; RFC 7233 imposes no size limit on byte range values - Build store binary GET requests with a Range header now return 206 Partial Content with `Content-Range` (previously returned 200 OK without it)
Diffstat (limited to 'src')
-rw-r--r--src/zenhttp/httpserver.cpp95
-rw-r--r--src/zenhttp/include/zenhttp/httpcommon.h14
-rw-r--r--src/zenhttp/include/zenhttp/httpserver.h4
-rw-r--r--src/zenhttp/servers/httpasio.cpp28
-rw-r--r--src/zenhttp/servers/httpplugin.cpp32
-rw-r--r--src/zenhttp/servers/httpsys.cpp30
-rw-r--r--src/zenserver-test/buildstore-tests.cpp142
-rw-r--r--src/zenserver-test/objectstore-tests.cpp101
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.cpp144
-rw-r--r--src/zenserver/storage/objectstore/objectstore.cpp50
10 files changed, 506 insertions, 134 deletions
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 38021be16..3668e1e8f 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -266,10 +266,10 @@ TryParseHttpRangeHeader(std::string_view RangeHeader, HttpRanges& Ranges)
return false;
}
- const auto Start = ParseInt<uint32_t>(Token.substr(0, Delim));
- const auto End = ParseInt<uint32_t>(Token.substr(Delim + 1));
+ const auto Start = ParseInt<uint64_t>(Token.substr(0, Delim));
+ const auto End = ParseInt<uint64_t>(Token.substr(Delim + 1));
- if (Start.has_value() && End.has_value() && End.value() > Start.value())
+ if (Start.has_value() && End.has_value() && End.value() >= Start.value())
{
Ranges.push_back({.Start = Start.value(), .End = End.value()});
}
@@ -286,6 +286,45 @@ TryParseHttpRangeHeader(std::string_view RangeHeader, HttpRanges& Ranges)
return Count != Ranges.size();
}
+MultipartByteRangesResult
+BuildMultipartByteRanges(const IoBuffer& Data, const HttpRanges& Ranges)
+{
+ Oid::String_t BoundaryStr;
+ Oid::NewOid().ToString(BoundaryStr);
+ std::string_view Boundary(BoundaryStr, Oid::StringLength);
+
+ const uint64_t TotalSize = Data.GetSize();
+
+ std::vector<IoBuffer> Parts;
+ Parts.reserve(Ranges.size() * 2 + 1);
+
+ for (const HttpRange& Range : Ranges)
+ {
+ uint64_t RangeEnd = (Range.End != ~uint64_t(0)) ? Range.End : TotalSize - 1;
+ if (RangeEnd >= TotalSize || Range.Start > RangeEnd)
+ {
+ return {};
+ }
+
+ uint64_t RangeSize = 1 + (RangeEnd - Range.Start);
+
+ std::string PartHeader = fmt::format("\r\n--{}\r\nContent-Type: application/octet-stream\r\nContent-Range: bytes {}-{}/{}\r\n\r\n",
+ Boundary,
+ Range.Start,
+ RangeEnd,
+ TotalSize);
+ Parts.push_back(IoBufferBuilder::MakeCloneFromMemory(PartHeader.data(), PartHeader.size()));
+
+ IoBuffer RangeData(Data, Range.Start, RangeSize);
+ Parts.push_back(RangeData);
+ }
+
+ std::string ClosingBoundary = fmt::format("\r\n--{}--", Boundary);
+ Parts.push_back(IoBufferBuilder::MakeCloneFromMemory(ClosingBoundary.data(), ClosingBoundary.size()));
+
+ return {.Parts = std::move(Parts), .ContentType = fmt::format("multipart/byteranges; boundary={}", Boundary)};
+}
+
//////////////////////////////////////////////////////////////////////////
const std::string_view
@@ -564,6 +603,56 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType
}
void
+HttpServerRequest::WriteResponse(HttpContentType ContentType, const IoBuffer& Data, const HttpRanges& Ranges)
+{
+ if (Ranges.empty())
+ {
+ WriteResponse(HttpResponseCode::OK, ContentType, IoBuffer(Data));
+ return;
+ }
+
+ if (Ranges.size() == 1)
+ {
+ const HttpRange& Range = Ranges[0];
+ const uint64_t TotalSize = Data.GetSize();
+ // ~uint64_t(0) is the sentinel meaning "end of file" (suffix range).
+ const uint64_t RangeEnd = (Range.End != ~uint64_t(0)) ? Range.End : TotalSize - 1;
+
+ if (RangeEnd >= TotalSize || Range.Start > RangeEnd)
+ {
+ m_ContentRangeHeader = fmt::format("bytes */{}", TotalSize);
+ WriteResponse(HttpResponseCode::RangeNotSatisfiable);
+ return;
+ }
+
+ const uint64_t RangeSize = 1 + (RangeEnd - Range.Start);
+ IoBuffer RangeBuf(Data, Range.Start, RangeSize);
+
+ m_ContentRangeHeader = fmt::format("bytes {}-{}/{}", Range.Start, RangeEnd, TotalSize);
+ WriteResponse(HttpResponseCode::PartialContent, ContentType, std::move(RangeBuf));
+ return;
+ }
+
+ // Multi-range
+ MultipartByteRangesResult MultipartResult = BuildMultipartByteRanges(Data, Ranges);
+ if (MultipartResult.Parts.empty())
+ {
+ m_ContentRangeHeader = fmt::format("bytes */{}", Data.GetSize());
+ WriteResponse(HttpResponseCode::RangeNotSatisfiable);
+ return;
+ }
+ WriteResponse(HttpResponseCode::PartialContent, std::move(MultipartResult.ContentType), std::span<IoBuffer>(MultipartResult.Parts));
+}
+
+void
+HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, const std::string& CustomContentType, std::span<IoBuffer> Blobs)
+{
+ ZEN_ASSERT(ParseContentType(CustomContentType) == HttpContentType::kUnknownContentType);
+ m_ContentTypeOverride = CustomContentType;
+ WriteResponse(ResponseCode, HttpContentType::kBinary, Blobs);
+}
+
+void
HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, CompositeBuffer& Payload)
{
std::span<const SharedBuffer> Segments = Payload.GetSegments();
diff --git a/src/zenhttp/include/zenhttp/httpcommon.h b/src/zenhttp/include/zenhttp/httpcommon.h
index f9a99f3cc..1d921600d 100644
--- a/src/zenhttp/include/zenhttp/httpcommon.h
+++ b/src/zenhttp/include/zenhttp/httpcommon.h
@@ -19,8 +19,8 @@ class StringBuilderBase;
struct HttpRange
{
- uint32_t Start = ~uint32_t(0);
- uint32_t End = ~uint32_t(0);
+ uint64_t Start = ~uint64_t(0);
+ uint64_t End = ~uint64_t(0);
};
using HttpRanges = std::vector<HttpRange>;
@@ -30,6 +30,16 @@ extern HttpContentType (*ParseContentType)(const std::string_view& ContentTypeSt
std::string_view ReasonStringForHttpResultCode(int HttpCode);
bool TryParseHttpRangeHeader(std::string_view RangeHeader, HttpRanges& Ranges);
+struct MultipartByteRangesResult
+{
+ std::vector<IoBuffer> Parts;
+ std::string ContentType;
+};
+
+// Build a multipart/byteranges response body from the given data and ranges.
+// Generates a unique boundary per call. Returns empty Parts if any range is out of bounds.
+MultipartByteRangesResult BuildMultipartByteRanges(const IoBuffer& Data, const HttpRanges& Ranges);
+
enum class HttpVerb : uint8_t
{
kGet = 1 << 0,
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index e8bfcfd4d..136304426 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -122,11 +122,13 @@ public:
virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) = 0;
virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, CompositeBuffer& Payload);
+ void WriteResponse(HttpResponseCode ResponseCode, const std::string& CustomContentType, std::span<IoBuffer> Blobs);
void WriteResponse(HttpResponseCode ResponseCode, CbObject Data);
void WriteResponse(HttpResponseCode ResponseCode, CbArray Array);
void WriteResponse(HttpResponseCode ResponseCode, CbPackage Package);
void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::string_view ResponseString);
void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, IoBuffer Blob);
+ void WriteResponse(HttpContentType ContentType, const IoBuffer& Data, const HttpRanges& Ranges);
virtual void WriteResponseAsync(std::function<void(HttpServerRequest&)>&& ContinuationHandler) = 0;
@@ -152,6 +154,8 @@ protected:
std::string_view m_QueryString;
mutable uint32_t m_RequestId = ~uint32_t(0);
mutable Oid m_SessionId = Oid::Zero;
+ std::string m_ContentTypeOverride;
+ std::string m_ContentRangeHeader;
inline void SetIsHandled() { m_Flags |= kIsHandled; }
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 6cda84875..cfba3c95f 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -625,6 +625,8 @@ public:
void SetAllowZeroCopyFileSend(bool Allow) { m_AllowZeroCopyFileSend = Allow; }
void SetKeepAlive(bool KeepAlive) { m_IsKeepAlive = KeepAlive; }
+ void SetContentTypeOverride(std::string Override) { m_ContentTypeOverride = std::move(Override); }
+ void SetContentRangeHeader(std::string V) { m_ContentRangeHeader = std::move(V); }
/**
* Initialize the response for sending a payload made up of multiple blobs
@@ -768,10 +770,18 @@ public:
{
ZEN_MEMSCOPE(GetHttpasioTag());
+ std::string_view ContentTypeStr =
+ m_ContentTypeOverride.empty() ? MapContentTypeToString(m_ContentType) : std::string_view(m_ContentTypeOverride);
+
m_Headers << "HTTP/1.1 " << ResponseCode() << " " << ReasonStringForHttpResultCode(ResponseCode()) << "\r\n"
- << "Content-Type: " << MapContentTypeToString(m_ContentType) << "\r\n"
+ << "Content-Type: " << ContentTypeStr << "\r\n"
<< "Content-Length: " << ContentLength() << "\r\n"sv;
+ if (!m_ContentRangeHeader.empty())
+ {
+ m_Headers << "Content-Range: " << m_ContentRangeHeader << "\r\n"sv;
+ }
+
if (!m_IsKeepAlive)
{
m_Headers << "Connection: close\r\n"sv;
@@ -898,7 +908,9 @@ private:
bool m_AllowZeroCopyFileSend = true;
State m_State = State::kUninitialized;
HttpContentType m_ContentType = HttpContentType::kBinary;
- uint64_t m_ContentLength = 0;
+ std::string m_ContentTypeOverride;
+ std::string m_ContentRangeHeader;
+ uint64_t m_ContentLength = 0;
eastl::fixed_vector<IoBuffer, 8> m_DataBuffers; // This is here to keep the IoBuffer buffers/handles alive
ExtendableStringBuilder<160> m_Headers;
@@ -2131,6 +2143,10 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode)
m_Response.reset(new HttpResponse(HttpContentType::kBinary, m_RequestNumber));
m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend);
m_Response->SetKeepAlive(m_Request.IsKeepAlive());
+ if (!m_ContentRangeHeader.empty())
+ {
+ m_Response->SetContentRangeHeader(std::move(m_ContentRangeHeader));
+ }
std::array<IoBuffer, 0> Empty;
m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty);
@@ -2146,6 +2162,14 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT
m_Response.reset(new HttpResponse(ContentType, m_RequestNumber));
m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend);
m_Response->SetKeepAlive(m_Request.IsKeepAlive());
+ if (!m_ContentTypeOverride.empty())
+ {
+ m_Response->SetContentTypeOverride(std::move(m_ContentTypeOverride));
+ }
+ if (!m_ContentRangeHeader.empty())
+ {
+ m_Response->SetContentRangeHeader(std::move(m_ContentRangeHeader));
+ }
m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs);
}
diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp
index 31b0315d4..b0fb020e0 100644
--- a/src/zenhttp/servers/httpplugin.cpp
+++ b/src/zenhttp/servers/httpplugin.cpp
@@ -185,13 +185,17 @@ public:
const std::vector<IoBuffer>& ResponseBuffers() const { return m_ResponseBuffers; }
void SuppressPayload() { m_ResponseBuffers.resize(1); }
+ void SetContentTypeOverride(std::string Override) { m_ContentTypeOverride = std::move(Override); }
+ void SetContentRangeHeader(std::string V) { m_ContentRangeHeader = std::move(V); }
std::string_view GetHeaders();
private:
- uint16_t m_ResponseCode = 0;
- bool m_IsKeepAlive = true;
- HttpContentType m_ContentType = HttpContentType::kBinary;
+ uint16_t m_ResponseCode = 0;
+ bool m_IsKeepAlive = true;
+ HttpContentType m_ContentType = HttpContentType::kBinary;
+ std::string m_ContentTypeOverride;
+ std::string m_ContentRangeHeader;
uint64_t m_ContentLength = 0;
std::vector<IoBuffer> m_ResponseBuffers;
ExtendableStringBuilder<160> m_Headers;
@@ -246,10 +250,18 @@ HttpPluginResponse::GetHeaders()
if (m_Headers.Size() == 0)
{
+ std::string_view ContentTypeStr =
+ m_ContentTypeOverride.empty() ? MapContentTypeToString(m_ContentType) : std::string_view(m_ContentTypeOverride);
+
m_Headers << "HTTP/1.1 " << ResponseCode() << " " << ReasonStringForHttpResultCode(ResponseCode()) << "\r\n"
- << "Content-Type: " << MapContentTypeToString(m_ContentType) << "\r\n"
+ << "Content-Type: " << ContentTypeStr << "\r\n"
<< "Content-Length: " << ContentLength() << "\r\n"sv;
+ if (!m_ContentRangeHeader.empty())
+ {
+ m_Headers << "Content-Range: " << m_ContentRangeHeader << "\r\n"sv;
+ }
+
if (!m_IsKeepAlive)
{
m_Headers << "Connection: close\r\n"sv;
@@ -669,6 +681,10 @@ HttpPluginServerRequest::WriteResponse(HttpResponseCode ResponseCode)
ZEN_MEMSCOPE(GetHttppluginTag());
m_Response.reset(new HttpPluginResponse(HttpContentType::kBinary));
+ if (!m_ContentRangeHeader.empty())
+ {
+ m_Response->SetContentRangeHeader(std::move(m_ContentRangeHeader));
+ }
std::array<IoBuffer, 0> Empty;
m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty);
@@ -681,6 +697,14 @@ HttpPluginServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpConten
ZEN_MEMSCOPE(GetHttppluginTag());
m_Response.reset(new HttpPluginResponse(ContentType));
+ if (!m_ContentTypeOverride.empty())
+ {
+ m_Response->SetContentTypeOverride(std::move(m_ContentTypeOverride));
+ }
+ if (!m_ContentRangeHeader.empty())
+ {
+ m_Response->SetContentRangeHeader(std::move(m_ContentRangeHeader));
+ }
m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs);
}
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 1b722940d..d45804c50 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -464,6 +464,8 @@ public:
inline int64_t GetResponseBodySize() const { return m_TotalDataSize; }
void SetLocationHeader(std::string_view Location) { m_LocationHeader = Location; }
+ void SetContentTypeOverride(std::string Override) { m_ContentTypeOverride = std::move(Override); }
+ void SetContentRangeHeader(std::string V) { m_ContentRangeHeader = std::move(V); }
private:
eastl::fixed_vector<HTTP_DATA_CHUNK, 16> m_HttpDataChunks;
@@ -473,6 +475,8 @@ private:
uint32_t m_RemainingChunkCount = 0; // Backlog for multi-call sends
bool m_IsInitialResponse = true;
HttpContentType m_ContentType = HttpContentType::kBinary;
+ std::string m_ContentTypeOverride;
+ std::string m_ContentRangeHeader;
eastl::fixed_vector<IoBuffer, 16> m_DataBuffers;
std::string m_LocationHeader;
@@ -725,7 +729,8 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
PHTTP_KNOWN_HEADER ContentTypeHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentType];
- std::string_view ContentTypeString = MapContentTypeToString(m_ContentType);
+ std::string_view ContentTypeString =
+ m_ContentTypeOverride.empty() ? MapContentTypeToString(m_ContentType) : std::string_view(m_ContentTypeOverride);
ContentTypeHeader->pRawValue = ContentTypeString.data();
ContentTypeHeader->RawValueLength = (USHORT)ContentTypeString.size();
@@ -739,6 +744,15 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
LocationHeader->RawValueLength = (USHORT)m_LocationHeader.size();
}
+ // Content-Range header (for 206 Partial Content single-range responses)
+
+ if (!m_ContentRangeHeader.empty())
+ {
+ PHTTP_KNOWN_HEADER ContentRangeHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentRange];
+ ContentRangeHeader->pRawValue = m_ContentRangeHeader.data();
+ ContentRangeHeader->RawValueLength = (USHORT)m_ContentRangeHeader.size();
+ }
+
std::string_view ReasonString = ReasonStringForHttpResultCode(m_ResponseCode);
HttpResponse.StatusCode = m_ResponseCode;
@@ -2279,6 +2293,11 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode)
HttpMessageResponseRequest* Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode);
+ if (!m_ContentRangeHeader.empty())
+ {
+ Response->SetContentRangeHeader(std::move(m_ContentRangeHeader));
+ }
+
if (SuppressBody())
{
Response->SuppressResponseBody();
@@ -2307,6 +2326,15 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy
HttpMessageResponseRequest* Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode, ContentType, Blobs);
+ if (!m_ContentTypeOverride.empty())
+ {
+ Response->SetContentTypeOverride(std::move(m_ContentTypeOverride));
+ }
+ if (!m_ContentRangeHeader.empty())
+ {
+ Response->SetContentRangeHeader(std::move(m_ContentRangeHeader));
+ }
+
if (SuppressBody())
{
Response->SuppressResponseBody();
diff --git a/src/zenserver-test/buildstore-tests.cpp b/src/zenserver-test/buildstore-tests.cpp
index 28fc4d7ea..1f2157993 100644
--- a/src/zenserver-test/buildstore-tests.cpp
+++ b/src/zenserver-test/buildstore-tests.cpp
@@ -183,6 +183,60 @@ TEST_CASE("buildstore.blobs")
CHECK(ActualRange.EqualBytes(RangeView));
}
+ {
+ // GET blob not found
+ IoHash FakeHash = IoHash::HashBuffer("nonexistent", 11);
+ HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, FakeHash));
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+ }
+
+ {
+ // GET with out-of-bounds range
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+ uint64_t BlobSize = CompressedBlobsSizes.front();
+
+ HttpClient::KeyValueMap Headers;
+ Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", BlobSize + 100, BlobSize + 200)});
+
+ HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), Headers);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::RangeNotSatisfiable);
+ }
+
+ {
+ // GET with multi-range header (uses Download for multipart boundary parsing)
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+ uint64_t BlobSize = CompressedBlobsSizes.front();
+
+ uint64_t Range1Start = 0;
+ uint64_t Range1End = BlobSize / 4 - 1;
+ uint64_t Range2Start = BlobSize / 2;
+ uint64_t Range2End = BlobSize / 2 + BlobSize / 4 - 1;
+
+ HttpClient::KeyValueMap Headers;
+ Headers.Entries.insert({"Range", fmt::format("bytes={}-{},{}-{}", Range1Start, Range1End, Range2Start, Range2End)});
+
+ HttpClient::Response Result =
+ Client.Download(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), SystemRootPath, Headers);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::PartialContent);
+ REQUIRE_EQ(Result.Ranges.size(), 2);
+
+ HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCompressedBinary));
+ REQUIRE(FullBlobResult);
+
+ uint64_t Range1Len = Range1End - Range1Start + 1;
+ uint64_t Range2Len = Range2End - Range2Start + 1;
+
+ MemoryView ExpectedRange1 = FullBlobResult.ResponsePayload.GetView().Mid(Range1Start, Range1Len);
+ MemoryView ExpectedRange2 = FullBlobResult.ResponsePayload.GetView().Mid(Range2Start, Range2Len);
+
+ MemoryView ActualRange1 = Result.ResponsePayload.GetView().Mid(Result.Ranges[0].OffsetInPayload, Range1Len);
+ MemoryView ActualRange2 = Result.ResponsePayload.GetView().Mid(Result.Ranges[1].OffsetInPayload, Range2Len);
+
+ CHECK(ExpectedRange1.EqualBytes(ActualRange1));
+ CHECK(ExpectedRange2.EqualBytes(ActualRange2));
+ }
+
// Single-range Post
{
uint64_t RangeSizeSum = 0;
@@ -248,6 +302,94 @@ TEST_CASE("buildstore.blobs")
}
}
+ {
+ // POST with wrong accept type
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ Writer.BeginObject();
+ Writer.AddInteger("offset"sv, uint64_t(0));
+ Writer.AddInteger("length"sv, uint64_t(10));
+ Writer.EndObject();
+ Writer.EndArray();
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ Writer.Save(),
+ HttpClient::Accept(ZenContentType::kBinary));
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+ }
+
+ {
+ // POST with missing payload
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCbPackage));
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+ }
+
+ {
+ // POST with empty ranges array
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ Writer.EndArray();
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ Writer.Save(),
+ HttpClient::Accept(ZenContentType::kCbPackage));
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+ }
+
+ {
+ // POST with range count exceeding maximum
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ for (uint32_t I = 0; I < 257; I++)
+ {
+ Writer.BeginObject();
+ Writer.AddInteger("offset"sv, uint64_t(0));
+ Writer.AddInteger("length"sv, uint64_t(1));
+ Writer.EndObject();
+ }
+ Writer.EndArray();
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ Writer.Save(),
+ HttpClient::Accept(ZenContentType::kCbPackage));
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+ }
+
+ {
+ // POST with out-of-bounds range returns length=0
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+ uint64_t BlobSize = CompressedBlobsSizes.front();
+
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ Writer.BeginObject();
+ Writer.AddInteger("offset"sv, BlobSize + 100);
+ Writer.AddInteger("length"sv, uint64_t(50));
+ Writer.EndObject();
+ Writer.EndArray();
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ Writer.Save(),
+ HttpClient::Accept(ZenContentType::kCbPackage));
+ REQUIRE(Result);
+ CbPackage ResponsePackage = ParsePackageMessage(Result.ResponsePayload);
+ CbObjectView ResponseObject = ResponsePackage.GetObject();
+ CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView();
+ REQUIRE_EQ(RangeArray.Num(), uint64_t(1));
+ CbObjectView Range = (*begin(RangeArray)).AsObjectView();
+ CHECK_EQ(Range["offset"sv].AsUInt64(), BlobSize + 100);
+ CHECK_EQ(Range["length"sv].AsUInt64(), uint64_t(0));
+ }
+
// Multi-range
{
uint64_t RangeSizeSum = 0;
diff --git a/src/zenserver-test/objectstore-tests.cpp b/src/zenserver-test/objectstore-tests.cpp
index 2139d4d3a..99c92e15f 100644
--- a/src/zenserver-test/objectstore-tests.cpp
+++ b/src/zenserver-test/objectstore-tests.cpp
@@ -201,19 +201,108 @@ TEST_CASE("objectstore")
CHECK(Result.ResponsePayload.GetView().EqualBytes(BlobView.Mid(1000, 24)));
}
- // Multiple ranges: not supported, falls back to 200 with full body per RFC 7233
+ // Multiple ranges: bytes 0-49 and 100-149
{
HttpClient::Response Result = Client.Get(ObjectPath, {{"Range", "bytes=0-49,100-149"}});
- CHECK(Result.StatusCode == HttpResponseCode::OK);
- CHECK_EQ(Result.ResponsePayload.GetSize(), 1024u);
- CHECK(Result.ResponsePayload.GetView().EqualBytes(BlobView));
+ CHECK(Result.StatusCode == HttpResponseCode::PartialContent);
+
+ std::string_view Body(reinterpret_cast<const char*>(Result.ResponsePayload.GetData()), Result.ResponsePayload.GetSize());
+
+ // Verify multipart structure contains both range payloads
+ CHECK(Body.find("Content-Range: bytes 0-49/1024") != std::string_view::npos);
+ CHECK(Body.find("Content-Range: bytes 100-149/1024") != std::string_view::npos);
+
+ // Extract and verify actual data for first range
+ auto FindPartData = [&](std::string_view ContentRange) -> std::string_view {
+ size_t Pos = Body.find(ContentRange);
+ if (Pos == std::string_view::npos)
+ {
+ return {};
+ }
+ // Skip past the Content-Range line and the blank line separator
+ Pos = Body.find("\r\n\r\n", Pos);
+ if (Pos == std::string_view::npos)
+ {
+ return {};
+ }
+ Pos += 4;
+ size_t End = Body.find("\r\n--", Pos);
+ if (End == std::string_view::npos)
+ {
+ return {};
+ }
+ return Body.substr(Pos, End - Pos);
+ };
+
+ std::string_view Part1 = FindPartData("Content-Range: bytes 0-49/1024");
+ CHECK_EQ(Part1.size(), 50u);
+ CHECK(MemoryView(Part1.data(), Part1.size()).EqualBytes(BlobView.Mid(0, 50)));
+
+ std::string_view Part2 = FindPartData("Content-Range: bytes 100-149/1024");
+ CHECK_EQ(Part2.size(), 50u);
+ CHECK(MemoryView(Part2.data(), Part2.size()).EqualBytes(BlobView.Mid(100, 50)));
}
- // Out-of-bounds range: should return 400
+ // Out-of-bounds single range
{
HttpClient::Response Result = Client.Get(ObjectPath, {{"Range", "bytes=2000-2099"}});
- CHECK(Result.StatusCode == HttpResponseCode::BadRequest);
+ CHECK(Result.StatusCode == HttpResponseCode::RangeNotSatisfiable);
}
+
+ // Out-of-bounds multi-range
+ {
+ HttpClient::Response Result = Client.Get(ObjectPath, {{"Range", "bytes=0-49,2000-2099"}});
+ CHECK(Result.StatusCode == HttpResponseCode::RangeNotSatisfiable);
+ }
+ }
+}
+
+TEST_CASE("objectstore.range-requests-download")
+{
+ ZenServerInstance Instance(TestEnv);
+ const uint16_t Port = Instance.SpawnServerAndWaitUntilReady("--objectstore-enabled");
+ REQUIRE(Port != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/obj/");
+
+ IoBuffer Blob = CreateRandomBlob(1024);
+ MemoryView BlobView = Blob.GetView();
+ std::string ObjectPath = "bucket/bkt/range-download-test/data.bin";
+
+ HttpClient::Response PutResult = Client.Put(ObjectPath, IoBuffer(Blob));
+ REQUIRE(PutResult);
+
+ ScopedTemporaryDirectory DownloadDir;
+
+ // Single range via Download: verify Ranges is populated and GetRanges maps correctly
+ {
+ HttpClient::Response Result = Client.Download(ObjectPath, DownloadDir.Path(), {{"Range", "bytes=100-199"}});
+ CHECK(Result.StatusCode == HttpResponseCode::PartialContent);
+ REQUIRE_EQ(Result.Ranges.size(), 1u);
+ CHECK_EQ(Result.Ranges[0].RangeOffset, 100u);
+ CHECK_EQ(Result.Ranges[0].RangeLength, 100u);
+
+ std::vector<std::pair<uint64_t, uint64_t>> RequestedRanges = {{100, 100}};
+ std::vector<std::pair<uint64_t, uint64_t>> PayloadRanges = Result.GetRanges(RequestedRanges);
+ REQUIRE_EQ(PayloadRanges.size(), 1u);
+ CHECK(Result.ResponsePayload.GetView().Mid(PayloadRanges[0].first, PayloadRanges[0].second).EqualBytes(BlobView.Mid(100, 100)));
+ }
+
+ // Multi-range via Download: verify Ranges is populated for both parts and GetRanges maps correctly
+ {
+ HttpClient::Response Result = Client.Download(ObjectPath, DownloadDir.Path(), {{"Range", "bytes=0-49,100-149"}});
+ CHECK(Result.StatusCode == HttpResponseCode::PartialContent);
+ REQUIRE_EQ(Result.Ranges.size(), 2u);
+ CHECK_EQ(Result.Ranges[0].RangeOffset, 0u);
+ CHECK_EQ(Result.Ranges[0].RangeLength, 50u);
+ CHECK_EQ(Result.Ranges[1].RangeOffset, 100u);
+ CHECK_EQ(Result.Ranges[1].RangeLength, 50u);
+
+ std::vector<std::pair<uint64_t, uint64_t>> RequestedRanges = {{0, 50}, {100, 50}};
+ std::vector<std::pair<uint64_t, uint64_t>> PayloadRanges = Result.GetRanges(RequestedRanges);
+ REQUIRE_EQ(PayloadRanges.size(), 2u);
+ CHECK(Result.ResponsePayload.GetView().Mid(PayloadRanges[0].first, PayloadRanges[0].second).EqualBytes(BlobView.Mid(0, 50)));
+ CHECK(Result.ResponsePayload.GetView().Mid(PayloadRanges[1].first, PayloadRanges[1].second).EqualBytes(BlobView.Mid(100, 50)));
}
}
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index bbbb0c37b..f935e2c6b 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -162,96 +162,81 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
fmt::format("Invalid blob hash '{}'", Hash));
}
- std::vector<std::pair<uint64_t, uint64_t>> OffsetAndLengthPairs;
+ m_BuildStoreStats.BlobReadCount++;
+ IoBuffer Blob = m_BuildStore.GetBlob(BlobHash);
+ if (!Blob)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, fmt::format("Blob {} not found", Hash));
+ }
+ m_BuildStoreStats.BlobHitCount++;
+
if (ServerRequest.RequestVerb() == HttpVerb::kPost)
{
+ if (ServerRequest.AcceptContentType() != HttpContentType::kCbPackage)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Accept type '{}' is not supported for blob {}, expected '{}'",
+ ToString(ServerRequest.AcceptContentType()),
+ Hash,
+ ToString(HttpContentType::kCbPackage)));
+ }
+
CbObject RangePayload = ServerRequest.ReadPayloadObject();
- if (RangePayload)
+ if (!RangePayload)
{
- CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView();
- OffsetAndLengthPairs.reserve(RangesArray.Num());
- for (CbFieldView FieldView : RangesArray)
- {
- CbObjectView RangeView = FieldView.AsObjectView();
- uint64_t RangeOffset = RangeView["offset"sv].AsUInt64();
- uint64_t RangeLength = RangeView["length"sv].AsUInt64();
- OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength));
- }
- if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported)
- {
- return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}",
- OffsetAndLengthPairs.size(),
- MaxRangeCountPerRequestSupported));
- }
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Missing payload for range request on blob {}", BlobHash));
}
- if (OffsetAndLengthPairs.empty())
+
+ CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView();
+ const uint64_t RangeCount = RangesArray.Num();
+ if (RangeCount == 0)
{
m_BuildStoreStats.BadRequestCount++;
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
HttpContentType::kText,
- "Fetching blob without ranges must be done with the GET verb");
+ "POST request must include a non-empty 'ranges' array");
}
- }
- else
- {
- HttpRanges Ranges;
- bool HasRange = ServerRequest.TryGetRanges(Ranges);
- if (HasRange)
+ if (RangeCount > MaxRangeCountPerRequestSupported)
{
- if (Ranges.size() > 1)
- {
- // Only a single http range is supported, we have limited support for http multirange responses
- m_BuildStoreStats.BadRequestCount++;
- return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Multiple ranges in blob request is only supported for {} accept type",
- ToString(HttpContentType::kCbPackage)));
- }
- const HttpRange& FirstRange = Ranges.front();
- OffsetAndLengthPairs.push_back(std::make_pair<uint64_t, uint64_t>(FirstRange.Start, FirstRange.End - FirstRange.Start + 1));
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Range count {} exceeds maximum of {}", RangeCount, MaxRangeCountPerRequestSupported));
}
- }
-
- m_BuildStoreStats.BlobReadCount++;
- IoBuffer Blob = m_BuildStore.GetBlob(BlobHash);
- if (!Blob)
- {
- return ServerRequest.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("Blob with hash '{}' could not be found", Hash));
- }
- m_BuildStoreStats.BlobHitCount++;
- if (OffsetAndLengthPairs.empty())
- {
- return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
- }
+ const uint64_t BlobSize = Blob.GetSize();
+ std::vector<IoBuffer> RangeBuffers;
+ RangeBuffers.reserve(RangeCount);
- if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage)
- {
- const uint64_t BlobSize = Blob.GetSize();
+ CbPackage ResponsePackage;
+ CbObjectWriter Writer;
- CbPackage ResponsePackage;
- std::vector<IoBuffer> RangeBuffers;
- CbObjectWriter Writer;
Writer.BeginArray("ranges"sv);
- for (const std::pair<uint64_t, uint64_t>& Range : OffsetAndLengthPairs)
+ for (CbFieldView FieldView : RangesArray)
{
- const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0;
- const uint64_t RangeSize = Min(Range.second, MaxBlobSize);
+ CbObjectView RangeView = FieldView.AsObjectView();
+ uint64_t RangeOffset = RangeView["offset"sv].AsUInt64();
+ uint64_t RangeLength = RangeView["length"sv].AsUInt64();
+
+ const uint64_t MaxBlobSize = RangeOffset < BlobSize ? BlobSize - RangeOffset : 0;
+ const uint64_t RangeSize = Min(RangeLength, MaxBlobSize);
Writer.BeginObject();
{
- if (Range.first + RangeSize <= BlobSize)
+ if (RangeOffset + RangeSize <= BlobSize)
{
- RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize));
- Writer.AddInteger("offset"sv, Range.first);
+ RangeBuffers.push_back(IoBuffer(Blob, RangeOffset, RangeSize));
+ Writer.AddInteger("offset"sv, RangeOffset);
Writer.AddInteger("length"sv, RangeSize);
}
else
{
- Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("offset"sv, RangeOffset);
Writer.AddInteger("length"sv, 0);
}
}
@@ -259,7 +244,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
}
Writer.EndArray();
- CompositeBuffer Ranges(RangeBuffers);
+ CompositeBuffer Ranges(std::move(RangeBuffers));
CbAttachment PayloadAttachment(std::move(Ranges), BlobHash);
Writer.AddAttachment("payload", PayloadAttachment);
@@ -269,32 +254,21 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
ResponsePackage.SetObject(HeaderObject);
CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
- uint64_t ResponseSize = RpcResponseBuffer.GetSize();
- ZEN_UNUSED(ResponseSize);
return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
}
else
{
- if (OffsetAndLengthPairs.size() != 1)
+ HttpRanges RequestedRangeHeader;
+ bool HasRange = ServerRequest.TryGetRanges(RequestedRangeHeader);
+ if (HasRange)
{
- // Only a single http range is supported, we have limited support for http multirange responses
- m_BuildStoreStats.BadRequestCount++;
- return ServerRequest.WriteResponse(
- HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage)));
+ // Standard HTTP GET with Range header: framework handles 206, Content-Range, and 416 on OOB.
+ return ServerRequest.WriteResponse(HttpContentType::kBinary, Blob, RequestedRangeHeader);
}
-
- const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front();
- const uint64_t BlobSize = Blob.GetSize();
- const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0;
- const uint64_t RangeSize = Min(OffsetAndLength.second, MaxBlobSize);
- if (OffsetAndLength.first + RangeSize > BlobSize)
+ else
{
- return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
}
- Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize);
- return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
}
}
diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
index e34cd0445..bab9df06d 100644
--- a/src/zenserver/storage/objectstore/objectstore.cpp
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -637,11 +637,7 @@ HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_
}
HttpRanges Ranges;
- if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1)
- {
- // Multi-range is not supported, fall back to full response per RFC 7233
- Ranges.clear();
- }
+ Request.ServerRequest().TryGetRanges(Ranges);
FileContents File;
{
@@ -665,42 +661,34 @@ HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_
if (Ranges.empty())
{
- const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
-
+ const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.GetSize()) + FileBuf.GetSize();
ZEN_LOG_DEBUG(LogObj,
"GET - '{}/{}' ({}) [OK] (Served: {})",
BucketName,
RelativeBucketPath,
- NiceBytes(FileBuf.Size()),
+ NiceBytes(FileBuf.GetSize()),
NiceBytes(TotalServed));
-
- Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf);
}
- else
+ else if (Ranges.size() == 1)
{
- const auto Range = Ranges[0];
- const uint64_t RangeSize = 1 + (Range.End - Range.Start);
- const uint64_t TotalServed = m_TotalBytesServed.fetch_add(RangeSize) + RangeSize;
-
- ZEN_LOG_DEBUG(LogObj,
- "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
- BucketName,
- RelativeBucketPath,
- Range.Start,
- Range.End,
- NiceBytes(RangeSize),
- NiceBytes(FileBuf.Size()),
- NiceBytes(TotalServed));
-
- MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize);
- if (RangeView.GetSize() != RangeSize)
+ const uint64_t TotalSize = FileBuf.GetSize();
+ const uint64_t RangeEnd = (Ranges[0].End != ~uint64_t(0)) ? Ranges[0].End : TotalSize - 1;
+ if (RangeEnd < TotalSize && Ranges[0].Start <= RangeEnd)
{
- return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+ const uint64_t RangeSize = 1 + (RangeEnd - Ranges[0].Start);
+ const uint64_t TotalServed = m_TotalBytesServed.fetch_add(RangeSize) + RangeSize;
+ ZEN_LOG_DEBUG(LogObj,
+ "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
+ BucketName,
+ RelativeBucketPath,
+ Ranges[0].Start,
+ RangeEnd,
+ NiceBytes(RangeSize),
+ NiceBytes(TotalSize),
+ NiceBytes(TotalServed));
}
-
- IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize());
- Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf);
}
+ Request.ServerRequest().WriteResponse(HttpContentType::kBinary, FileBuf, Ranges);
}
void