aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver-test/buildstore-tests.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver-test/buildstore-tests.cpp')
-rw-r--r--src/zenserver-test/buildstore-tests.cpp220
1 files changed, 211 insertions, 9 deletions
diff --git a/src/zenserver-test/buildstore-tests.cpp b/src/zenserver-test/buildstore-tests.cpp
index 02b308485..cf9b10896 100644
--- a/src/zenserver-test/buildstore-tests.cpp
+++ b/src/zenserver-test/buildstore-tests.cpp
@@ -27,6 +27,8 @@ namespace zen::tests {
using namespace std::literals;
+TEST_SUITE_BEGIN("server.buildstore");
+
TEST_CASE("buildstore.blobs")
{
std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir();
@@ -36,7 +38,8 @@ TEST_CASE("buildstore.blobs")
std::string_view Bucket = "bkt"sv;
Oid BuildId = Oid::NewOid();
- std::vector<IoHash> CompressedBlobsHashes;
+ std::vector<IoHash> CompressedBlobsHashes;
+ std::vector<uint64_t> CompressedBlobsSizes;
{
ZenServerInstance Instance(TestEnv);
@@ -51,6 +54,7 @@ TEST_CASE("buildstore.blobs")
IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7);
CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+ CompressedBlobsSizes.push_back(CompressedBlob.GetCompressedSize());
IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
Payload.SetContentType(ZenContentType::kCompressedBinary);
@@ -107,6 +111,7 @@ TEST_CASE("buildstore.blobs")
IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7);
CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+ CompressedBlobsSizes.push_back(CompressedBlob.GetCompressedSize());
IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
Payload.SetContentType(ZenContentType::kCompressedBinary);
@@ -141,6 +146,201 @@ TEST_CASE("buildstore.blobs")
CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
}
}
+
+ {
+ // Single-range Get
+
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ {
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+ uint64_t BlobSize = CompressedBlobsSizes.front();
+
+ std::vector<std::pair<uint64_t, uint64_t>> Ranges = {{BlobSize / 16 * 1, BlobSize / 2}};
+
+ uint64_t RangeSizeSum = Ranges.front().second;
+
+ HttpClient::KeyValueMap Headers;
+
+ Headers.Entries.insert(
+ {"Range", fmt::format("bytes={}-{}", Ranges.front().first, Ranges.front().first + Ranges.front().second - 1)});
+
+ HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), Headers);
+ REQUIRE(Result);
+ IoBuffer Payload = Result.ResponsePayload;
+ CHECK_EQ(RangeSizeSum, Payload.GetSize());
+
+ HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCompressedBinary));
+ REQUIRE(FullBlobResult);
+ MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Ranges.front().first, Ranges.front().second);
+ MemoryView RangeView = Payload.GetView();
+ CHECK(ActualRange.EqualBytes(RangeView));
+ }
+ }
+
+ {
+ // Single-range Post
+
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ {
+ uint64_t RangeSizeSum = 0;
+
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+ uint64_t BlobSize = CompressedBlobsSizes.front();
+
+ std::vector<std::pair<uint64_t, uint64_t>> Ranges = {{BlobSize / 16 * 1, BlobSize / 2}};
+
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ {
+ for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+ {
+ Writer.BeginObject();
+ {
+ Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("length"sv, Range.second);
+ RangeSizeSum += Range.second;
+ }
+ Writer.EndObject();
+ }
+ }
+ Writer.EndArray(); // ranges
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ Writer.Save(),
+ HttpClient::Accept(ZenContentType::kCbPackage));
+ REQUIRE(Result);
+ IoBuffer Payload = Result.ResponsePayload;
+ REQUIRE(Payload.GetContentType() == ZenContentType::kCbPackage);
+
+ CbPackage ResponsePackage = ParsePackageMessage(Payload);
+ CbObjectView ResponseObject = ResponsePackage.GetObject();
+
+ CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView();
+ CHECK_EQ(RangeArray.Num(), Ranges.size());
+ size_t RangeOffset = 0;
+ for (CbFieldView View : RangeArray)
+ {
+ CbObjectView Range = View.AsObjectView();
+ CHECK_EQ(Range["offset"sv].AsUInt64(), Ranges[RangeOffset].first);
+ CHECK_EQ(Range["length"sv].AsUInt64(), Ranges[RangeOffset].second);
+ RangeOffset++;
+ }
+
+ const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash);
+ REQUIRE(DataAttachment);
+ SharedBuffer PayloadRanges = DataAttachment->AsBinary();
+ CHECK_EQ(RangeSizeSum, PayloadRanges.GetSize());
+
+ HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCompressedBinary));
+ REQUIRE(FullBlobResult);
+
+ uint64_t Offset = 0;
+ for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+ {
+ MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Range.first, Range.second);
+ MemoryView RangeView = PayloadRanges.GetView().Mid(Offset, Range.second);
+ CHECK(ActualRange.EqualBytes(RangeView));
+ Offset += Range.second;
+ }
+ }
+ }
+
+ {
+ // Multi-range
+
+ ZenServerInstance Instance(TestEnv);
+
+ const uint16_t PortNumber =
+ Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath));
+ CHECK(PortNumber != 0);
+
+ HttpClient Client(Instance.GetBaseUri() + "/builds/");
+
+ {
+ uint64_t RangeSizeSum = 0;
+
+ const IoHash& RawHash = CompressedBlobsHashes.front();
+ uint64_t BlobSize = CompressedBlobsSizes.front();
+
+ std::vector<std::pair<uint64_t, uint64_t>> Ranges = {
+ {BlobSize / 16 * 1, BlobSize / 20},
+ {BlobSize / 16 * 3, BlobSize / 32},
+ {BlobSize / 16 * 5, BlobSize / 16},
+ {BlobSize - BlobSize / 16, BlobSize / 16 - 1},
+ };
+
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ {
+ for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+ {
+ Writer.BeginObject();
+ {
+ Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("length"sv, Range.second);
+ RangeSizeSum += Range.second;
+ }
+ Writer.EndObject();
+ }
+ }
+ Writer.EndArray(); // ranges
+
+ HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ Writer.Save(),
+ HttpClient::Accept(ZenContentType::kCbPackage));
+ REQUIRE(Result);
+ IoBuffer Payload = Result.ResponsePayload;
+ REQUIRE(Payload.GetContentType() == ZenContentType::kCbPackage);
+
+ CbPackage ResponsePackage = ParsePackageMessage(Payload);
+ CbObjectView ResponseObject = ResponsePackage.GetObject();
+
+ CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView();
+ CHECK_EQ(RangeArray.Num(), Ranges.size());
+ size_t RangeOffset = 0;
+ for (CbFieldView View : RangeArray)
+ {
+ CbObjectView Range = View.AsObjectView();
+ CHECK_EQ(Range["offset"sv].AsUInt64(), Ranges[RangeOffset].first);
+ CHECK_EQ(Range["length"sv].AsUInt64(), Ranges[RangeOffset].second);
+ RangeOffset++;
+ }
+
+ const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash);
+ REQUIRE(DataAttachment);
+ SharedBuffer PayloadRanges = DataAttachment->AsBinary();
+ CHECK_EQ(RangeSizeSum, PayloadRanges.GetSize());
+
+ HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash),
+ HttpClient::Accept(ZenContentType::kCompressedBinary));
+ REQUIRE(FullBlobResult);
+
+ uint64_t Offset = 0;
+ for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+ {
+ MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Range.first, Range.second);
+ MemoryView RangeView = PayloadRanges.GetView().Mid(Offset, Range.second);
+ CHECK(ActualRange.EqualBytes(RangeView));
+ Offset += Range.second;
+ }
+ }
+ }
}
namespace {
@@ -191,7 +391,7 @@ TEST_CASE("buildstore.metadata")
HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/getBlobMetadata", Namespace, Bucket, BuildId),
Payload,
HttpClient::Accept(ZenContentType::kCbObject));
- CHECK(Result);
+ REQUIRE(Result);
std::vector<CbObject> ResultMetadatas;
@@ -372,7 +572,7 @@ TEST_CASE("buildstore.cache")
{
std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
- CHECK(Exists.size() == BlobHashes.size());
+ REQUIRE(Exists.size() == BlobHashes.size());
for (size_t I = 0; I < BlobCount; I++)
{
CHECK(Exists[I].HasBody);
@@ -411,7 +611,7 @@ TEST_CASE("buildstore.cache")
{
std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
- CHECK(Exists.size() == BlobHashes.size());
+ REQUIRE(Exists.size() == BlobHashes.size());
for (size_t I = 0; I < BlobCount; I++)
{
CHECK(Exists[I].HasBody);
@@ -419,7 +619,7 @@ TEST_CASE("buildstore.cache")
}
std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
- CHECK_EQ(BlobCount, FetchedMetadatas.size());
+ REQUIRE_EQ(BlobCount, FetchedMetadatas.size());
for (size_t I = 0; I < BlobCount; I++)
{
@@ -440,7 +640,7 @@ TEST_CASE("buildstore.cache")
{
std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
- CHECK(Exists.size() == BlobHashes.size());
+ REQUIRE(Exists.size() == BlobHashes.size());
for (size_t I = 0; I < BlobCount * 2; I++)
{
CHECK(Exists[I].HasBody);
@@ -451,7 +651,7 @@ TEST_CASE("buildstore.cache")
CHECK_EQ(BlobCount, MetaDatas.size());
std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
- CHECK_EQ(BlobCount, FetchedMetadatas.size());
+ REQUIRE_EQ(BlobCount, FetchedMetadatas.size());
for (size_t I = 0; I < BlobCount; I++)
{
@@ -474,7 +674,7 @@ TEST_CASE("buildstore.cache")
CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, GetTinyWorkerPool(EWorkloadType::Background)));
std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes);
- CHECK(Exists.size() == BlobHashes.size());
+ REQUIRE(Exists.size() == BlobHashes.size());
for (size_t I = 0; I < BlobCount * 2; I++)
{
CHECK(Exists[I].HasBody);
@@ -493,7 +693,7 @@ TEST_CASE("buildstore.cache")
CHECK_EQ(BlobCount, MetaDatas.size());
std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes);
- CHECK_EQ(BlobCount, FetchedMetadatas.size());
+ REQUIRE_EQ(BlobCount, FetchedMetadatas.size());
for (size_t I = 0; I < BlobCount; I++)
{
@@ -502,5 +702,7 @@ TEST_CASE("buildstore.cache")
}
}
+TEST_SUITE_END();
+
} // namespace zen::tests
#endif
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-18 00:36:44 +0000