diff options
| author | Dan Engelbrecht <[email protected]> | 2022-06-30 02:16:51 -0700 |
|---|---|---|
| committer | GitHub <[email protected]> | 2022-06-30 02:16:51 -0700 |
| commit | 0b2fcba5a82a6d8041d597c22f7b8a095fd4de4e (patch) | |
| tree | 105622a078d60fd6e1dcf7c400fe385561cb0460 /zenserver-test/zenserver-test.cpp | |
| parent | Add ability to control where in the result array results go in GetCacheRecord... (diff) | |
| download | zen-0b2fcba5a82a6d8041d597c22f7b8a095fd4de4e.tar.xz zen-0b2fcba5a82a6d8041d597c22f7b8a095fd4de4e.zip | |
Use cacherequest::* for zenserver tests (#135)
* use cacherequests in zcache.rpc
* use cacherequests for zcache.rpc.allpolicies
Diffstat (limited to 'zenserver-test/zenserver-test.cpp')
| -rw-r--r-- | zenserver-test/zenserver-test.cpp | 563 |
1 files changed, 162 insertions, 401 deletions
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index ad621028f..a61c16916 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1301,12 +1301,10 @@ TEST_CASE("zcache.rpc") { using namespace std::literals; - auto AppendCacheRecord = [](CbPackage& Package, - CbWriter& Writer, - const zen::CacheKey& CacheKey, - size_t PayloadSize, - CachePolicy /* BatchDefaultPolicy */, - CachePolicy RecordPolicy) { + auto AppendCacheRecord = [](cacherequests::PutCacheRecordsRequest& Request, + const zen::CacheKey& CacheKey, + size_t PayloadSize, + CachePolicy RecordPolicy) { std::vector<uint8_t> Data; Data.resize(PayloadSize); for (size_t Idx = 0; Idx < PayloadSize; ++Idx) @@ -1314,50 +1312,15 @@ TEST_CASE("zcache.rpc") Data[Idx] = Idx % 255; } - CompressedBuffer Value = zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size())); - zen::CbAttachment Attachment(Value); - - Writer.BeginObject(); - { - Writer.BeginObject("Record"sv); - { - Writer.BeginObject("Key"sv); - { - Writer << "Bucket"sv << CacheKey.Bucket << "Hash"sv << CacheKey.Hash; - } - Writer.EndObject(); - Writer.BeginArray("Values"sv); - { - Writer.BeginObject(); - { - Writer.AddObjectId("Id"sv, Oid::NewOid()); - Writer.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(Value.GetRawHash())); - Writer.AddInteger("RawSize"sv, Value.GetRawSize()); - } - Writer.EndObject(); - } - Writer.EndArray(); - } - Writer.EndObject(); - Writer.SetName("Policy"sv); - CacheRecordPolicy(RecordPolicy).Save(Writer); - } - Writer.EndObject(); - - Package.AddAttachment(Attachment); - }; - - auto ToIoBuffer = [](zen::CbPackage Package) -> zen::IoBuffer { - zen::BinaryWriter MemStream; - Package.Save(MemStream); - return zen::IoBuffer(zen::IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + CompressedBuffer Value = zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size())); + Request.Requests.push_back({.Key = CacheKey, .Values = {{.Id = Oid::NewOid(), .Body = std::move(Value)}}, .Policy = RecordPolicy}); }; - auto PutCacheRecords = [&AppendCacheRecord, &ToIoBuffer](std::string_view BaseUri, - std::string_view Namespace, - std::string_view Bucket, - size_t Num, - size_t PayloadSize = 1024) -> std::vector<CacheKey> { + auto PutCacheRecords = [&AppendCacheRecord](std::string_view BaseUri, + std::string_view Namespace, + std::string_view Bucket, + size_t Num, + size_t PayloadSize = 1024) -> std::vector<CacheKey> { std::vector<zen::CacheKey> OutKeys; for (uint32_t Key = 1; Key <= Num; ++Key) @@ -1365,31 +1328,14 @@ TEST_CASE("zcache.rpc") zen::IoHash KeyHash; ((uint32_t*)(KeyHash.Hash))[0] = Key; const zen::CacheKey CacheKey = zen::CacheKey::Create(Bucket, KeyHash); - CbPackage Package; - CbWriter Writer; - - Writer.BeginObject(); - { - Writer << "Method"sv - << "PutCacheRecords"sv; - Writer.BeginObject("Params"sv); - { - CachePolicy BatchDefaultPolicy = CachePolicy::Default; - Writer << "DefaultPolicy"sv << WriteToString<128>(BatchDefaultPolicy); - Writer << "Namespace"sv << Namespace; - Writer.BeginArray("Requests"sv); - { - AppendCacheRecord(Package, Writer, CacheKey, PayloadSize, BatchDefaultPolicy, CachePolicy::Default); - } - Writer.EndArray(); - } - Writer.EndObject(); - } - Writer.EndObject(); - Package.SetObject(Writer.Save().AsObject()); + cacherequests::PutCacheRecordsRequest Request = {.Namespace = std::string(Namespace)}; + AppendCacheRecord(Request, CacheKey, PayloadSize, CachePolicy::Default); OutKeys.push_back(CacheKey); + CbPackage Package; + CHECK(Request.Format(Package)); + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, @@ -1403,44 +1349,27 @@ TEST_CASE("zcache.rpc") struct GetCacheRecordResult { - zen::CbPackage Response; - std::vector<zen::CbFieldView> Records; - bool Success; + zen::CbPackage Response; + cacherequests::GetCacheRecordsResult Result; + // std::vector<zen::CbFieldView> Records; + bool Success; }; auto GetCacheRecords = [](std::string_view BaseUri, std::string_view Namespace, std::span<zen::CacheKey> Keys, zen::CachePolicy Policy) -> GetCacheRecordResult { - using namespace zen; - - CbObjectWriter Request; - Request << "Method"sv - << "GetCacheRecords"sv; - Request.BeginObject("Params"sv); + cacherequests::GetCacheRecordsRequest Request = {.DefaultPolicy = Policy, .Namespace = std::string(Namespace)}; + for (const CacheKey& Key : Keys) { - Request << "DefaultPolicy"sv << WriteToString<128>(Policy); - Request << "Namespace"sv << Namespace; - - Request.BeginArray("Requests"sv); - for (const CacheKey& Key : Keys) - { - Request.BeginObject(); - { - Request.BeginObject("Key"sv); - { - Request << "Bucket"sv << Key.Bucket << "Hash"sv << Key.Hash; - } - Request.EndObject(); - } - Request.EndObject(); - } - Request.EndArray(); + Request.Requests.push_back({.Key = Key}); } - Request.EndObject(); + + CbObjectWriter RequestWriter; + CHECK(Request.Format(RequestWriter)); BinaryWriter Body; - Request.Save(Body); + RequestWriter.Save(Body); cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, cpr::Header{{"Content-Type", "application/x-ue-cb"}, {"Accept", "application/x-ue-cbpkg"}}, @@ -1453,14 +1382,8 @@ TEST_CASE("zcache.rpc") CbPackage Response; if (Response.TryLoad(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()))) { - OutResult.Response = std::move(Response); - CbObjectView ResponseObject = OutResult.Response.GetObject(); - - for (CbFieldView RecordView : ResponseObject["Result"]) - { - OutResult.Records.push_back(RecordView); - } - + OutResult.Response = std::move(Response); + CHECK(OutResult.Result.Parse(OutResult.Response)); OutResult.Success = true; } } @@ -1468,14 +1391,6 @@ TEST_CASE("zcache.rpc") return OutResult; }; - auto LoadKey = [](zen::CbFieldView KeyView) -> zen::CacheKey { - if (zen::CbObjectView KeyObj = KeyView.AsObjectView()) - { - return CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash()); - } - return CacheKey::Empty; - }; - SUBCASE("get cache records") { std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); @@ -1491,27 +1406,19 @@ TEST_CASE("zcache.rpc") std::vector<zen::CacheKey> Keys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128); GetCacheRecordResult Result = GetCacheRecords(BaseUri, "ue4.ddc"sv, Keys, Policy); - CHECK(Result.Records.size() == Keys.size()); + CHECK(Result.Result.Results.size() == Keys.size()); - for (size_t Index = 0; CbFieldView RecordView : Result.Records) + for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) { const CacheKey& ExpectedKey = Keys[Index++]; + CHECK(Record); + CHECK(Record->Key == ExpectedKey); + CHECK(Record->Values.size() == 1); - CbObjectView RecordObj = RecordView.AsObjectView(); - CbObjectView KeyObj = RecordObj["Key"sv].AsObjectView(); - const CacheKey Key = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash()); - IoHash AttachmentHash; - size_t NumValues = 0; - for (CbFieldView Value : RecordObj["Values"sv]) + for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values) { - AttachmentHash = Value.AsObjectView()["RawHash"sv].AsHash(); - ++NumValues; + CHECK(Value.Body); } - CHECK(NumValues == 1); - const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); - - CHECK(Key == ExpectedKey); - CHECK(Attachment != nullptr); } } @@ -1538,33 +1445,25 @@ TEST_CASE("zcache.rpc") GetCacheRecordResult Result = GetCacheRecords(BaseUri, "ue4.ddc"sv, Keys, Policy); - CHECK(Result.Records.size() == Keys.size()); + CHECK(Result.Result.Results.size() == Keys.size()); size_t KeyIndex = 0; - for (size_t Index = 0; CbFieldView RecordView : Result.Records) + for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) { const bool Missing = Index++ % 2 != 0; if (Missing) { - CHECK(RecordView.IsNull()); + CHECK(!Record); } else { const CacheKey& ExpectedKey = ExistingKeys[KeyIndex++]; - CbObjectView RecordObj = RecordView.AsObjectView(); - zen::CacheKey Key = LoadKey(RecordObj["Key"sv]); - IoHash AttachmentHash; - size_t NumValues = 0; - for (CbFieldView Value : RecordObj["Values"sv]) + CHECK(Record->Key == ExpectedKey); + for (const cacherequests::GetCacheRecordResultValue& Value : Record->Values) { - AttachmentHash = Value.AsObjectView()["RawHash"sv].AsHash(); - ++NumValues; + CHECK(Value.Body); } - CHECK(NumValues == 1); - const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); - CHECK(Key == ExpectedKey); - CHECK(Attachment != nullptr); } } } @@ -1586,11 +1485,11 @@ TEST_CASE("zcache.rpc") CachePolicy Policy = CachePolicy::QueryLocal; GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, "ue4.ddc"sv, Keys, Policy); - CHECK(Result.Records.size() == Keys.size()); + CHECK(Result.Result.Results.size() == Keys.size()); - for (CbFieldView RecordView : Result.Records) + for (const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) { - CHECK(RecordView.IsNull()); + CHECK(!Record); } } @@ -1611,14 +1510,13 @@ TEST_CASE("zcache.rpc") CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, "ue4.ddc"sv, Keys, Policy); - CHECK(Result.Records.size() == Keys.size()); + CHECK(Result.Result.Results.size() == Keys.size()); - for (size_t Index = 0; CbFieldView RecordView : Result.Response.GetObject()["Result"sv]) + for (size_t Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& Record : Result.Result.Results) { - const zen::CacheKey& ExpectedKey = Keys[Index++]; - CbObjectView RecordObj = RecordView.AsObjectView(); - zen::CacheKey Key = LoadKey(RecordObj["Key"sv]); - CHECK(Key == ExpectedKey); + CHECK(Record); + const CacheKey& ExpectedKey = Keys[Index++]; + CHECK(Record->Key == ExpectedKey); } } } @@ -1833,53 +1731,24 @@ TEST_CASE("zcache.rpc.allpolicies") // PutCacheRecords { - CbPackage Package; - CbObjectWriter Writer; - Writer << "Method"sv - << "PutCacheRecords"sv; - Writer.BeginObject("Params"sv); - { - CachePolicy BatchDefaultPolicy = CachePolicy::Default; - Writer << "DefaultPolicy"sv << WriteToString<128>(BatchDefaultPolicy); - Writer << "Namespace"sv << TestNamespace; - Writer.BeginArray("Requests"sv); - for (CachePutRequest& Request : PutRequests) + CachePolicy BatchDefaultPolicy = CachePolicy::Default; + cacherequests::PutCacheRecordsRequest Request = {.DefaultPolicy = BatchDefaultPolicy, .Namespace = std::string(TestNamespace)}; + Request.Requests.reserve(PutRequests.size()); + for (CachePutRequest& PutRequest : PutRequests) + { + cacherequests::PutCacheRecordRequest& RecordRequest = Request.Requests.emplace_back(); + RecordRequest.Key = PutRequest.Key; + RecordRequest.Policy = PutRequest.Policy; + RecordRequest.Values.reserve(NumValues); + for (int ValueIndex = 0; ValueIndex < NumValues; ++ValueIndex) { - Writer.BeginObject(); - { - Writer.BeginObject("Record"sv); - { - Writer.BeginObject("Key"sv); - { - Writer << "Bucket"sv << Request.Key.Bucket << "Hash"sv << Request.Key.Hash; - } - Writer.EndObject(); - Writer.BeginArray("Values"sv); - for (int ValueIndex = 0; ValueIndex < NumValues; ++ValueIndex) - { - Writer.BeginObject(); - { - CompressedBuffer Buffer = Request.Values->BufferValues[ValueIndex]; - Writer.AddObjectId("Id"sv, ValueIds[ValueIndex]); - Writer.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(Buffer.GetRawHash())); - Package.AddAttachment(CbAttachment(Buffer)); - Writer.AddInteger("RawSize"sv, Buffer.GetRawSize()); - } - Writer.EndObject(); - } - Writer.EndArray(); - } - Writer.EndObject(); - Writer.SetName("Policy"sv); - Request.Policy.Save(Writer); - } - Writer.EndObject(); - Request.Data->Data->ReceivedPut = true; + RecordRequest.Values.push_back({.Id = ValueIds[ValueIndex], .Body = PutRequest.Values->BufferValues[ValueIndex]}); } - Writer.EndArray(); + PutRequest.Data->Data->ReceivedPut = true; } - Writer.EndObject(); - Package.SetObject(Writer.Save()); + + CbPackage Package; + CHECK(Request.Format(Package)); IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, @@ -1889,37 +1758,19 @@ TEST_CASE("zcache.rpc.allpolicies") // PutCacheValues { - CbPackage Package; - CbObjectWriter Writer; - Writer << "Method"sv - << "PutCacheValues"sv; - Writer.BeginObject("Params"sv); - { - CachePolicy BatchDefaultPolicy = CachePolicy::Default; - Writer << "DefaultPolicy"sv << WriteToString<128>(BatchDefaultPolicy); - Writer << "Namespace"sv << TestNamespace; - Writer.BeginArray("Requests"sv); - for (CachePutValueRequest& Request : PutValueRequests) - { - Writer.BeginObject(); - { - Writer.BeginObject("Key"sv); - { - Writer << "Bucket"sv << Request.Key.Bucket << "Hash"sv << Request.Key.Hash; - } - Writer.EndObject(); - CompressedBuffer Buffer = Request.Value; - Writer.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(Buffer.GetRawHash())); - Package.AddAttachment(CbAttachment(Buffer)); - Writer.AddString("Policy"sv, WriteToString<128>(Request.Policy)); - } - Writer.EndObject(); - Request.Data->Data->ReceivedPutValue = true; - } - Writer.EndArray(); + CachePolicy BatchDefaultPolicy = CachePolicy::Default; + + cacherequests::PutCacheValuesRequest Request = {.DefaultPolicy = BatchDefaultPolicy, .Namespace = std::string(TestNamespace)}; + Request.Requests.reserve(PutValueRequests.size()); + for (CachePutValueRequest& PutRequest : PutValueRequests) + { + Request.Requests.push_back({.Key = PutRequest.Key, .Body = PutRequest.Value, .Policy = PutRequest.Policy}); + PutRequest.Data->Data->ReceivedPutValue = true; } - Writer.EndObject(); - Package.SetObject(Writer.Save()); + + CbPackage Package; + CHECK(Request.Format(Package)); + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, @@ -1945,34 +1796,16 @@ TEST_CASE("zcache.rpc.allpolicies") // GetCacheRecords { - CbPackage Package; - CbObjectWriter Writer; - Writer << "Method"sv - << "GetCacheRecords"sv; - Writer.BeginObject("Params"sv); - { - CachePolicy BatchDefaultPolicy = CachePolicy::Default; - Writer << "DefaultPolicy"sv << WriteToString<128>(BatchDefaultPolicy); - Writer << "Namespace"sv << TestNamespace; - Writer.BeginArray("Requests"sv); - for (CacheGetRequest& Request : GetRequests) - { - Writer.BeginObject(); - { - Writer.BeginObject("Key"sv); - { - Writer << "Bucket"sv << Request.Key.Bucket << "Hash"sv << Request.Key.Hash; - } - Writer.EndObject(); - Writer.SetName("Policy"sv); - Request.Policy.Save(Writer); - } - Writer.EndObject(); - } - Writer.EndArray(); + CachePolicy BatchDefaultPolicy = CachePolicy::Default; + cacherequests::GetCacheRecordsRequest Request = {.DefaultPolicy = BatchDefaultPolicy, .Namespace = std::string(TestNamespace)}; + Request.Requests.reserve(GetRequests.size()); + for (CacheGetRequest& GetRequest : GetRequests) + { + Request.Requests.push_back({.Key = GetRequest.Key, .Policy = GetRequest.Policy}); } - Writer.EndObject(); - Package.SetObject(Writer.Save()); + + CbPackage Package; + CHECK(Request.Format(Package)); IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, @@ -1981,20 +1814,16 @@ TEST_CASE("zcache.rpc.allpolicies") CbPackage Response; bool Loaded = Response.TryLoad(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); CHECK_MESSAGE(Loaded, "GetCacheRecords response failed to load."); - CbObjectView ResponseObject = Response.GetObject(); - CbArrayView Responses = ResponseObject["Result"sv].AsArrayView(); - CHECK_MESSAGE(Responses.Num() == GetRequests.size(), "GetCacheRecords response count did not match request count."); - int Index = 0; - for (CbFieldView ResponseField : Responses) - { - CbObjectView RecordView = ResponseField.AsObjectView(); - bool Succeeded = !ResponseField.HasError(); - - CacheGetRequest& Request = GetRequests[Index++]; - KeyData* KeyData = Request.Data->Data; - KeyData->ReceivedGet = true; + cacherequests::GetCacheRecordsResult RequestResult; + CHECK(RequestResult.Parse(Response)); + CHECK_MESSAGE(RequestResult.Results.size() == GetRequests.size(), "GetCacheRecords response count did not match request count."); + for (int Index = 0; const std::optional<cacherequests::GetCacheRecordResult>& RecordResult : RequestResult.Results) + { + bool Succeeded = RecordResult.has_value(); + CacheGetRequest& GetRequest = GetRequests[Index++]; + KeyData* KeyData = GetRequest.Data->Data; + KeyData->ReceivedGet = true; WriteToString<32> Name("Get(", KeyData->KeyIndex, ")"); - if (KeyData->ShouldBeHit) { CHECK_MESSAGE(Succeeded, WriteToString<32>(Name, " unexpectedly failed.").c_str()); @@ -2005,45 +1834,32 @@ TEST_CASE("zcache.rpc.allpolicies") } if (!KeyData->ForceMiss && Succeeded) { - CbArrayView ValuesArray = RecordView["Values"sv].AsArrayView(); - CHECK_MESSAGE(ValuesArray.Num() == NumValues, WriteToString<32>(Name, " number of values did not match.").c_str()); - - for (CbFieldView ValueField : ValuesArray) + CHECK_MESSAGE(RecordResult->Values.size() == NumValues, + WriteToString<32>(Name, " number of values did not match.").c_str()); + for (const cacherequests::GetCacheRecordResultValue& Value : RecordResult->Values) { - CbObjectView ValueObject = ValueField.AsObjectView(); - Oid ActualValueId = ValueObject["Id"sv].AsObjectId(); - int ExpectedValueIndex = 0; + int ExpectedValueIndex = 0; for (; ExpectedValueIndex < NumValues; ++ExpectedValueIndex) { - if (ValueIds[ExpectedValueIndex] == ActualValueId) + if (ValueIds[ExpectedValueIndex] == Value.Id) { break; } } CHECK_MESSAGE(ExpectedValueIndex < NumValues, WriteToString<32>(Name, " could not find matching ValueId.").c_str()); - WriteToString<32> ValueName("Get(", KeyData->KeyIndex, ",", ExpectedValueIndex, ")"); - IoHash ActualRawHash = ValueObject["RawHash"sv].AsHash(); - const CbAttachment* Attachment = Response.FindAttachment(ActualRawHash); - CompressedBuffer ActualBuffer = Attachment ? Attachment->AsCompressedBinary() : CompressedBuffer(); - uint64_t ActualRawSize = UINT64_MAX; - if (ActualBuffer) - { - ActualRawSize = ActualBuffer.GetRawSize(); - } - else - { - ActualRawSize = ValueObject["RawSize"sv].AsUInt64(UINT64_MAX); - } + + WriteToString<32> ValueName("Get(", KeyData->KeyIndex, ",", ExpectedValueIndex, ")"); + CompressedBuffer ExpectedValue = KeyData->BufferValues[ExpectedValueIndex]; - CHECK_MESSAGE(ActualRawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()), + CHECK_MESSAGE(Value.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()), WriteToString<32>(ValueName, " RawHash did not match.").c_str()); - CHECK_MESSAGE(ActualRawSize == ExpectedValue.GetRawSize(), + CHECK_MESSAGE(Value.RawSize == ExpectedValue.GetRawSize(), WriteToString<32>(ValueName, " RawSize did not match.").c_str()); if (KeyData->GetRequestsData) { - SharedBuffer Buffer = ActualBuffer.Decompress(); - CHECK_MESSAGE(Buffer.GetSize() == ActualRawSize, + SharedBuffer Buffer = Value.Body.Decompress(); + CHECK_MESSAGE(Buffer.GetSize() == Value.RawSize, WriteToString<32>(ValueName, " BufferSize did not match RawSize.").c_str()); uint64_t ActualIntValue = ((const uint64_t*)Buffer.GetData())[0]; uint64_t ExpectedIntValue = KeyData->IntValues[ExpectedValueIndex]; @@ -2056,33 +1872,19 @@ TEST_CASE("zcache.rpc.allpolicies") // GetCacheValues { - CbPackage Package; - CbObjectWriter Writer; - Writer << "Method"sv - << "GetCacheValues"sv; - Writer.BeginObject("Params"sv); - { - CachePolicy BatchDefaultPolicy = CachePolicy::Default; - Writer << "DefaultPolicy"sv << WriteToString<128>(BatchDefaultPolicy); - Writer << "Namespace"sv << TestNamespace; - Writer.BeginArray("Requests"sv); - for (CacheGetValueRequest& Request : GetValueRequests) - { - Writer.BeginObject(); - { - Writer.BeginObject("Key"sv); - { - Writer << "Bucket"sv << Request.Key.Bucket << "Hash"sv << Request.Key.Hash; - } - Writer.EndObject(); - Writer.AddString("Policy"sv, WriteToString<128>(Request.Policy)); - } - Writer.EndObject(); - } - Writer.EndArray(); + CachePolicy BatchDefaultPolicy = CachePolicy::Default; + + cacherequests::GetCacheValuesRequest GetCacheValuesRequest = {.DefaultPolicy = BatchDefaultPolicy, + .Namespace = std::string(TestNamespace)}; + GetCacheValuesRequest.Requests.reserve(GetValueRequests.size()); + for (CacheGetValueRequest& GetRequest : GetValueRequests) + { + GetCacheValuesRequest.Requests.push_back({.Key = GetRequest.Key, .Policy = GetRequest.Policy}); } - Writer.EndObject(); - Package.SetObject(Writer.Save()); + + CbPackage Package; + CHECK(GetCacheValuesRequest.Format(Package)); + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, @@ -2091,20 +1893,14 @@ TEST_CASE("zcache.rpc.allpolicies") CbPackage Response; bool Loaded = Response.TryLoad(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); CHECK_MESSAGE(Loaded, "GetCacheValues response failed to load."); - CbObjectView ResponseObject = Response.GetObject(); - CbArrayView Responses = ResponseObject["Result"sv].AsArrayView(); - CHECK_MESSAGE(Responses.Num() == GetValueRequests.size(), "GetCacheValues response count did not match request count."); - int Index = 0; - for (CbFieldView RequestResultView : Responses) - { - CbObjectView RequestResultObject = RequestResultView.AsObjectView(); - CbFieldView RawHashField = RequestResultObject["RawHash"sv]; - IoHash ActualRawHash = RawHashField.AsHash(); - bool Succeeded = !RawHashField.HasError(); - - CacheGetValueRequest& Request = GetValueRequests[Index++]; - KeyData* KeyData = Request.Data->Data; - KeyData->ReceivedGetValue = true; + cacherequests::GetCacheValuesResult GetCacheValuesResult; + CHECK(GetCacheValuesResult.Parse(Response)); + for (int Index = 0; const cacherequests::CacheValueResult& ValueResult : GetCacheValuesResult.Results) + { + bool Succeeded = ValueResult.RawHash != IoHash::Zero; + CacheGetValueRequest& Request = GetValueRequests[Index++]; + KeyData* KeyData = Request.Data->Data; + KeyData->ReceivedGetValue = true; WriteToString<32> Name("GetValue("sv, KeyData->KeyIndex, ")"sv); if (KeyData->ShouldBeHit) @@ -2117,26 +1913,17 @@ TEST_CASE("zcache.rpc.allpolicies") } if (!KeyData->ForceMiss && Succeeded) { - const CbAttachment* Attachment = Response.FindAttachment(ActualRawHash); - CompressedBuffer ActualBuffer = Attachment ? Attachment->AsCompressedBinary() : CompressedBuffer(); - uint64_t ActualRawSize = UINT64_MAX; - if (ActualBuffer) - { - ActualRawSize = ActualBuffer.GetRawSize(); - } - else - { - ActualRawSize = RequestResultObject["RawSize"sv].AsUInt64(UINT64_MAX); - } CompressedBuffer ExpectedValue = KeyData->BufferValues[0]; - CHECK_MESSAGE(ActualRawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()), + CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()), WriteToString<32>(Name, " RawHash did not match.").c_str()); - CHECK_MESSAGE(ActualRawSize == ExpectedValue.GetRawSize(), WriteToString<32>(Name, " RawSize did not match.").c_str()); + CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.GetRawSize(), + WriteToString<32>(Name, " RawSize did not match.").c_str()); if (KeyData->GetRequestsData) { - SharedBuffer Buffer = ActualBuffer.Decompress(); - CHECK_MESSAGE(Buffer.GetSize() == ActualRawSize, WriteToString<32>(Name, " BufferSize did not match RawSize.").c_str()); + SharedBuffer Buffer = ValueResult.Body.Decompress(); + CHECK_MESSAGE(Buffer.GetSize() == ValueResult.RawSize, + WriteToString<32>(Name, " BufferSize did not match RawSize.").c_str()); uint64_t ActualIntValue = ((const uint64_t*)Buffer.GetData())[0]; uint64_t ExpectedIntValue = KeyData->IntValues[0]; CHECK_MESSAGE(ActualIntValue == ExpectedIntValue, WriteToString<32>(Name, " had unexpected data.").c_str()); @@ -2147,40 +1934,25 @@ TEST_CASE("zcache.rpc.allpolicies") // GetCacheChunks { - CbPackage Package; - CbObjectWriter Writer; std::sort(ChunkRequests.begin(), ChunkRequests.end(), [](CacheGetChunkRequest& A, CacheGetChunkRequest& B) { return A.Key.Hash < B.Key.Hash; }); - Writer << "Method"sv - << "GetCacheChunks"sv; - Writer.BeginObject("Params"sv); - { - CachePolicy BatchDefaultPolicy = CachePolicy::Default; - Writer << "DefaultPolicy"sv << WriteToString<128>(BatchDefaultPolicy); - Writer << "Namespace"sv << TestNamespace; - Writer.BeginArray("ChunkRequests"sv); - for (CacheGetChunkRequest& Request : ChunkRequests) - { - Writer.BeginObject(); - { - Writer.BeginObject("Key"sv); - { - Writer << "Bucket"sv << Request.Key.Bucket << "Hash"sv << Request.Key.Hash; - } - Writer.EndObject(); - Writer.AddObjectId("ValueId"sv, Request.ValueId); - Writer.AddInteger("RawOffset"sv, Request.RawOffset); - Writer.AddInteger("RawSize"sv, Request.RawSize); - Writer.AddHash("ChunkId"sv, IoHash()); - Writer.AddString("Policy"sv, WriteToString<128>(Request.Policy)); - } - Writer.EndObject(); - } - Writer.EndArray(); - } - Writer.EndObject(); - Package.SetObject(Writer.Save()); + CachePolicy BatchDefaultPolicy = CachePolicy::Default; + cacherequests::GetCacheChunksRequest GetCacheChunksRequest = {.DefaultPolicy = BatchDefaultPolicy, + .Namespace = std::string(TestNamespace)}; + GetCacheChunksRequest.Requests.reserve(ChunkRequests.size()); + for (CacheGetChunkRequest& ChunkRequest : ChunkRequests) + { + GetCacheChunksRequest.Requests.push_back({.Key = ChunkRequest.Key, + .ValueId = ChunkRequest.ValueId, + .ChunkId = IoHash(), + .RawOffset = ChunkRequest.RawOffset, + .RawSize = ChunkRequest.RawSize, + .Policy = ChunkRequest.Policy}); + } + CbPackage Package; + CHECK(GetCacheChunksRequest.Format(Package)); + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, @@ -2189,16 +1961,14 @@ TEST_CASE("zcache.rpc.allpolicies") CbPackage Response; bool Loaded = Response.TryLoad(zen::IoBuffer(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size())); CHECK_MESSAGE(Loaded, "GetCacheChunks response failed to load."); - CbObjectView ResponseObject = Response.GetObject(); - CbArrayView Responses = ResponseObject["Result"sv].AsArrayView(); - CHECK_MESSAGE(Responses.Num() == ChunkRequests.size(), "GetCacheChunks response count did not match request count."); - int Index = 0; - for (CbFieldView RequestResultView : Responses) + cacherequests::GetCacheChunksResult GetCacheChunksResult; + CHECK(GetCacheChunksResult.Parse(Response)); + CHECK_MESSAGE(GetCacheChunksResult.Results.size() == ChunkRequests.size(), + "GetCacheChunks response count did not match request count."); + + for (int Index = 0; const cacherequests::CacheValueResult& ValueResult : GetCacheChunksResult.Results) { - CbObjectView RequestResultObject = RequestResultView.AsObjectView(); - CbFieldView RawHashField = RequestResultObject["RawHash"sv]; - IoHash ActualRawHash = RawHashField.AsHash(); - bool Succeeded = !RawHashField.HasError(); + bool Succeeded = ValueResult.RawHash != IoHash::Zero; CacheGetChunkRequest& Request = ChunkRequests[Index++]; KeyData* KeyData = Request.Data->Data; @@ -2216,26 +1986,17 @@ TEST_CASE("zcache.rpc.allpolicies") } if (KeyData->ShouldBeHit && Succeeded) { - const CbAttachment* Attachment = Response.FindAttachment(ActualRawHash); - CompressedBuffer ActualBuffer = Attachment ? Attachment->AsCompressedBinary() : CompressedBuffer(); - uint64_t ActualRawSize = UINT64_MAX; - if (ActualBuffer) - { - ActualRawSize = ActualBuffer.GetRawSize(); - } - else - { - ActualRawSize = RequestResultObject["RawSize"sv].AsUInt64(UINT64_MAX); - } CompressedBuffer ExpectedValue = KeyData->BufferValues[ValueIndex]; - CHECK_MESSAGE(ActualRawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()), + CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()), WriteToString<32>(Name, " had unexpected RawHash.").c_str()); - CHECK_MESSAGE(ActualRawSize == ExpectedValue.GetRawSize(), WriteToString<32>(Name, " had unexpected RawSize.").c_str()); + CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.GetRawSize(), + WriteToString<32>(Name, " had unexpected RawSize.").c_str()); if (KeyData->GetRequestsData) { - SharedBuffer Buffer = ActualBuffer.Decompress(); - CHECK_MESSAGE(Buffer.GetSize() == ActualRawSize, WriteToString<32>(Name, " BufferSize did not match RawSize.").c_str()); + SharedBuffer Buffer = ValueResult.Body.Decompress(); + CHECK_MESSAGE(Buffer.GetSize() == ValueResult.RawSize, + WriteToString<32>(Name, " BufferSize did not match RawSize.").c_str()); uint64_t ActualIntValue = ((const uint64_t*)Buffer.GetData())[0]; uint64_t ExpectedIntValue = KeyData->IntValues[ValueIndex]; CHECK_MESSAGE(ActualIntValue == ExpectedIntValue, WriteToString<32>(Name, " had unexpected data.").c_str()); |