From 88ed064ce4e4e5e3fe5d3856b3c4eac9550e69d7 Mon Sep 17 00:00:00 2001 From: mattpetersepic <58296718+mattpetersepic@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:54:39 -0700 Subject: Implement SkipData,QueryLocal,StoreLocal for GET-verb CacheGet requests (#39) * Implement SkipData,QueryLocal,StoreLocal for GET-verb CacheGet requests --- zenserver-test/zenserver-test.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'zenserver-test/zenserver-test.cpp') diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index c1d48502e..f51fd1504 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1729,7 +1729,7 @@ TEST_CASE("zcache.policy") } } - SUBCASE("skip - 'data' returns empty cache record/payload") + SUBCASE("skip - 'data' returns cache record without attachments/empty payload") { ZenConfig Cfg = ZenConfig::New(); ZenServerInstance Instance(TestEnv); @@ -1755,7 +1755,10 @@ TEST_CASE("zcache.policy") cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}}); CHECK(IsHttpSuccessCode(Result.status_code)); - CHECK(Result.text.size() == 0); + IoBuffer Buffer(IoBuffer::Wrap, Result.text.c_str(), Result.text.size()); + CbPackage ResponsePackage; + CHECK(ResponsePackage.TryLoad(Buffer)); + CHECK(ResponsePackage.GetAttachments().size() == 0); } // Get record @@ -1763,7 +1766,9 @@ TEST_CASE("zcache.policy") cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbobject"}}); CHECK(IsHttpSuccessCode(Result.status_code)); - CHECK(Result.text.size() == 0); + IoBuffer Buffer(IoBuffer::Wrap, Result.text.c_str(), Result.text.size()); + CbObject ResponseObject = zen::LoadCompactBinaryObject(Buffer); + CHECK((bool)ResponseObject); } // Get payload -- cgit v1.2.3 From 9f60e3a53ea082e4dd2553d50c4c03fa9e9a42c8 Mon Sep 17 00:00:00 2001 From: mattpetersepic <58296718+mattpetersepic@users.noreply.github.com> Date: Thu, 27 Jan 2022 09:16:05 -0700 Subject: Add batched CacheRecord put rpc (#38) * Add batched CacheRecord put rpc --- zenserver-test/zenserver-test.cpp | 91 ++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 30 deletions(-) (limited to 'zenserver-test/zenserver-test.cpp') diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index f51fd1504..85393aed2 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1814,7 +1814,12 @@ TEST_CASE("zcache.rpc") { using namespace std::literals; - auto CreateCacheRecord = [](const zen::CacheKey& CacheKey, size_t PayloadSize) -> zen::CbPackage { + auto AppendCacheRecord = [](CbPackage& Package, + CbWriter& Writer, + const zen::CacheKey& CacheKey, + size_t PayloadSize, + CachePolicy /* BatchDefaultPolicy */, + CachePolicy RecordPolicy) { std::vector Data; Data.resize(PayloadSize); for (size_t Idx = 0; Idx < PayloadSize; ++Idx) @@ -1824,17 +1829,24 @@ TEST_CASE("zcache.rpc") zen::CbAttachment Attachment(zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size()))); - zen::CbObjectWriter CacheRecord; - CacheRecord.BeginObject("CacheKey"sv); - CacheRecord << "Bucket"sv << CacheKey.Bucket << "Hash"sv << CacheKey.Hash; - CacheRecord.EndObject(); - CacheRecord << "Data"sv << Attachment; + Writer.BeginObject(); + { + Writer.BeginObject("Record"sv); + { + Writer.BeginObject("Key"sv); + { + Writer << "Bucket"sv << CacheKey.Bucket << "Hash"sv << CacheKey.Hash; + } + Writer.EndObject(); + Writer << "Data"sv << Attachment; + } + Writer.EndObject(); + Writer.SetName("Policy"sv); + CacheRecordPolicy(RecordPolicy).Save(Writer); + } + Writer.EndObject(); - zen::CbPackage Package; - Package.SetObject(CacheRecord.Save()); Package.AddAttachment(Attachment); - - return Package; }; auto ToIoBuffer = [](zen::CbPackage Package) -> zen::IoBuffer { @@ -1843,27 +1855,46 @@ TEST_CASE("zcache.rpc") return zen::IoBuffer(zen::IoBuffer::Clone, MemStream.Data(), MemStream.Size()); }; - auto PutCacheRecords = [&CreateCacheRecord, &ToIoBuffer](std::string_view BaseUri, - std::string_view Query, - std::string_view Bucket, - size_t Num, - size_t PayloadSize = 1024) -> std::vector { + auto PutCacheRecords = + [&AppendCacheRecord, + &ToIoBuffer](std::string_view BaseUri, std::string_view Bucket, size_t Num, size_t PayloadSize = 1024) -> std::vector { std::vector OutKeys; for (uint32_t Key = 1; Key <= Num; ++Key) { - const zen::CacheKey CacheKey = zen::CacheKey::Create(Bucket, zen::IoHash::HashBuffer(&Key, sizeof(uint32_t))); - CbPackage CacheRecord = CreateCacheRecord(CacheKey, PayloadSize); + zen::IoHash KeyHash; + ((uint32_t*)(KeyHash.Hash))[0] = Key; + const zen::CacheKey CacheKey = zen::CacheKey::Create(Bucket, KeyHash); + CbPackage Package; + CbWriter Writer; - OutKeys.push_back(CacheKey); + Writer.BeginObject(); + { + Writer << "Method"sv + << "PutCacheRecords"sv; + Writer.BeginObject("Params"sv); + { + CachePolicy BatchDefaultPolicy = CachePolicy::Default; + Writer << "DefaultPolicy"sv << WriteToString<128>(BatchDefaultPolicy); + Writer.BeginArray("Requests"sv); + { + AppendCacheRecord(Package, Writer, CacheKey, PayloadSize, BatchDefaultPolicy, CachePolicy::Default); + } + Writer.EndArray(); + } + Writer.EndObject(); + } + Writer.EndObject(); + Package.SetObject(Writer.Save().AsObject()); - IoBuffer Payload = ToIoBuffer(CacheRecord); + OutKeys.push_back(CacheKey); - cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}{}", BaseUri, CacheKey.Bucket, CacheKey.Hash, Query)}, - cpr::Body{(const char*)Payload.Data(), Payload.Size()}, - cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}}); + IoBuffer Body = FormatPackageMessageBuffer(Package).Flatten().AsIoBuffer(); + cpr::Response Result = cpr::Post(cpr::Url{fmt::format("{}/$rpc", BaseUri)}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}, + cpr::Body{(const char*)Body.GetData(), Body.GetSize()}); - CHECK(Result.status_code == 201); + CHECK(Result.status_code == 200); } return OutKeys; @@ -1948,7 +1979,7 @@ TEST_CASE("zcache.rpc") Inst.WaitUntilReady(); CacheRecordPolicy Policy; - std::vector Keys = PutCacheRecords(BaseUri, ""sv, "mastodon"sv, 128); + std::vector Keys = PutCacheRecords(BaseUri, "mastodon"sv, 128); GetCacheRecordResult Result = GetCacheRecords(BaseUri, Keys, Policy); CHECK(Result.Records.size() == Keys.size()); @@ -1958,7 +1989,7 @@ TEST_CASE("zcache.rpc") const CacheKey& ExpectedKey = Keys[Index++]; CbObjectView RecordObj = RecordView.AsObjectView(); - CbObjectView KeyObj = RecordObj["CacheKey"sv].AsObjectView(); + CbObjectView KeyObj = RecordObj["Key"sv].AsObjectView(); const CacheKey Key = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash()); const IoHash AttachmentHash = RecordObj["Data"sv].AsHash(); const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); @@ -1980,7 +2011,7 @@ TEST_CASE("zcache.rpc") Inst.WaitUntilReady(); CacheRecordPolicy Policy; - std::vector ExistingKeys = PutCacheRecords(BaseUri, ""sv, "mastodon"sv, 128); + std::vector ExistingKeys = PutCacheRecords(BaseUri, "mastodon"sv, 128); std::vector Keys; for (const zen::CacheKey& Key : ExistingKeys) @@ -2006,7 +2037,7 @@ TEST_CASE("zcache.rpc") { const CacheKey& ExpectedKey = ExistingKeys[KeyIndex++]; CbObjectView RecordObj = RecordView.AsObjectView(); - zen::CacheKey Key = LoadKey(RecordObj["CacheKey"sv]); + zen::CacheKey Key = LoadKey(RecordObj["Key"sv]); const IoHash AttachmentHash = RecordObj["Data"sv].AsHash(); const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); @@ -2028,7 +2059,7 @@ TEST_CASE("zcache.rpc") SpawnServer(UpstreamServer, UpstreamCfg); SpawnServer(LocalServer, LocalCfg); - std::vector Keys = PutCacheRecords(UpstreamCfg.BaseUri, ""sv, "mastodon"sv, 4); + std::vector Keys = PutCacheRecords(UpstreamCfg.BaseUri, "mastodon"sv, 4); CacheRecordPolicy Policy(CachePolicy::QueryLocal); GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, Keys, Policy); @@ -2053,7 +2084,7 @@ TEST_CASE("zcache.rpc") SpawnServer(UpstreamServer, UpstreamCfg); SpawnServer(LocalServer, LocalCfg); - std::vector Keys = PutCacheRecords(UpstreamCfg.BaseUri, ""sv, "mastodon"sv, 4); + std::vector Keys = PutCacheRecords(UpstreamCfg.BaseUri, "mastodon"sv, 4); CacheRecordPolicy Policy(CachePolicy::QueryLocal | CachePolicy::QueryRemote); GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, Keys, Policy); @@ -2064,7 +2095,7 @@ TEST_CASE("zcache.rpc") { const zen::CacheKey& ExpectedKey = Keys[Index++]; CbObjectView RecordObj = RecordView.AsObjectView(); - zen::CacheKey Key = LoadKey(RecordObj["CacheKey"sv]); + zen::CacheKey Key = LoadKey(RecordObj["Key"sv]); CHECK(Key == ExpectedKey); } } -- cgit v1.2.3