From 154743f2d2ff2b7163bcf8d7b76eea3e3579aaba Mon Sep 17 00:00:00 2001 From: mattpetersepic <58296718+mattpetersepic@users.noreply.github.com> Date: Tue, 1 Feb 2022 08:06:36 -0700 Subject: Cache policy support (#47) Add HandleRpc methods for the remaining ICacheStore requests from unreal: PutCacheValues/GetCacheValues. We now have batched versions for PutCacheRecords,GetCacheRecords,PutCacheValues,GetCacheValues,GetCacheChunks. Add support for CachePolicy flags to all of these batched methods. * Add Batched PutCacheValues/GetCacheValues. Rename old GetCacheValues to GetCacheChunks. * HandleRpcGetCacheRecords: Receive a CacheRecordPolicy with each key, and skipdata on attachments we already have. * Changes to CachePolicy copied from Release-5.0 depot. Change serialization to use the key BasePolicy instead of DefaultValuePolicy. * GetChunks: Read CacheRecords from remote if necessary to find ContentId. Implement QueryLocal, StoreLocal, and SkipData. --- zenserver-test/zenserver-test.cpp | 89 +++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 31 deletions(-) (limited to 'zenserver-test/zenserver-test.cpp') diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 85393aed2..425f43946 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1827,7 +1827,8 @@ TEST_CASE("zcache.rpc") Data[Idx] = Idx % 255; } - zen::CbAttachment Attachment(zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size()))); + CompressedBuffer Value = zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size())); + zen::CbAttachment Attachment(Value); Writer.BeginObject(); { @@ -1838,7 +1839,17 @@ TEST_CASE("zcache.rpc") Writer << "Bucket"sv << CacheKey.Bucket << "Hash"sv << CacheKey.Hash; } Writer.EndObject(); - Writer << "Data"sv << Attachment; + Writer.BeginArray("Values"sv); + { + Writer.BeginObject(); + { + Writer.AddObjectId("Id"sv, Oid::NewOid()); + Writer.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(Value.GetRawHash())); + Writer.AddInteger("RawSize"sv, Value.GetRawSize()); + } + Writer.EndObject(); + } + Writer.EndArray(); } Writer.EndObject(); Writer.SetName("Policy"sv); @@ -1862,7 +1873,7 @@ TEST_CASE("zcache.rpc") for (uint32_t Key = 1; Key <= Num; ++Key) { - zen::IoHash KeyHash; + zen::IoHash KeyHash; ((uint32_t*)(KeyHash.Hash))[0] = Key; const zen::CacheKey CacheKey = zen::CacheKey::Create(Bucket, KeyHash); CbPackage Package; @@ -1907,27 +1918,30 @@ TEST_CASE("zcache.rpc") bool Success; }; - auto GetCacheRecords = - [](std::string_view BaseUri, std::span Keys, const zen::CacheRecordPolicy& Policy) -> GetCacheRecordResult { + auto GetCacheRecords = [](std::string_view BaseUri, std::span Keys, zen::CachePolicy Policy) -> GetCacheRecordResult { using namespace zen; CbObjectWriter Request; Request << "Method"sv << "GetCacheRecords"sv; Request.BeginObject("Params"sv); - - Request.BeginArray("CacheKeys"sv); - for (const CacheKey& Key : Keys) { - Request.BeginObject(); - Request << "Bucket"sv << Key.Bucket << "Hash"sv << Key.Hash; - Request.EndObject(); + Request << "DefaultPolicy"sv << WriteToString<128>(Policy); + Request.BeginArray("Requests"sv); + for (const CacheKey& Key : Keys) + { + Request.BeginObject(); + { + Request.BeginObject("Key"sv); + { + Request << "Bucket"sv << Key.Bucket << "Hash"sv << Key.Hash; + } + Request.EndObject(); + } + Request.EndObject(); + } + Request.EndArray(); } - Request.EndArray(); - - Request.SetName("Policy"sv); - Policy.Save(Request); - Request.EndObject(); BinaryWriter Body; @@ -1978,7 +1992,7 @@ TEST_CASE("zcache.rpc") Inst.SpawnServer(PortNumber); Inst.WaitUntilReady(); - CacheRecordPolicy Policy; + CachePolicy Policy = CachePolicy::Default; std::vector Keys = PutCacheRecords(BaseUri, "mastodon"sv, 128); GetCacheRecordResult Result = GetCacheRecords(BaseUri, Keys, Policy); @@ -1988,11 +2002,18 @@ TEST_CASE("zcache.rpc") { const CacheKey& ExpectedKey = Keys[Index++]; - CbObjectView RecordObj = RecordView.AsObjectView(); - CbObjectView KeyObj = RecordObj["Key"sv].AsObjectView(); - const CacheKey Key = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash()); - const IoHash AttachmentHash = RecordObj["Data"sv].AsHash(); - const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); + CbObjectView RecordObj = RecordView.AsObjectView(); + CbObjectView KeyObj = RecordObj["Key"sv].AsObjectView(); + const CacheKey Key = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash()); + IoHash AttachmentHash; + size_t NumValues = 0; + for (CbFieldView Value : RecordObj["Values"sv]) + { + AttachmentHash = Value.AsObjectView()["RawHash"sv].AsHash(); + ++NumValues; + } + CHECK(NumValues == 1); + const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); CHECK(Key == ExpectedKey); CHECK(Attachment != nullptr); @@ -2010,7 +2031,7 @@ TEST_CASE("zcache.rpc") Inst.SpawnServer(PortNumber); Inst.WaitUntilReady(); - CacheRecordPolicy Policy; + CachePolicy Policy = CachePolicy::Default; std::vector ExistingKeys = PutCacheRecords(BaseUri, "mastodon"sv, 128); std::vector Keys; @@ -2035,12 +2056,18 @@ TEST_CASE("zcache.rpc") } else { - const CacheKey& ExpectedKey = ExistingKeys[KeyIndex++]; - CbObjectView RecordObj = RecordView.AsObjectView(); - zen::CacheKey Key = LoadKey(RecordObj["Key"sv]); - const IoHash AttachmentHash = RecordObj["Data"sv].AsHash(); - const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); - + const CacheKey& ExpectedKey = ExistingKeys[KeyIndex++]; + CbObjectView RecordObj = RecordView.AsObjectView(); + zen::CacheKey Key = LoadKey(RecordObj["Key"sv]); + IoHash AttachmentHash; + size_t NumValues = 0; + for (CbFieldView Value : RecordObj["Values"sv]) + { + AttachmentHash = Value.AsObjectView()["RawHash"sv].AsHash(); + ++NumValues; + } + CHECK(NumValues == 1); + const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash); CHECK(Key == ExpectedKey); CHECK(Attachment != nullptr); } @@ -2061,7 +2088,7 @@ TEST_CASE("zcache.rpc") std::vector Keys = PutCacheRecords(UpstreamCfg.BaseUri, "mastodon"sv, 4); - CacheRecordPolicy Policy(CachePolicy::QueryLocal); + CachePolicy Policy = CachePolicy::QueryLocal; GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, Keys, Policy); CHECK(Result.Records.size() == Keys.size()); @@ -2086,7 +2113,7 @@ TEST_CASE("zcache.rpc") std::vector Keys = PutCacheRecords(UpstreamCfg.BaseUri, "mastodon"sv, 4); - CacheRecordPolicy Policy(CachePolicy::QueryLocal | CachePolicy::QueryRemote); + CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, Keys, Policy); CHECK(Result.Records.size() == Keys.size()); -- cgit v1.2.3