diff options
Diffstat (limited to 'zenserver/cache/structuredcache.cpp')
| -rw-r--r-- | zenserver/cache/structuredcache.cpp | 246 |
1 files changed, 32 insertions, 214 deletions
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 49e5896d1..e23030e24 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -996,155 +996,9 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack return PutResult::Success; } -#if BACKWARDS_COMPATABILITY_JAN2022 -void -HttpStructuredCacheService::HandleRpcGetCacheRecordsLegacy(zen::HttpServerRequest& Request, CbObjectView RpcRequest) -{ - ZEN_TRACE_CPU("Z$::RpcGetCacheRecords"); - - CbPackage RpcResponse; - CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); - CacheRecordPolicy BatchPolicy = LoadCacheRecordPolicy(Params["Policy"sv].AsObjectView()); - std::vector<CacheKey> CacheKeys; - std::vector<IoBuffer> CacheValues; - std::vector<size_t> UpstreamRequests; - - ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheRecords"sv); - - for (CbFieldView KeyView : Params["CacheKeys"sv]) - { - CbObjectView KeyObject = KeyView.AsObjectView(); - CacheKeys.push_back(CacheKey::Create(KeyObject["Bucket"sv].AsString(), KeyObject["Hash"sv].AsHash())); - } - - if (CacheKeys.empty()) - { - return Request.WriteResponse(HttpResponseCode::BadRequest); - } - - CacheValues.resize(CacheKeys.size()); - - for (size_t KeyIndex = 0; const CacheKey& Key : CacheKeys) - { - ZenCacheValue CacheValue; - uint32_t MissingCount = 0; - uint32_t MissingReadFromUpstreamCount = 0; - - if (EnumHasAllFlags(BatchPolicy.GetRecordPolicy(), CachePolicy::QueryLocal) && m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue) && - CacheValue.Value.GetContentType() == ZenContentType::kCbObject) - { - CbObjectView CacheRecord(CacheValue.Value.Data()); - CacheRecord.IterateAttachments( - [this, &MissingCount, &MissingReadFromUpstreamCount, &RpcResponse, &BatchPolicy](CbFieldView AttachmentHash) { - CachePolicy ValuePolicy = BatchPolicy.GetRecordPolicy(); - if (!EnumHasAllFlags(ValuePolicy, CachePolicy::QueryLocal)) - { - // A value that is requested without the Query flag (such as None/Disable) does not count as missing, because we - // didn't ask for it and thus the record is complete in its absence. - if (EnumHasAllFlags(ValuePolicy, CachePolicy::QueryRemote)) - { - MissingReadFromUpstreamCount++; - MissingCount++; - } - } - else if (EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData)) - { - if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) - { - if (EnumHasAllFlags(ValuePolicy, CachePolicy::QueryRemote)) - { - MissingReadFromUpstreamCount++; - } - MissingCount++; - } - } - else - { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) - { - ZEN_ASSERT(Chunk.GetSize() > 0); - RpcResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); - } - else - { - if (EnumHasAllFlags(ValuePolicy, CachePolicy::QueryRemote)) - { - MissingReadFromUpstreamCount++; - } - MissingCount++; - } - } - }); - } - - // Searching upstream is not implemented in this legacy support function - if (CacheValue.Value && (MissingCount == 0 || EnumHasAllFlags(BatchPolicy.GetRecordPolicy(), CachePolicy::PartialRecord))) - { - ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL) {}", - Key.Bucket, - Key.Hash, - NiceBytes(CacheValue.Value.Size()), - ToString(CacheValue.Value.GetContentType()), - MissingCount ? "(PARTIAL)" : ""sv); - - CacheValues[KeyIndex] = std::move(CacheValue.Value); - m_CacheStats.HitCount++; - } - else - { - if (!EnumHasAnyFlags(BatchPolicy.GetRecordPolicy(), CachePolicy::Query)) - { - // If they requested no query, do not record this as a miss - ZEN_DEBUG("DISABLEDQUERY - '{}/{}'", Key.Bucket, Key.Hash); - } - else - { - ZEN_DEBUG("MISS - '{}/{}' {}", Key.Bucket, Key.Hash, MissingCount ? "(PARTIAL)"sv : ""sv); - m_CacheStats.MissCount++; - } - } - - ++KeyIndex; - } - - CbObjectWriter ResponseObject; - - ResponseObject.BeginArray("Result"sv); - for (const IoBuffer& Value : CacheValues) - { - if (Value) - { - CbObjectView Record(Value.Data()); - ResponseObject << Record; - } - else - { - ResponseObject.AddNull(); - } - } - ResponseObject.EndArray(); - - RpcResponse.SetObject(ResponseObject.Save()); - - BinaryWriter MemStream; - RpcResponse.Save(MemStream); - - Request.WriteResponse(HttpResponseCode::OK, - HttpContentType::kCbPackage, - IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); -} -#endif - void HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& HttpRequest, CbObjectView RpcRequest) { -#if BACKWARDS_COMPATABILITY_JAN2022 - // Backwards compatability; - if (RpcRequest["Params"sv].AsObjectView()["CacheKeys"sv]) - { - return HandleRpcGetCacheRecordsLegacy(HttpRequest, RpcRequest); - } -#endif ZEN_TRACE_CPU("Z$::RpcGetCacheRecords"); CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); @@ -1559,13 +1413,6 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ void HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& HttpRequest, CbObjectView RpcRequest) { -#if BACKWARDS_COMPATABILITY_JAN2022 - if (RpcRequest["Params"sv].AsObjectView()["ChunkRequests"]) - { - return HandleRpcGetCacheChunks(HttpRequest, RpcRequest); - } -#endif - ZEN_TRACE_CPU("Z$::RpcGetCacheValues"); CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); @@ -1737,28 +1584,23 @@ HttpStructuredCacheService::HandleRpcGetCacheChunks(zen::HttpServerRequest& Http std::vector<KeyRequestData> KeyRequests; std::vector<ChunkRequestData> Chunks; - BACKWARDS_COMPATABILITY_JAN2022_CODE(bool SendValueOnly = false;) - if (!TryGetCacheChunks_Parse(KeyRequests, Chunks BACKWARDS_COMPATABILITY_JAN2022_CODE(, SendValueOnly), RpcRequest)) + if (!TryGetCacheChunks_Parse(KeyRequests, Chunks, RpcRequest)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } GetCacheChunks_LoadKeys(KeyRequests); GetCacheChunks_LoadChunks(Chunks); - GetCacheChunks_SendResults(Chunks, HttpRequest BACKWARDS_COMPATABILITY_JAN2022_CODE(, SendValueOnly)); + GetCacheChunks_SendResults(Chunks, HttpRequest); } bool -HttpStructuredCacheService::TryGetCacheChunks_Parse(std::vector<GetCacheChunks::detail::KeyRequestData>& KeyRequests, - std::vector<GetCacheChunks::detail::ChunkRequestData>& Chunks, - BACKWARDS_COMPATABILITY_JAN2022_CODE(bool& SendValueOnly, ) CbObjectView RpcRequest) +HttpStructuredCacheService::TryGetCacheChunks_Parse(std::vector<GetCacheChunks::detail::KeyRequestData>& KeyRequests, + std::vector<GetCacheChunks::detail::ChunkRequestData>& Chunks, + CbObjectView RpcRequest) { using namespace GetCacheChunks::detail; -#if BACKWARDS_COMPATABILITY_JAN2022 - SendValueOnly = RpcRequest["MethodVersion"sv].AsInt32() < 1; -#else ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheChunks"sv); -#endif CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); std::string_view DefaultPolicyText = Params["DefaultPolicy"sv].AsString(); @@ -1809,13 +1651,7 @@ HttpStructuredCacheService::TryGetCacheChunks_Parse(std::vector<GetCacheChunks:: Chunk.Upstream.RawSize = RequestObject["RawSize"sv].AsUInt64(UINT64_MAX); std::string_view PolicyText = RequestObject["Policy"sv].AsString(); Chunk.DownstreamPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; -#if BACKWARDS_COMPATABILITY_JAN2022 - if (SendValueOnly) - { - Chunk.DownstreamPolicy = Chunk.DownstreamPolicy & (~CachePolicy::SkipData); - } -#endif - Chunk.IsRecordRequest = (bool)Chunk.Upstream.ValueId; + Chunk.IsRecordRequest = (bool)Chunk.Upstream.ValueId; if (!Chunk.IsRecordRequest || Chunk.Upstream.ChunkId == IoHash::Zero) { @@ -2088,8 +1924,7 @@ HttpStructuredCacheService::GetCacheChunks_LoadChunks(std::vector<GetCacheChunks void HttpStructuredCacheService::GetCacheChunks_SendResults(std::vector<GetCacheChunks::detail::ChunkRequestData>& Chunks, - zen::HttpServerRequest& HttpRequest - BACKWARDS_COMPATABILITY_JAN2022_CODE(, bool SendValueOnly)) + zen::HttpServerRequest& HttpRequest) { using namespace GetCacheChunks::detail; @@ -2099,57 +1934,40 @@ HttpStructuredCacheService::GetCacheChunks_SendResults(std::vector<GetCacheChunk Writer.BeginArray("Result"sv); for (ChunkRequestData& Chunk : Chunks) { -#if BACKWARDS_COMPATABILITY_JAN2022 - if (SendValueOnly) - { - if (Chunk.Value) - { - Writer << Chunk.Upstream.ChunkId; - RpcResponse.AddAttachment(CbAttachment(Chunk.Value)); - } - else - { - Writer << IoHash::Zero; - } - } - else -#endif + Writer.BeginObject(); { - Writer.BeginObject(); + if (Chunk.Exists) { - if (Chunk.Exists) + Writer.AddHash("RawHash"sv, Chunk.Upstream.ChunkId); + if (Chunk.Value && !EnumHasAllFlags(Chunk.DownstreamPolicy, CachePolicy::SkipData)) { - Writer.AddHash("RawHash"sv, Chunk.Upstream.ChunkId); - if (Chunk.Value && !EnumHasAllFlags(Chunk.DownstreamPolicy, CachePolicy::SkipData)) - { - RpcResponse.AddAttachment(CbAttachment(Chunk.Value)); - } - else - { - Writer.AddInteger("RawSize"sv, Chunk.TotalSize); - } - - ZEN_DEBUG("CHUNKHIT - '{}/{}/{}' {} '{}' ({})", - Chunk.Upstream.Key.Bucket, - Chunk.Upstream.Key.Hash, - Chunk.Upstream.ValueId, - NiceBytes(Chunk.TotalSize), - Chunk.IsRecordRequest ? "Record"sv : "Value"sv, - Chunk.Source); - m_CacheStats.HitCount++; - } - else if (!EnumHasAnyFlags(Chunk.DownstreamPolicy, CachePolicy::Query)) - { - ZEN_DEBUG("CHUNKSKIP - '{}/{}/{}'", Chunk.Upstream.Key.Bucket, Chunk.Upstream.Key.Hash, Chunk.Upstream.ValueId); + RpcResponse.AddAttachment(CbAttachment(Chunk.Value)); } else { - ZEN_DEBUG("MISS - '{}/{}/{}'", Chunk.Upstream.Key.Bucket, Chunk.Upstream.Key.Hash, Chunk.Upstream.ValueId); - m_CacheStats.MissCount++; + Writer.AddInteger("RawSize"sv, Chunk.TotalSize); } + + ZEN_DEBUG("CHUNKHIT - '{}/{}/{}' {} '{}' ({})", + Chunk.Upstream.Key.Bucket, + Chunk.Upstream.Key.Hash, + Chunk.Upstream.ValueId, + NiceBytes(Chunk.TotalSize), + Chunk.IsRecordRequest ? "Record"sv : "Value"sv, + Chunk.Source); + m_CacheStats.HitCount++; + } + else if (!EnumHasAnyFlags(Chunk.DownstreamPolicy, CachePolicy::Query)) + { + ZEN_DEBUG("CHUNKSKIP - '{}/{}/{}'", Chunk.Upstream.Key.Bucket, Chunk.Upstream.Key.Hash, Chunk.Upstream.ValueId); + } + else + { + ZEN_DEBUG("MISS - '{}/{}/{}'", Chunk.Upstream.Key.Bucket, Chunk.Upstream.Key.Hash, Chunk.Upstream.ValueId); + m_CacheStats.MissCount++; } - Writer.EndObject(); } + Writer.EndObject(); } Writer.EndArray(); |