From 33fa76a35a96cad1865854068e60c1ca0b53864e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 6 May 2022 16:42:27 +0200 Subject: parameterize namespace for upstream (first hack) --- zenserver/cache/structuredcache.cpp | 240 +++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 89 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 0f16f6785..74438ac2e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -141,7 +141,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { // Bucket reference - return HandleCacheBucketRequest(Request, Key); + return HandleCacheBucketRequest(Request, ZenCacheStore::DefaultNamespace, Key); } return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL @@ -162,7 +162,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket) { switch (Request.RequestVerb()) { @@ -176,7 +176,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, case HttpVerb::kDelete: // Drop bucket - if (m_CacheStore.DropBucket(ZenCacheStore::DefaultNamespace, Bucket)) + if (m_CacheStore.DropBucket(Namespace, Bucket)) { return Request.WriteResponse(HttpResponseCode::OK); } @@ -226,7 +226,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && - m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) + m_CacheStore.Get(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { Success = true; ZenContentType ContentType = ClientResultValue.Value.GetContentType(); @@ -287,7 +287,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (Success) { - ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL)", + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' (LOCAL)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(ClientResultValue.Value.Size()), @@ -306,7 +307,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } else if (!EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryRemote)) { - ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); + ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); m_CacheStats.MissCount++; return Request.WriteResponse(HttpResponseCode::NotFound); } @@ -324,7 +325,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); - if (GetUpstreamCacheResult UpstreamResult = m_UpstreamCache.GetCacheRecord({Ref.BucketSegment, Ref.HashKey}, AcceptType); + if (GetUpstreamCacheResult UpstreamResult = + m_UpstreamCache.GetCacheRecord({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, AcceptType); UpstreamResult.Success) { Success = true; @@ -340,7 +342,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (ValidationResult != CbValidateError::None) { Success = false; - ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid compact binary object from upstream", + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid compact binary object from upstream", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); @@ -351,7 +354,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (Success && StoreLocal) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue); } } else if (AcceptType == ZenContentType::kCbPackage) @@ -405,7 +408,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (StoreLocal) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue); } BinaryWriter MemStream; @@ -434,14 +437,19 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request else { Success = false; - ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid upstream package", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid upstream package", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); } } } if (Success) { - ZEN_DEBUG("HIT - '{}/{}' {} '{}' (UPSTREAM)", + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' (UPSTREAM)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(ClientResultValue.Value.Size()), @@ -463,7 +471,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); + ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); m_CacheStats.MissCount++; AsyncRequest.WriteResponse(HttpResponseCode::NotFound); } @@ -486,12 +494,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) { - ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote)) { - m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.BucketSegment, Ref.HashKey}}); + m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}}); } Request.WriteResponse(HttpResponseCode::Created); @@ -502,7 +510,11 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ValidationResult != CbValidateError::None) { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid compact binary", Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid compact binary", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); } @@ -520,7 +532,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request TotalCount++; }); - ZEN_DEBUG("PUT - '{}/{}' {} '{}' attachments '{}/{}' (valid/total)", + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), @@ -529,14 +542,14 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ValidAttachments.size()); Body.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); const bool IsPartialRecord = TotalCount != static_cast(ValidAttachments.size()); if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, - .Key = {Ref.BucketSegment, Ref.HashKey}, + .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, .ValueContentIds = std::move(ValidAttachments)}); } @@ -548,7 +561,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (!Package.TryLoad(Body)) { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid package", Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid package", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); } CachePolicy Policy = PolicyFromURL; @@ -578,7 +591,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(HttpContentType::kCbPackage), @@ -599,7 +613,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); } - ZEN_DEBUG("PUT - '{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total)", + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.GetSize()), @@ -612,14 +627,14 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .Key = {Ref.BucketSegment, Ref.HashKey}, + .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, .ValueContentIds = std::move(ValidAttachments)}); } @@ -661,7 +676,7 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, if (QueryUpstream) { - if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); + if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); UpstreamResult.Success) { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) @@ -679,7 +694,8 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, if (!Value) { - ZEN_DEBUG("MISS - '{}/{}/{}' '{}' in {}", + ZEN_DEBUG("MISS - '{}/{}/{}/{}' '{}' in {}", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, @@ -689,7 +705,8 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, return Request.WriteResponse(HttpResponseCode::NotFound); } - ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({}) in {}", + ZEN_DEBUG("HIT - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, @@ -747,7 +764,8 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed); - ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({}) in {}", + ZEN_DEBUG("PUT - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, @@ -772,8 +790,14 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& return false; } + OutRef.Namespace = ToLower(""); // TODO: Should we add namespace to URI? OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); + if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) + { + return false; + } + if (!std::all_of(begin(OutRef.BucketSegment), end(OutRef.BucketSegment), [](const char c) { return std::isalnum(c); })) { return false; @@ -907,12 +931,15 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); - CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); + CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyView["Namespace"sv]; + CbFieldView BucketField = KeyView["Bucket"sv]; + CbFieldView HashField = KeyView["Hash"sv]; + CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -981,7 +1008,8 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack } else { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, ToString(HttpContentType::kCbPackage), @@ -1002,7 +1030,8 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack return PutResult::Invalid; } - ZEN_DEBUG("PUT - '{}/{}' {}, attachments '{}/{}/{}' (new/valid/total)", + ZEN_DEBUG("PUT - '{}/{}/{}' {}, attachments '{}/{}/{}' (new/valid/total)", + Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, NiceBytes(TransferredSize), @@ -1014,7 +1043,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack CacheValue.Value = IoBuffer(Record.GetSize()); Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize())); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Request.Key.Bucket, Request.Key.Hash, CacheValue); + m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; @@ -1080,13 +1109,16 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt for (CbFieldView RequestField : RequestsArray) { - RecordRequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + RecordRequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyObject["Namespace"sv]; + CbFieldView BucketField = KeyObject["Bucket"sv]; + CbFieldView HashField = KeyObject["Hash"sv]; + CacheKey& Key = Request.Upstream.Key; + Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (HashField.HasError() || Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1100,7 +1132,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt ZenCacheValue RecordCacheValue; if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && - m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, RecordCacheValue)) + m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, RecordCacheValue)) { Request.RecordCacheValue = std::move(RecordCacheValue.Value); if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject) @@ -1231,7 +1263,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt Request.RecordObject = ObjectBuffer; if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal)) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); + m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); } ParseValues(Request); Request.UsedUpstream = true; @@ -1269,7 +1301,11 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt } else { - ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}'", Value.ContentId, Key.Bucket, Key.Hash); + ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}/{}'", + Value.ContentId, + Key.Namespace, + Key.Bucket, + Key.Hash); } } if (!Value.Exists && !EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData)) @@ -1306,7 +1342,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt } } - ZEN_DEBUG("HIT - '{}/{}' {}{}{}", + ZEN_DEBUG("HIT - '{}/{}/{}' {}{}{}", + Key.Namespace, Key.Bucket, Key.Hash, NiceBytes(Request.RecordCacheValue.Size()), @@ -1322,11 +1359,11 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt if (!EnumHasAnyFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::Query)) { // If they requested no query, do not record this as a miss - ZEN_DEBUG("DISABLEDQUERY - '{}/{}'", Key.Bucket, Key.Hash); + ZEN_DEBUG("DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash); } else { - ZEN_DEBUG("MISS - '{}/{}' {}", Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv); + ZEN_DEBUG("MISS - '{}/{}/{}' {}", Key.Namespace, Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv); m_CacheStats.MissCount++; } } @@ -1357,11 +1394,14 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyView["Namespace"sv]; + CbFieldView BucketField = KeyView["Bucket"sv]; + CbFieldView HashField = KeyView["Hash"sv]; + CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -1388,21 +1428,21 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ { IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); Value.SetContentType(ZenContentType::kCompressedBinary); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Value}); + m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Value}); TransferredSize = Chunk.GetCompressedSize(); } Succeeded = true; } else { - ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}' FAILED, value is not compressed", Key.Bucket, Key.Hash, RawHash); + ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}/{}' FAILED, value is not compressed", Key.Namespace, Key.Bucket, Key.Hash, RawHash); return Request.WriteResponse(HttpResponseCode::BadRequest); } } else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { ZenCacheValue ExistingValue; - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, ExistingValue) && + if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType())) { Succeeded = true; @@ -1416,7 +1456,12 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCompressedBinary, .Key = Key}); } Results.push_back(Succeeded); - ZEN_DEBUG("PUTCACHEVALUES - '{}/{}' {}, '{}'", Key.Bucket, Key.Hash, NiceBytes(TransferredSize), Succeeded ? "Added"sv : "Invalid"); + ZEN_DEBUG("PUTCACHEVALUES - '{}/{}/{}' {}, '{}'", + Key.Namespace, + Key.Bucket, + Key.Hash, + NiceBytes(TransferredSize), + Succeeded ? "Added"sv : "Invalid"); } if (Results.empty()) { @@ -1466,12 +1511,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http { Stopwatch Timer; - RequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + RequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyObject["Namespace"sv]; + CbFieldView BucketField = KeyObject["Bucket"sv]; + CbFieldView HashField = KeyObject["Hash"sv]; + Request.Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1486,15 +1534,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http ZenCacheValue CacheValue; if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, CacheValue) && - IsCompressedBinary(CacheValue.Value.GetContentType())) + if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) { Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value)); } } if (Result) { - ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", + ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", + Key.Namespace, Key.Bucket, Key.Hash, NiceBytes(Result.GetCompressed().GetSize()), @@ -1509,11 +1557,12 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http else if (!EnumHasAnyFlags(Policy, CachePolicy::Query)) { // If they requested no query, do not record this as a miss - ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}'", Key.Bucket, Key.Hash); + ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash); } else { - ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}", + ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}", + Key.Namespace, Key.Bucket, Key.Hash, "LOCAL"sv, @@ -1531,7 +1580,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http for (size_t Index : RemoteRequestIndexes) { RequestData& Request = Requests[Index]; - RequestedRecordsData.push_back({{Request.Key.Bucket, Request.Key.Hash}}); + RequestedRecordsData.push_back({{Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash}}); CacheChunkRequests.push_back(&RequestedRecordsData.back()); } Stopwatch Timer; @@ -1551,11 +1600,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http // that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive // for us to keep the data only on the upstream server. // if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, - Request.Key.Bucket, - Request.Key.Hash, - ZenCacheValue{Params.Value}); - ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", + m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); + ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", + ChunkRequest.Key.Namespace, ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, NiceBytes(Request.Result.GetCompressed().GetSize()), @@ -1566,7 +1613,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http return; } } - ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}", + ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}", + ChunkRequest.Key.Namespace, ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, "UPSTREAM"sv, @@ -1723,9 +1771,12 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vectorKey.Namespace, PreviousRecordKey->Key.Bucket, PreviousRecordKey->Key.Hash); return false; @@ -1810,7 +1863,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) + if (m_CacheStore.Get(RecordKey.Key.Namespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) { Record.Exists = true; Record.CacheValue = std::move(CacheValue.Value); @@ -1845,7 +1898,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal)) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); + m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); } }; m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete)); @@ -1942,7 +1995,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vectorExists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) + if (m_CacheStore.Get(Request->Key->Key.Namespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) { if (IsCompressedBinary(CacheValue.Value.GetContentType())) { @@ -2011,7 +2064,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vectorKey.Namespace, Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId, @@ -2069,11 +2123,19 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(std::vectorKey.Bucket, Request.Key->Key.Hash, Request.Key->ValueId); + ZEN_DEBUG("SKIP - '{}/{}/{}/{}'", + Request.Key->Key.Namespace, + Request.Key->Key.Bucket, + Request.Key->Key.Hash, + Request.Key->ValueId); } else { - ZEN_DEBUG("MISS - '{}/{}/{}'", Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId); + ZEN_DEBUG("MISS - '{}/{}/{}/{}'", + Request.Key->Key.Namespace, + Request.Key->Key.Bucket, + Request.Key->Key.Hash, + Request.Key->ValueId); m_CacheStats.MissCount++; } } -- cgit v1.2.3 From 1b681b96424e0c25e26a2dbc24b42038539ac5af Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 May 2022 09:53:00 +0200 Subject: cleanup --- zenserver/cache/structuredcache.cpp | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 74438ac2e..98272722c 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -140,8 +140,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) { // Bucket reference - - return HandleCacheBucketRequest(Request, ZenCacheStore::DefaultNamespace, Key); + return HandleCacheBucketRequest(Request, Key); } return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL @@ -162,7 +161,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Key) { switch (Request.RequestVerb()) { @@ -175,14 +174,19 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, case HttpVerb::kDelete: // Drop bucket - - if (m_CacheStore.DropBucket(Namespace, Bucket)) - { - return Request.WriteResponse(HttpResponseCode::OK); - } - else { - return Request.WriteResponse(HttpResponseCode::NotFound); + // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability + std::string_view Namespace = ZenCacheStore::DefaultNamespace; + std::string_view Bucket = Key; + + if (m_CacheStore.DropBucket(Namespace, Bucket)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } } break; @@ -790,7 +794,8 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& return false; } - OutRef.Namespace = ToLower(""); // TODO: Should we add namespace to URI? + OutRef.Namespace = ToLower(ZenCacheStore::DefaultNamespace); // TODO: Should add namespace to URI and handle if the namespace is + // missing for backwards compatability OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) @@ -937,9 +942,8 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req CbFieldView NamespaceField = KeyView["Namespace"sv]; CbFieldView BucketField = KeyView["Bucket"sv]; CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + CacheKey Key = + CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -1116,9 +1120,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt CbFieldView BucketField = KeyObject["Bucket"sv]; CbFieldView HashField = KeyObject["Hash"sv]; CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (HashField.HasError() || Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1399,9 +1401,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ CbFieldView NamespaceField = KeyView["Namespace"sv]; CbFieldView BucketField = KeyView["Bucket"sv]; CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + CacheKey Key = + CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -1517,9 +1518,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http CbFieldView NamespaceField = KeyObject["Namespace"sv]; CbFieldView BucketField = KeyObject["Bucket"sv]; CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + Request.Key = + CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1774,9 +1774,9 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector Date: Thu, 12 May 2022 10:08:12 +0200 Subject: revert Bucket - Key parameter change --- zenserver/cache/structuredcache.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 98272722c..eed7a4420 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -161,7 +161,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Key) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) { switch (Request.RequestVerb()) { @@ -177,7 +177,6 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, { // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability std::string_view Namespace = ZenCacheStore::DefaultNamespace; - std::string_view Bucket = Key; if (m_CacheStore.DropBucket(Namespace, Bucket)) { -- cgit v1.2.3