From 4e538014c4e5089f10aca69657a948e4d912292d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 26 Apr 2022 23:18:44 +0200 Subject: Use GetCacheValues when checking upstream to reduce number of calls to upstream Added some timing info to debug logs --- zenserver/cache/structuredcache.cpp | 148 ++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 47 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 8ae531720..8daf08bff 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -650,42 +650,52 @@ HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, void HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) { + Stopwatch Timer; + IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); bool InUpstreamCache = false; CachePolicy Policy = PolicyFromURL; - const bool QueryUpstream = !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); - - if (QueryUpstream) { - if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); - UpstreamResult.Success) + const bool QueryUpstream = !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); + + if (QueryUpstream) { - if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) - { - m_CidStore.AddChunk(Compressed); - InUpstreamCache = true; - } - else + if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); + UpstreamResult.Success) { - ZEN_WARN("got uncompressed upstream cache value"); + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) + { + m_CidStore.AddChunk(Compressed); + InUpstreamCache = true; + } + else + { + ZEN_WARN("got uncompressed upstream cache value"); + } } } } if (!Value) { - ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, ToString(Request.AcceptContentType())); + ZEN_DEBUG("MISS - '{}/{}/{}' '{}' in {}", + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + ToString(Request.AcceptContentType()), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); m_CacheStats.MissCount++; return Request.WriteResponse(HttpResponseCode::NotFound); } - ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({}) in {}", Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, NiceBytes(Value.Size()), ToString(Value.GetContentType()), - InUpstreamCache ? "UPSTREAM" : "LOCAL"); + InUpstreamCache ? "UPSTREAM" : "LOCAL", + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); m_CacheStats.HitCount++; if (InUpstreamCache) @@ -709,6 +719,8 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored ZEN_UNUSED(PolicyFromURL); + Stopwatch Timer; + IoBuffer Body = Request.ReadPayload(); if (!Body || Body.Size() == 0) @@ -734,13 +746,14 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed); - ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({})", + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({}) in {}", Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, NiceBytes(Body.Size()), ToString(Body.GetContentType()), - Result.New ? "NEW" : "OLD"); + Result.New ? "NEW" : "OLD", + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); const HttpResponseCode ResponseCode = Result.New ? HttpResponseCode::Created : HttpResponseCode::OK; @@ -1444,8 +1457,12 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheValues"sv); + std::vector RemoteRequestIndexes; + for (CbFieldView RequestField : Params["Requests"sv]) { + Stopwatch Timer; + RequestData& Request = Requests.emplace_back(); CbObjectView RequestObject = RequestField.AsObjectView(); CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); @@ -1463,46 +1480,28 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http CachePolicy Policy = Request.Policy; CompressedBuffer& Result = Request.Result; - ZenCacheValue CacheValue; - std::string_view Source; + ZenCacheValue CacheValue; if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) { Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value)); - if (Result) - { - Source = "LOCAL"sv; - } - } - } - if (!Result && EnumHasAllFlags(Policy, CachePolicy::QueryRemote)) - { - GetUpstreamCacheResult UpstreamResult = - m_UpstreamCache.GetCacheRecord({Key.Bucket, Key.Hash}, ZenContentType::kCompressedBinary); - if (UpstreamResult.Success && IsCompressedBinary(UpstreamResult.Value.GetContentType())) - { - Result = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value)); - if (Result) - { - UpstreamResult.Value.SetContentType(ZenContentType::kCompressedBinary); - Source = "UPSTREAM"sv; - // TODO: Respect the StoreLocal flag once we have upstream existence-only checks. For now the requirement - // that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive - // for us to keep the data only on the upstream server. - // if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) - { - m_CacheStore.Put(Key.Bucket, Key.Hash, ZenCacheValue{UpstreamResult.Value}); - } - } } } - if (Result) { - ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({})", Key.Bucket, Key.Hash, NiceBytes(Result.GetCompressed().GetSize()), Source); + ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", + Key.Bucket, + Key.Hash, + NiceBytes(Result.GetCompressed().GetSize()), + "LOCAL"sv, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); m_CacheStats.HitCount++; } + else if (EnumHasAllFlags(Policy, CachePolicy::QueryRemote)) + { + RemoteRequestIndexes.push_back(Requests.size() - 1); + } else if (!EnumHasAnyFlags(Policy, CachePolicy::Query)) { // If they requested no query, do not record this as a miss @@ -1510,10 +1509,65 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http } else { - ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}'", Key.Bucket, Key.Hash); + ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}", + Key.Bucket, + Key.Hash, + "LOCAL"sv, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); m_CacheStats.MissCount++; } } + + if (!RemoteRequestIndexes.empty()) + { + std::vector RequestedRecordsData; + std::vector CacheChunkRequests; + RequestedRecordsData.reserve(RemoteRequestIndexes.size()); + CacheChunkRequests.reserve(RemoteRequestIndexes.size()); + for (size_t Index : RemoteRequestIndexes) + { + RequestData& Request = Requests[Index]; + RequestedRecordsData.push_back({Request.Key.Bucket, Request.Key.Hash}); + CacheChunkRequests.push_back(&RequestedRecordsData.back()); + } + Stopwatch Timer; + m_UpstreamCache.GetCacheValues( + CacheChunkRequests, + [this, &RequestedRecordsData, &Requests, &RemoteRequestIndexes, &Timer](CacheValueGetCompleteParams&& Params) { + CacheChunkRequest& ChunkRequest = Params.Request; + if (Params.Value) + { + size_t RequestOffset = std::distance(RequestedRecordsData.data(), &ChunkRequest); + size_t RequestIndex = RemoteRequestIndexes[RequestOffset]; + RequestData& Request = Requests[RequestIndex]; + Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value)); + if (Request.Result && IsCompressedBinary(Params.Value.GetContentType())) + { + // TODO: Respect the StoreLocal flag once we have upstream existence-only checks. For now the requirement + // that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive + // for us to keep the data only on the upstream server. + // if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) + m_CacheStore.Put(Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); + ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", + ChunkRequest.Key.Bucket, + ChunkRequest.Key.Hash, + NiceBytes(Request.Result.GetCompressed().GetSize()), + "UPSTREAM"sv, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + m_CacheStats.HitCount++; + m_CacheStats.UpstreamHitCount++; + return; + } + } + ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}", + ChunkRequest.Key.Bucket, + ChunkRequest.Key.Hash, + "UPSTREAM"sv, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + m_CacheStats.MissCount++; + }); + } + if (Requests.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); -- cgit v1.2.3 From cfbd92d50918d45583d970baf938c0e634e12d95 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 29 Apr 2022 08:57:01 +0200 Subject: mac compilation fix --- zenserver/cache/structuredcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 8daf08bff..e1d9de976 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -1527,7 +1527,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http for (size_t Index : RemoteRequestIndexes) { RequestData& Request = Requests[Index]; - RequestedRecordsData.push_back({Request.Key.Bucket, Request.Key.Hash}); + RequestedRecordsData.push_back({{Request.Key.Bucket, Request.Key.Hash}}); CacheChunkRequests.push_back(&RequestedRecordsData.back()); } Stopwatch Timer; -- cgit v1.2.3 From 5b95a4fba97aa66cec935ef3e0d969893223f9d6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 May 2022 15:25:35 +0200 Subject: Add namespacecachestore layer to allow multiple structured cache namespaces --- zenserver/cache/structuredcache.cpp | 50 +++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 24 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index e1d9de976..276c99081 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -21,6 +21,7 @@ //#include "cachekey.h" #include "monitoring/httpstats.h" +#include "namespacecachestore.h" #include "structuredcachestore.h" #include "upstream/jupiter.h" #include "upstream/upstreamcache.h" @@ -72,13 +73,13 @@ struct PutRequestData ////////////////////////////////////////////////////////////////////////// -HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, - CidStore& InCidStore, - HttpStatsService& StatsService, - HttpStatusService& StatusService, - UpstreamCache& UpstreamCache) +HttpStructuredCacheService::HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache) : m_Log(logging::Get("cache")) -, m_CacheStore(InCacheStore) +, m_CacheStore(InNamespaceCacheStore) , m_StatsService(StatsService) , m_StatusService(StatusService) , m_CidStore(InCidStore) @@ -176,7 +177,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, case HttpVerb::kDelete: // Drop bucket - if (m_CacheStore.DropBucket(Bucket)) + if (m_CacheStore.DropBucket("", Bucket)) { return Request.WriteResponse(HttpResponseCode::OK); } @@ -225,7 +226,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::OK); } - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, ClientResultValue)) + if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && m_CacheStore.Get("", Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { Success = true; ZenContentType ContentType = ClientResultValue.Value.GetContentType(); @@ -350,7 +351,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (Success && StoreLocal) { - m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, ClientResultValue); + m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, ClientResultValue); } } else if (AcceptType == ZenContentType::kCbPackage) @@ -404,7 +405,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (StoreLocal) { - m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, CacheValue); } BinaryWriter MemStream; @@ -486,7 +487,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) { ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); - m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, {.Value = Body}); if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote)) { @@ -528,7 +529,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ValidAttachments.size()); Body.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, {.Value = Body}); const bool IsPartialRecord = TotalCount != static_cast(ValidAttachments.size()); @@ -611,7 +612,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; @@ -1013,7 +1014,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack CacheValue.Value = IoBuffer(Record.GetSize()); Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize())); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(Request.Key.Bucket, Request.Key.Hash, CacheValue); + m_CacheStore.Put("", Request.Key.Bucket, Request.Key.Hash, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; @@ -1098,7 +1099,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt bool FoundLocalInvalid = false; ZenCacheValue RecordCacheValue; - if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && m_CacheStore.Get(Key.Bucket, Key.Hash, RecordCacheValue)) + if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && + m_CacheStore.Get("", Key.Bucket, Key.Hash, RecordCacheValue)) { Request.RecordCacheValue = std::move(RecordCacheValue.Value); if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject) @@ -1229,7 +1231,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt Request.RecordObject = ObjectBuffer; if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal)) { - m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); + m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); } ParseValues(Request); Request.UsedUpstream = true; @@ -1386,7 +1388,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ { IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); Value.SetContentType(ZenContentType::kCompressedBinary); - m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = Value}); + m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = Value}); TransferredSize = Chunk.GetCompressedSize(); } Succeeded = true; @@ -1400,7 +1402,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { ZenCacheValue ExistingValue; - if (m_CacheStore.Get(Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType())) + if (m_CacheStore.Get("", Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType())) { Succeeded = true; } @@ -1483,7 +1485,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http ZenCacheValue CacheValue; if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { - if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) + if (m_CacheStore.Get("", Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) { Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value)); } @@ -1547,7 +1549,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http // that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive // for us to keep the data only on the upstream server. // if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) - m_CacheStore.Put(Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); + m_CacheStore.Put("", Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, @@ -1803,7 +1805,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) + if (m_CacheStore.Get("", RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) { Record.Exists = true; Record.CacheValue = std::move(CacheValue.Value); @@ -1838,7 +1840,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal)) { - m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); + m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); } }; m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete)); @@ -1935,7 +1937,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vectorExists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) + if (m_CacheStore.Get("", Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) { if (IsCompressedBinary(CacheValue.Value.GetContentType())) { @@ -2004,7 +2006,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vector Date: Wed, 4 May 2022 19:45:57 +0200 Subject: cleanup --- zenserver/cache/structuredcache.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 276c99081..8deb958be 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -21,7 +21,6 @@ //#include "cachekey.h" #include "monitoring/httpstats.h" -#include "namespacecachestore.h" #include "structuredcachestore.h" #include "upstream/jupiter.h" #include "upstream/upstreamcache.h" @@ -73,13 +72,13 @@ struct PutRequestData ////////////////////////////////////////////////////////////////////////// -HttpStructuredCacheService::HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore, - CidStore& InCidStore, - HttpStatsService& StatsService, - HttpStatusService& StatusService, - UpstreamCache& UpstreamCache) +HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& CacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache) : m_Log(logging::Get("cache")) -, m_CacheStore(InNamespaceCacheStore) +, m_CacheStore(CacheStore) , m_StatsService(StatsService) , m_StatusService(StatusService) , m_CidStore(InCidStore) -- cgit v1.2.3 From 861a92d1ee6c54eeb9035190501baf8ea888591f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 May 2022 09:55:09 +0200 Subject: cleanup and review feedback --- zenserver/cache/structuredcache.cpp | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 8deb958be..69ee32bd6 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -176,7 +176,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, case HttpVerb::kDelete: // Drop bucket - if (m_CacheStore.DropBucket("", Bucket)) + if (m_CacheStore.DropBucket(ZenCacheStore::DefaultNamespace, Bucket)) { return Request.WriteResponse(HttpResponseCode::OK); } @@ -225,7 +225,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::OK); } - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && m_CacheStore.Get("", Ref.BucketSegment, Ref.HashKey, ClientResultValue)) + if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && + m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { Success = true; ZenContentType ContentType = ClientResultValue.Value.GetContentType(); @@ -350,7 +351,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (Success && StoreLocal) { - m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, ClientResultValue); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue); } } else if (AcceptType == ZenContentType::kCbPackage) @@ -404,7 +405,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (StoreLocal) { - m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue); } BinaryWriter MemStream; @@ -486,7 +487,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) { ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); - m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote)) { @@ -528,7 +529,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ValidAttachments.size()); Body.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); const bool IsPartialRecord = TotalCount != static_cast(ValidAttachments.size()); @@ -611,7 +612,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; @@ -1013,7 +1014,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack CacheValue.Value = IoBuffer(Record.GetSize()); Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize())); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put("", Request.Key.Bucket, Request.Key.Hash, CacheValue); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Request.Key.Bucket, Request.Key.Hash, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; @@ -1099,7 +1100,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt ZenCacheValue RecordCacheValue; if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && - m_CacheStore.Get("", Key.Bucket, Key.Hash, RecordCacheValue)) + m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, RecordCacheValue)) { Request.RecordCacheValue = std::move(RecordCacheValue.Value); if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject) @@ -1230,7 +1231,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt Request.RecordObject = ObjectBuffer; if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal)) { - m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); } ParseValues(Request); Request.UsedUpstream = true; @@ -1387,7 +1388,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ { IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); Value.SetContentType(ZenContentType::kCompressedBinary); - m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = Value}); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Value}); TransferredSize = Chunk.GetCompressedSize(); } Succeeded = true; @@ -1401,7 +1402,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { ZenCacheValue ExistingValue; - if (m_CacheStore.Get("", Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType())) + if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, ExistingValue) && + IsCompressedBinary(ExistingValue.Value.GetContentType())) { Succeeded = true; } @@ -1484,7 +1486,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http ZenCacheValue CacheValue; if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { - if (m_CacheStore.Get("", Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) + if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, CacheValue) && + IsCompressedBinary(CacheValue.Value.GetContentType())) { Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value)); } @@ -1548,7 +1551,10 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http // that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive // for us to keep the data only on the upstream server. // if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) - m_CacheStore.Put("", Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, + Request.Key.Bucket, + Request.Key.Hash, + ZenCacheValue{Params.Value}); ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, @@ -1804,7 +1810,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get("", RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) + if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) { Record.Exists = true; Record.CacheValue = std::move(CacheValue.Value); @@ -1839,7 +1845,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal)) { - m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); + m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); } }; m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete)); @@ -1936,7 +1942,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vectorExists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get("", Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) + if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) { if (IsCompressedBinary(CacheValue.Value.GetContentType())) { @@ -2005,7 +2011,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vector Date: Thu, 5 May 2022 11:01:44 +0200 Subject: reverted unnecessary changes --- zenserver/cache/structuredcache.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 69ee32bd6..0f16f6785 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -72,13 +72,13 @@ struct PutRequestData ////////////////////////////////////////////////////////////////////////// -HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& CacheStore, +HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, CidStore& InCidStore, HttpStatsService& StatsService, HttpStatusService& StatusService, UpstreamCache& UpstreamCache) : m_Log(logging::Get("cache")) -, m_CacheStore(CacheStore) +, m_CacheStore(InCacheStore) , m_StatsService(StatsService) , m_StatusService(StatusService) , m_CidStore(InCidStore) -- cgit v1.2.3 From 33fa76a35a96cad1865854068e60c1ca0b53864e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 6 May 2022 16:42:27 +0200 Subject: parameterize namespace for upstream (first hack) --- zenserver/cache/structuredcache.cpp | 240 +++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 89 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 0f16f6785..74438ac2e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -141,7 +141,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { // Bucket reference - return HandleCacheBucketRequest(Request, Key); + return HandleCacheBucketRequest(Request, ZenCacheStore::DefaultNamespace, Key); } return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL @@ -162,7 +162,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket) { switch (Request.RequestVerb()) { @@ -176,7 +176,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, case HttpVerb::kDelete: // Drop bucket - if (m_CacheStore.DropBucket(ZenCacheStore::DefaultNamespace, Bucket)) + if (m_CacheStore.DropBucket(Namespace, Bucket)) { return Request.WriteResponse(HttpResponseCode::OK); } @@ -226,7 +226,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && - m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) + m_CacheStore.Get(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { Success = true; ZenContentType ContentType = ClientResultValue.Value.GetContentType(); @@ -287,7 +287,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (Success) { - ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL)", + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' (LOCAL)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(ClientResultValue.Value.Size()), @@ -306,7 +307,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } else if (!EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryRemote)) { - ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); + ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); m_CacheStats.MissCount++; return Request.WriteResponse(HttpResponseCode::NotFound); } @@ -324,7 +325,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); - if (GetUpstreamCacheResult UpstreamResult = m_UpstreamCache.GetCacheRecord({Ref.BucketSegment, Ref.HashKey}, AcceptType); + if (GetUpstreamCacheResult UpstreamResult = + m_UpstreamCache.GetCacheRecord({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, AcceptType); UpstreamResult.Success) { Success = true; @@ -340,7 +342,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (ValidationResult != CbValidateError::None) { Success = false; - ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid compact binary object from upstream", + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid compact binary object from upstream", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); @@ -351,7 +354,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (Success && StoreLocal) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue); } } else if (AcceptType == ZenContentType::kCbPackage) @@ -405,7 +408,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (StoreLocal) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue); } BinaryWriter MemStream; @@ -434,14 +437,19 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request else { Success = false; - ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid upstream package", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid upstream package", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); } } } if (Success) { - ZEN_DEBUG("HIT - '{}/{}' {} '{}' (UPSTREAM)", + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' (UPSTREAM)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(ClientResultValue.Value.Size()), @@ -463,7 +471,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); + ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); m_CacheStats.MissCount++; AsyncRequest.WriteResponse(HttpResponseCode::NotFound); } @@ -486,12 +494,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) { - ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote)) { - m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.BucketSegment, Ref.HashKey}}); + m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}}); } Request.WriteResponse(HttpResponseCode::Created); @@ -502,7 +510,11 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ValidationResult != CbValidateError::None) { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid compact binary", Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid compact binary", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); } @@ -520,7 +532,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request TotalCount++; }); - ZEN_DEBUG("PUT - '{}/{}' {} '{}' attachments '{}/{}' (valid/total)", + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), @@ -529,14 +542,14 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ValidAttachments.size()); Body.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); const bool IsPartialRecord = TotalCount != static_cast(ValidAttachments.size()); if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, - .Key = {Ref.BucketSegment, Ref.HashKey}, + .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, .ValueContentIds = std::move(ValidAttachments)}); } @@ -548,7 +561,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (!Package.TryLoad(Body)) { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid package", Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid package", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); } CachePolicy Policy = PolicyFromURL; @@ -578,7 +591,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(HttpContentType::kCbPackage), @@ -599,7 +613,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); } - ZEN_DEBUG("PUT - '{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total)", + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total)", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.GetSize()), @@ -612,14 +627,14 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue); + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .Key = {Ref.BucketSegment, Ref.HashKey}, + .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, .ValueContentIds = std::move(ValidAttachments)}); } @@ -661,7 +676,7 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, if (QueryUpstream) { - if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); + if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); UpstreamResult.Success) { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) @@ -679,7 +694,8 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, if (!Value) { - ZEN_DEBUG("MISS - '{}/{}/{}' '{}' in {}", + ZEN_DEBUG("MISS - '{}/{}/{}/{}' '{}' in {}", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, @@ -689,7 +705,8 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, return Request.WriteResponse(HttpResponseCode::NotFound); } - ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({}) in {}", + ZEN_DEBUG("HIT - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, @@ -747,7 +764,8 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed); - ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({}) in {}", + ZEN_DEBUG("PUT - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, Ref.BucketSegment, Ref.HashKey, Ref.ValueContentId, @@ -772,8 +790,14 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& return false; } + OutRef.Namespace = ToLower(""); // TODO: Should we add namespace to URI? OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); + if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) + { + return false; + } + if (!std::all_of(begin(OutRef.BucketSegment), end(OutRef.BucketSegment), [](const char c) { return std::isalnum(c); })) { return false; @@ -907,12 +931,15 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); - CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); + CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyView["Namespace"sv]; + CbFieldView BucketField = KeyView["Bucket"sv]; + CbFieldView HashField = KeyView["Hash"sv]; + CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -981,7 +1008,8 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack } else { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", + ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, ToString(HttpContentType::kCbPackage), @@ -1002,7 +1030,8 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack return PutResult::Invalid; } - ZEN_DEBUG("PUT - '{}/{}' {}, attachments '{}/{}/{}' (new/valid/total)", + ZEN_DEBUG("PUT - '{}/{}/{}' {}, attachments '{}/{}/{}' (new/valid/total)", + Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, NiceBytes(TransferredSize), @@ -1014,7 +1043,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack CacheValue.Value = IoBuffer(Record.GetSize()); Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize())); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Request.Key.Bucket, Request.Key.Hash, CacheValue); + m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; @@ -1080,13 +1109,16 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt for (CbFieldView RequestField : RequestsArray) { - RecordRequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + RecordRequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyObject["Namespace"sv]; + CbFieldView BucketField = KeyObject["Bucket"sv]; + CbFieldView HashField = KeyObject["Hash"sv]; + CacheKey& Key = Request.Upstream.Key; + Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (HashField.HasError() || Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1100,7 +1132,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt ZenCacheValue RecordCacheValue; if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && - m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, RecordCacheValue)) + m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, RecordCacheValue)) { Request.RecordCacheValue = std::move(RecordCacheValue.Value); if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject) @@ -1231,7 +1263,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt Request.RecordObject = ObjectBuffer; if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal)) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); + m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); } ParseValues(Request); Request.UsedUpstream = true; @@ -1269,7 +1301,11 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt } else { - ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}'", Value.ContentId, Key.Bucket, Key.Hash); + ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}/{}'", + Value.ContentId, + Key.Namespace, + Key.Bucket, + Key.Hash); } } if (!Value.Exists && !EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData)) @@ -1306,7 +1342,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt } } - ZEN_DEBUG("HIT - '{}/{}' {}{}{}", + ZEN_DEBUG("HIT - '{}/{}/{}' {}{}{}", + Key.Namespace, Key.Bucket, Key.Hash, NiceBytes(Request.RecordCacheValue.Size()), @@ -1322,11 +1359,11 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt if (!EnumHasAnyFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::Query)) { // If they requested no query, do not record this as a miss - ZEN_DEBUG("DISABLEDQUERY - '{}/{}'", Key.Bucket, Key.Hash); + ZEN_DEBUG("DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash); } else { - ZEN_DEBUG("MISS - '{}/{}' {}", Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv); + ZEN_DEBUG("MISS - '{}/{}/{}' {}", Key.Namespace, Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv); m_CacheStats.MissCount++; } } @@ -1357,11 +1394,14 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyView["Namespace"sv]; + CbFieldView BucketField = KeyView["Bucket"sv]; + CbFieldView HashField = KeyView["Hash"sv]; + CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -1388,21 +1428,21 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ { IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); Value.SetContentType(ZenContentType::kCompressedBinary); - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Value}); + m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Value}); TransferredSize = Chunk.GetCompressedSize(); } Succeeded = true; } else { - ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}' FAILED, value is not compressed", Key.Bucket, Key.Hash, RawHash); + ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}/{}' FAILED, value is not compressed", Key.Namespace, Key.Bucket, Key.Hash, RawHash); return Request.WriteResponse(HttpResponseCode::BadRequest); } } else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { ZenCacheValue ExistingValue; - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, ExistingValue) && + if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType())) { Succeeded = true; @@ -1416,7 +1456,12 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCompressedBinary, .Key = Key}); } Results.push_back(Succeeded); - ZEN_DEBUG("PUTCACHEVALUES - '{}/{}' {}, '{}'", Key.Bucket, Key.Hash, NiceBytes(TransferredSize), Succeeded ? "Added"sv : "Invalid"); + ZEN_DEBUG("PUTCACHEVALUES - '{}/{}/{}' {}, '{}'", + Key.Namespace, + Key.Bucket, + Key.Hash, + NiceBytes(TransferredSize), + Succeeded ? "Added"sv : "Invalid"); } if (Results.empty()) { @@ -1466,12 +1511,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http { Stopwatch Timer; - RequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash()); + RequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + CbFieldView NamespaceField = KeyObject["Namespace"sv]; + CbFieldView BucketField = KeyObject["Bucket"sv]; + CbFieldView HashField = KeyObject["Hash"sv]; + Request.Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, + BucketField.AsString(), + HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1486,15 +1534,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http ZenCacheValue CacheValue; if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, CacheValue) && - IsCompressedBinary(CacheValue.Value.GetContentType())) + if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) { Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value)); } } if (Result) { - ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", + ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", + Key.Namespace, Key.Bucket, Key.Hash, NiceBytes(Result.GetCompressed().GetSize()), @@ -1509,11 +1557,12 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http else if (!EnumHasAnyFlags(Policy, CachePolicy::Query)) { // If they requested no query, do not record this as a miss - ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}'", Key.Bucket, Key.Hash); + ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash); } else { - ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}", + ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}", + Key.Namespace, Key.Bucket, Key.Hash, "LOCAL"sv, @@ -1531,7 +1580,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http for (size_t Index : RemoteRequestIndexes) { RequestData& Request = Requests[Index]; - RequestedRecordsData.push_back({{Request.Key.Bucket, Request.Key.Hash}}); + RequestedRecordsData.push_back({{Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash}}); CacheChunkRequests.push_back(&RequestedRecordsData.back()); } Stopwatch Timer; @@ -1551,11 +1600,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http // that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive // for us to keep the data only on the upstream server. // if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, - Request.Key.Bucket, - Request.Key.Hash, - ZenCacheValue{Params.Value}); - ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}", + m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); + ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", + ChunkRequest.Key.Namespace, ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, NiceBytes(Request.Result.GetCompressed().GetSize()), @@ -1566,7 +1613,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http return; } } - ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}", + ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}", + ChunkRequest.Key.Namespace, ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, "UPSTREAM"sv, @@ -1723,9 +1771,12 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vectorKey.Namespace, PreviousRecordKey->Key.Bucket, PreviousRecordKey->Key.Hash); return false; @@ -1810,7 +1863,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) + if (m_CacheStore.Get(RecordKey.Key.Namespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) { Record.Exists = true; Record.CacheValue = std::move(CacheValue.Value); @@ -1845,7 +1898,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal)) { - m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); + m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); } }; m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete)); @@ -1942,7 +1995,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vectorExists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) + if (m_CacheStore.Get(Request->Key->Key.Namespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) { if (IsCompressedBinary(CacheValue.Value.GetContentType())) { @@ -2011,7 +2064,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vectorKey.Namespace, Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId, @@ -2069,11 +2123,19 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(std::vectorKey.Bucket, Request.Key->Key.Hash, Request.Key->ValueId); + ZEN_DEBUG("SKIP - '{}/{}/{}/{}'", + Request.Key->Key.Namespace, + Request.Key->Key.Bucket, + Request.Key->Key.Hash, + Request.Key->ValueId); } else { - ZEN_DEBUG("MISS - '{}/{}/{}'", Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId); + ZEN_DEBUG("MISS - '{}/{}/{}/{}'", + Request.Key->Key.Namespace, + Request.Key->Key.Bucket, + Request.Key->Key.Hash, + Request.Key->ValueId); m_CacheStats.MissCount++; } } -- cgit v1.2.3 From 1b681b96424e0c25e26a2dbc24b42038539ac5af Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 May 2022 09:53:00 +0200 Subject: cleanup --- zenserver/cache/structuredcache.cpp | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 74438ac2e..98272722c 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -140,8 +140,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) { // Bucket reference - - return HandleCacheBucketRequest(Request, ZenCacheStore::DefaultNamespace, Key); + return HandleCacheBucketRequest(Request, Key); } return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL @@ -162,7 +161,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Key) { switch (Request.RequestVerb()) { @@ -175,14 +174,19 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, case HttpVerb::kDelete: // Drop bucket - - if (m_CacheStore.DropBucket(Namespace, Bucket)) - { - return Request.WriteResponse(HttpResponseCode::OK); - } - else { - return Request.WriteResponse(HttpResponseCode::NotFound); + // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability + std::string_view Namespace = ZenCacheStore::DefaultNamespace; + std::string_view Bucket = Key; + + if (m_CacheStore.DropBucket(Namespace, Bucket)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } } break; @@ -790,7 +794,8 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& return false; } - OutRef.Namespace = ToLower(""); // TODO: Should we add namespace to URI? + OutRef.Namespace = ToLower(ZenCacheStore::DefaultNamespace); // TODO: Should add namespace to URI and handle if the namespace is + // missing for backwards compatability OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) @@ -937,9 +942,8 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req CbFieldView NamespaceField = KeyView["Namespace"sv]; CbFieldView BucketField = KeyView["Bucket"sv]; CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + CacheKey Key = + CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -1116,9 +1120,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt CbFieldView BucketField = KeyObject["Bucket"sv]; CbFieldView HashField = KeyObject["Hash"sv]; CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (HashField.HasError() || Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1399,9 +1401,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ CbFieldView NamespaceField = KeyView["Namespace"sv]; CbFieldView BucketField = KeyView["Bucket"sv]; CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + CacheKey Key = + CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); @@ -1517,9 +1518,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http CbFieldView NamespaceField = KeyObject["Namespace"sv]; CbFieldView BucketField = KeyObject["Bucket"sv]; CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = CacheKey::Create(NamespaceField ? NamespaceField.AsString() : ZenCacheStore::DefaultNamespace, - BucketField.AsString(), - HashField.AsHash()); + Request.Key = + CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); @@ -1774,9 +1774,9 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector Date: Thu, 12 May 2022 10:08:12 +0200 Subject: revert Bucket - Key parameter change --- zenserver/cache/structuredcache.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 98272722c..eed7a4420 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -161,7 +161,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Key) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) { switch (Request.RequestVerb()) { @@ -177,7 +177,6 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, { // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability std::string_view Namespace = ZenCacheStore::DefaultNamespace; - std::string_view Bucket = Key; if (m_CacheStore.DropBucket(Namespace, Bucket)) { -- cgit v1.2.3 From 1b235fc947589dfbac0d09024947c37171c5dc7f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 13:04:59 +0200 Subject: Add support for /api/v2/ URI requests with namespace support --- zenserver/cache/structuredcache.cpp | 177 +++++++++++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 25 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index eed7a4420..691da36fa 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -121,47 +121,148 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx) m_CacheStore.Scrub(Ctx); } -void -HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) +static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; +static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + +struct HttpRequestData { - CacheRef Ref; + std::optional Namespace; + std::optional Bucket; + std::optional HashKey; + std::optional ValueContentId; +}; - metrics::OperationTiming::Scope $(m_HttpRequests); +static bool +HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) +{ + std::string_view Namespace = ZenCacheStore::DefaultNamespace; + if (Key.starts_with(HttpZCacheAPIV2Prefix)) + { + std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/', HttpZCacheAPIV2Prefix.length()); + if (NamespaceSplitOffset == std::string_view::npos) + { + // Namespace reference + if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Namespace = ToLower(Key); + return true; + } + Data.Namespace = Key.substr(0, NamespaceSplitOffset); + Key = Key.substr(NamespaceSplitOffset + 1); + } + + std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); + if (BucketSplitOffset == std::string_view::npos) + { + if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Bucket = ToLower(Key); + return true; + } + + std::string_view HashSegment; + std::string_view ValueSegment; + + std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); + if (ValueSplitOffset == BucketSplitOffset) + { + // Basic cache record lookup + HashSegment = Key.substr(BucketSplitOffset + 1); + } + else + { + // Cache record + valueid lookup + HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); + ValueSegment = Key.substr(ValueSplitOffset + 1); + } - if (!ValidateKeyUri(Request, /* out */ Ref)) + if (HashSegment.size() != IoHash::StringLength) { - std::string_view Key = Request.RelativeUri(); + return false; + } + + IoHash KeyHash; + if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) + { + return false; + } - if (Key == "$rpc") + Data.HashKey = KeyHash; + + if (!ValueSegment.empty()) + { + if (ValueSegment.size() != IoHash::StringLength) { - return HandleRpcRequest(Request); + return false; } - if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) + IoHash ValueHash; + if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) { - // Bucket reference - return HandleCacheBucketRequest(Request, Key); + return false; } + Data.ValueContentId = ValueHash; + } + + return true; +} + +void +HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) +{ + metrics::OperationTiming::Scope $(m_HttpRequests); + + std::string_view Key = Request.RelativeUri(); + if (Key == HttpZCacheRPCPrefix) + { + return HandleRpcRequest(Request); + } + HttpRequestData RequestData; + if (!HttpRequestParseRelativeUri(Key, RequestData)) + { return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL } - CachePolicy PolicyFromURL = ParseCachePolicy(Request.GetQueryParams()); + if (RequestData.ValueContentId.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + ZEN_ASSERT(RequestData.HashKey.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = RequestData.ValueContentId.value()}; + return HandleCacheValueRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); + } - if (Ref.ValueContentId == IoHash::Zero) + if (RequestData.HashKey.has_value()) { - return HandleCacheRecordRequest(Request, Ref, PolicyFromURL); + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = IoHash::Zero}; + return HandleCacheRecordRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); } - else + + if (RequestData.Bucket.has_value()) { - return HandleCacheValueRequest(Request, Ref, PolicyFromURL); + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheBucketRequest(Request, RequestData.Namespace.value(), RequestData.Bucket.value()); } - return; + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheNamespaceRequest(Request, RequestData.Namespace.value()); } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view) { switch (Request.RequestVerb()) { @@ -173,11 +274,39 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, break; case HttpVerb::kDelete: - // Drop bucket + // Drop namespace { - // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability - std::string_view Namespace = ZenCacheStore::DefaultNamespace; + // if (m_CacheStore.DropNamespace(Namespace)) + // { + // return Request.WriteResponse(HttpResponseCode::OK); + // } + // else + // { + // return Request.WriteResponse(HttpResponseCode::NotFound); + // } + } + break; + default: + break; + } +} + +void +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + // Query stats + } + break; + + case HttpVerb::kDelete: + // Drop bucket + { if (m_CacheStore.DropBucket(Namespace, Bucket)) { return Request.WriteResponse(HttpResponseCode::OK); @@ -783,9 +912,8 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, } bool -HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& OutRef) +HttpStructuredCacheService::ValidateKeyUri(std::string_view Namespace, std::string_view Key, CacheRef& OutRef) { - std::string_view Key = Request.RelativeUri(); std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); if (BucketSplitOffset == std::string_view::npos) @@ -793,8 +921,7 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& return false; } - OutRef.Namespace = ToLower(ZenCacheStore::DefaultNamespace); // TODO: Should add namespace to URI and handle if the namespace is - // missing for backwards compatability + OutRef.Namespace = ToLower(Namespace); OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) -- cgit v1.2.3 From 1274f92cf7ce890b7aa1fc9354503e2508c185eb Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 13:48:45 +0200 Subject: Tests for HttpRequestParseRelativeUri --- zenserver/cache/structuredcache.cpp | 168 ++++++++++++++++++++++++++++++------ 1 file changed, 142 insertions(+), 26 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 691da36fa..9299911cf 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -35,6 +35,11 @@ #include +#if ZEN_WITH_TESTS +# include +# include +#endif + namespace zen { using namespace std::literals; @@ -135,32 +140,51 @@ struct HttpRequestData static bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) { - std::string_view Namespace = ZenCacheStore::DefaultNamespace; if (Key.starts_with(HttpZCacheAPIV2Prefix)) { - std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/', HttpZCacheAPIV2Prefix.length()); + Key = Key.substr(HttpZCacheAPIV2Prefix.length()); + // Namespace reference + if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) + { + return false; + } + Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); + std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); + + std::string_view Namespace = Key.substr(0, NamespaceSplitOffset); + if (!std::all_of(begin(Namespace), end(Namespace), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Namespace = ToLower(Namespace); + if (NamespaceSplitOffset == std::string_view::npos) { - // Namespace reference - if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Namespace = ToLower(Key); return true; } - Data.Namespace = Key.substr(0, NamespaceSplitOffset); - Key = Key.substr(NamespaceSplitOffset + 1); + + Key = Key.substr(NamespaceSplitOffset + 1); + } + else + { + Data.Namespace = ZenCacheStore::DefaultNamespace; } std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); + std::string_view Bucket = Key.substr(0, BucketSplitOffset); + if (Bucket.empty()) + { + return false; + } + if (!std::all_of(begin(Bucket), end(Bucket), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Bucket = ToLower(Bucket); + if (BucketSplitOffset == std::string_view::npos) { - if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Bucket = ToLower(Key); + // Bucket reference return true; } @@ -193,20 +217,22 @@ HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) Data.HashKey = KeyHash; - if (!ValueSegment.empty()) + if (ValueSegment.empty()) { - if (ValueSegment.size() != IoHash::StringLength) - { - return false; - } + return true; + } - IoHash ValueHash; - if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) - { - return false; - } - Data.ValueContentId = ValueHash; + if (ValueSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash ValueHash; + if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) + { + return false; } + Data.ValueContentId = ValueHash; return true; } @@ -2330,4 +2356,94 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request) Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +#if ZEN_WITH_TESTS + +TEST_CASE("z$service.parse.relative.Uri") +{ + HttpRequestData LegacyBucketRequest; + CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequest)); + CHECK(LegacyBucketRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyBucketRequest.Bucket == "test"sv); + CHECK(!LegacyBucketRequest.HashKey.has_value()); + CHECK(!LegacyBucketRequest.ValueContentId.has_value()); + + HttpRequestData LegacyHashKeyRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); + CHECK(LegacyHashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyHashKeyRequest.Bucket == "test"sv); + CHECK(LegacyHashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(!LegacyHashKeyRequest.ValueContentId.has_value()); + + HttpRequestData LegacyValueContentIdRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + LegacyValueContentIdRequest)); + CHECK(LegacyValueContentIdRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyValueContentIdRequest.Bucket == "test"sv); + CHECK(LegacyValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); + + HttpRequestData V2DefaultNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_", V2DefaultNamespaceRequest)); + CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); + CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); + CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2NamespaceRequest; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace", V2NamespaceRequest)); + CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); + CHECK(!V2NamespaceRequest.Bucket.has_value()); + CHECK(!V2NamespaceRequest.HashKey.has_value()); + CHECK(!V2NamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithDefaultNamespace; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test", V2BucketRequestWithDefaultNamespace)); + CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); + CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithNamespace; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); + CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); + CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); + + HttpRequestData V2HashKeyRequest; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(V2HashKeyRequest.Bucket == "test"); + CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(!V2HashKeyRequest.ValueContentId.has_value()); + + HttpRequestData V2ValueContentIdRequest; + CHECK(HttpRequestParseRelativeUri( + "api/v2/ns_nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + V2ValueContentIdRequest)); + CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); + CHECK(V2ValueContentIdRequest.Bucket == "test"sv); + CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); + + HttpRequestData Invalid; + CHECK(!HttpRequestParseRelativeUri("api/v2/bla", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2//", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri( + "api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + Invalid)); +} + +#endif + +void +z$service_forcelink() +{ +} + } // namespace zen -- cgit v1.2.3 From 4090ba3269bbebeb7dc772bd15b632560a7202b8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 15:23:57 +0200 Subject: Add validation to namespace names that follows Jupiters rules. Add unified validation of Namespace, Bucket and Hash for rpc requests. cleanup --- zenserver/cache/structuredcache.cpp | 417 +++++++++++++++++------------------- 1 file changed, 192 insertions(+), 225 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 9299911cf..06114ed1e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -75,6 +75,169 @@ struct PutRequestData CacheRecordPolicy Policy; }; +namespace { + static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; + static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + + struct HttpRequestData + { + std::optional Namespace; + std::optional Bucket; + std::optional HashKey; + std::optional ValueContentId; + }; + + bool IsValidNamespaceName(std::string_view Name) + { + if (Name == ZenCacheStore::DefaultNamespace) + { + return true; + } + if (Name.empty()) + { + return false; + } + return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + } + + bool IsValidBucketName(std::string_view Name) + { + if (Name.empty()) + { + return false; + } + return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + } + + bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + { + if (Key.starts_with(HttpZCacheAPIV2Prefix)) + { + Key = Key.substr(HttpZCacheAPIV2Prefix.length()); + // Namespace reference + if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) + { + return false; + } + Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); + std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); + + std::string Namespace = ToLower(Key.substr(0, NamespaceSplitOffset)); + if (!IsValidNamespaceName(Namespace)) + { + return false; + } + + Data.Namespace = Namespace; + + if (NamespaceSplitOffset == std::string_view::npos) + { + return true; + } + Key = Key.substr(NamespaceSplitOffset + 1); + } + else + { + Data.Namespace = ZenCacheStore::DefaultNamespace; + } + + std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); + std::string Bucket = ToLower(Key.substr(0, BucketSplitOffset)); + if (!IsValidBucketName(Bucket)) + { + return false; + } + Data.Bucket = Bucket; + + if (BucketSplitOffset == std::string_view::npos) + { + // Bucket reference + return true; + } + + std::string_view HashSegment; + std::string_view ValueSegment; + + std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); + if (ValueSplitOffset == BucketSplitOffset) + { + // Basic cache record lookup + HashSegment = Key.substr(BucketSplitOffset + 1); + } + else + { + // Cache record + valueid lookup + HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); + ValueSegment = Key.substr(ValueSplitOffset + 1); + } + + if (HashSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash KeyHash; + if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) + { + return false; + } + + Data.HashKey = KeyHash; + + if (ValueSegment.empty()) + { + return true; + } + + if (ValueSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash ValueHash; + if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) + { + return false; + } + Data.ValueContentId = ValueHash; + + return true; + } + + bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { + CbFieldView NamespaceField = KeyView["Namespace"sv]; + std::string_view Namespace = NamespaceField.AsString(ZenCacheStore::DefaultNamespace); + CbFieldView BucketField = KeyView["Bucket"sv]; + if (BucketField.HasError()) + { + return false; + } + if (!BucketField.IsString()) + { + return false; + } + std::string_view Bucket = BucketField.AsString(); + if (!IsValidBucketName(Bucket)) + { + return false; + } + CbFieldView HashField = KeyView["Hash"sv]; + if (HashField.HasError()) + { + return false; + } + if (!HashField.IsHash()) + { + return false; + } + IoHash Hash = HashField.AsHash(); + Key = CacheKey::Create(Namespace, Bucket, Hash); + return true; + } + +} // namespace + ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, @@ -126,117 +289,6 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx) m_CacheStore.Scrub(Ctx); } -static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; -static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; - -struct HttpRequestData -{ - std::optional Namespace; - std::optional Bucket; - std::optional HashKey; - std::optional ValueContentId; -}; - -static bool -HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) -{ - if (Key.starts_with(HttpZCacheAPIV2Prefix)) - { - Key = Key.substr(HttpZCacheAPIV2Prefix.length()); - // Namespace reference - if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) - { - return false; - } - Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); - std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); - - std::string_view Namespace = Key.substr(0, NamespaceSplitOffset); - if (!std::all_of(begin(Namespace), end(Namespace), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Namespace = ToLower(Namespace); - - if (NamespaceSplitOffset == std::string_view::npos) - { - return true; - } - - Key = Key.substr(NamespaceSplitOffset + 1); - } - else - { - Data.Namespace = ZenCacheStore::DefaultNamespace; - } - - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - std::string_view Bucket = Key.substr(0, BucketSplitOffset); - if (Bucket.empty()) - { - return false; - } - if (!std::all_of(begin(Bucket), end(Bucket), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Bucket = ToLower(Bucket); - - if (BucketSplitOffset == std::string_view::npos) - { - // Bucket reference - return true; - } - - std::string_view HashSegment; - std::string_view ValueSegment; - - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else - { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } - - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } - - IoHash KeyHash; - if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) - { - return false; - } - - Data.HashKey = KeyHash; - - if (ValueSegment.empty()) - { - return true; - } - - if (ValueSegment.size() != IoHash::StringLength) - { - return false; - } - - IoHash ValueHash; - if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) - { - return false; - } - Data.ValueContentId = ValueHash; - - return true; -} - void HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { @@ -937,77 +989,6 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, Request.WriteResponse(ResponseCode); } -bool -HttpStructuredCacheService::ValidateKeyUri(std::string_view Namespace, std::string_view Key, CacheRef& OutRef) -{ - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - - if (BucketSplitOffset == std::string_view::npos) - { - return false; - } - - OutRef.Namespace = ToLower(Namespace); - OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); - - if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - if (!std::all_of(begin(OutRef.BucketSegment), end(OutRef.BucketSegment), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - std::string_view HashSegment; - std::string_view ValueSegment; - - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - - // We know there is a slash so no need to check for npos return - - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else - { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } - - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } - - if (!ValueSegment.empty() && ValueSegment.size() == IoHash::StringLength) - { - const bool IsOk = ParseHexBytes(ValueSegment.data(), ValueSegment.size(), OutRef.ValueContentId.Hash); - - if (!IsOk) - { - return false; - } - } - else - { - OutRef.ValueContentId = IoHash::Zero; - } - - const bool IsOk = ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); - - if (!IsOk) - { - return false; - } - - return true; -} - void HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) { @@ -1088,15 +1069,12 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); - CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); + CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } @@ -1265,18 +1243,16 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt for (CbFieldView RequestField : RequestsArray) { - RecordRequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (HashField.HasError() || Key.Bucket.empty()) + RecordRequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + CacheKey& Key = Request.Upstream.Key; + if (!GetRpcRequestCacheKey(KeyObject, Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + Request.DownstreamPolicy = LoadCacheRecordPolicy(RequestObject["Policy"sv].AsObjectView(), DefaultPolicy); const CacheRecordPolicy& Policy = Request.DownstreamPolicy; @@ -1548,17 +1524,15 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); CachePolicy Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; IoHash RawHash = RequestObject["RawHash"sv].AsBinaryAttachment(); @@ -1664,18 +1638,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http { Stopwatch Timer; - RequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) + RequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + if (!GetRpcRequestCacheKey(KeyObject, Request.Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); Request.Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; @@ -1921,15 +1892,10 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vectorKey)) { ZEN_WARN("GetCacheChunks: Invalid key in ChunkRequest."); return false; @@ -2432,6 +2398,7 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(!HttpRequestParseRelativeUri("api/v2/ns_bad\2_namespace", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_nice/\2\1bucket", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); CHECK(!HttpRequestParseRelativeUri( -- cgit v1.2.3 From c9a2cc8afa11f3dfea05b91f0758ba3d0cae0784 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 15:33:18 +0200 Subject: keep compatability for valid bucket names --- zenserver/cache/structuredcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 06114ed1e..83f2f41d9 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -106,7 +106,7 @@ namespace { { return false; } - return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + return std::all_of(begin(Name), end(Name), [](const char c) { return std::isalnum(c); }); } bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) -- cgit v1.2.3 From e9466f9684d5479029d73fd2a60327d1daa89192 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 15:33:58 +0200 Subject: Validate max length for namespace name --- zenserver/cache/structuredcache.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 83f2f41d9..02cecf3e1 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -97,6 +97,10 @@ namespace { { return false; } + if (Name.length() > 64) + { + return false; + } return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; } -- cgit v1.2.3 From 4be7a0cb5b83732e780a9247636a27172f2fb3e1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 12:56:14 +0200 Subject: use "default" as the default namespace remove ns_ prefix for namespaces on disk and in requests --- zenserver/cache/structuredcache.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 02cecf3e1..9f0ff7408 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -2353,35 +2353,35 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData V2DefaultNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_", V2DefaultNamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("api/v2/default", V2DefaultNamespaceRequest)); CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); HttpRequestData V2NamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace", V2NamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace", V2NamespaceRequest)); CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); CHECK(!V2NamespaceRequest.Bucket.has_value()); CHECK(!V2NamespaceRequest.HashKey.has_value()); CHECK(!V2NamespaceRequest.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithDefaultNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test", V2BucketRequestWithDefaultNamespace)); + CHECK(HttpRequestParseRelativeUri("api/v2/default/test", V2BucketRequestWithDefaultNamespace)); CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace/test", V2BucketRequestWithNamespace)); CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); HttpRequestData V2HashKeyRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(HttpRequestParseRelativeUri("api/v2/default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2HashKeyRequest.Bucket == "test"); CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); @@ -2389,7 +2389,7 @@ TEST_CASE("z$service.parse.relative.Uri") HttpRequestData V2ValueContentIdRequest; CHECK(HttpRequestParseRelativeUri( - "api/v2/ns_nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + "api/v2/nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", V2ValueContentIdRequest)); CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); CHECK(V2ValueContentIdRequest.Bucket == "test"sv); @@ -2397,16 +2397,16 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData Invalid; - CHECK(!HttpRequestParseRelativeUri("api/v2/bla", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2//", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_bad\2_namespace", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_nice/\2\1bucket", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789a", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); CHECK(!HttpRequestParseRelativeUri( - "api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + "api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", Invalid)); } -- cgit v1.2.3 From 2f69a30c936bc475bf85daded3706dc67f2e8a0f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 13:10:10 +0200 Subject: use ns_ prefix on disk only --- zenserver/cache/structuredcache.cpp | 5 ----- 1 file changed, 5 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 9f0ff7408..6d3211c0b 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -119,11 +119,6 @@ namespace { { Key = Key.substr(HttpZCacheAPIV2Prefix.length()); // Namespace reference - if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) - { - return false; - } - Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); std::string Namespace = ToLower(Key.substr(0, NamespaceSplitOffset)); -- cgit v1.2.3 From c442837aff6212c711e959a44fba7c070bcdcaf1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 15:55:07 +0200 Subject: drop api/v2 prefix for non-legacy requests --- zenserver/cache/structuredcache.cpp | 284 ++++++++++++++++++++++-------------- 1 file changed, 172 insertions(+), 112 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 6d3211c0b..b6b5b218e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -87,127 +87,188 @@ namespace { std::optional ValueContentId; }; - bool IsValidNamespaceName(std::string_view Name) + const char* ValidNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + std::optional GetValidNamespaceName(std::string_view Name) { - if (Name == ZenCacheStore::DefaultNamespace) - { - return true; - } if (Name.empty()) { - return false; + return {}; } if (Name.length() > 64) { - return false; + return {}; } - return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; - } - bool IsValidBucketName(std::string_view Name) - { - if (Name.empty()) + if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) { - return false; + return {}; } - return std::all_of(begin(Name), end(Name), [](const char c) { return std::isalnum(c); }); + return ToLower(Name); } - bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + std::optional GetValidBucketName(std::string_view Name) { - if (Key.starts_with(HttpZCacheAPIV2Prefix)) + if (Name.empty()) { - Key = Key.substr(HttpZCacheAPIV2Prefix.length()); - // Namespace reference - std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); - - std::string Namespace = ToLower(Key.substr(0, NamespaceSplitOffset)); - if (!IsValidNamespaceName(Namespace)) - { - return false; - } - - Data.Namespace = Namespace; - - if (NamespaceSplitOffset == std::string_view::npos) - { - return true; - } - Key = Key.substr(NamespaceSplitOffset + 1); + return {}; } - else + if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) { - Data.Namespace = ZenCacheStore::DefaultNamespace; + return {}; } + return ToLower(Name); + } - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - std::string Bucket = ToLower(Key.substr(0, BucketSplitOffset)); - if (!IsValidBucketName(Bucket)) + std::optional GetValidIoHash(std::string_view Hash) + { + if (Hash.length() != IoHash::StringLength) { - return false; + return {}; } - Data.Bucket = Bucket; - if (BucketSplitOffset == std::string_view::npos) + IoHash KeyHash; + if (!ParseHexBytes(Hash.data(), Hash.size(), KeyHash.Hash)) { - // Bucket reference - return true; + return {}; } + return KeyHash; + } - std::string_view HashSegment; - std::string_view ValueSegment; + bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + { + std::vector Tokens; + uint32_t TokenCount = zen::ForEachStrTok(Key, '/', [&](const std::string_view& Token) { + Tokens.push_back(Token); + return true; + }); - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else + switch (TokenCount) { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } + case 1: + Data.Namespace = GetValidNamespaceName(Tokens[0]); + return Data.Namespace.has_value(); + case 2: + { + std::optional PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + Data.Namespace = ZenCacheStore::DefaultNamespace; + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + return true; + } + case 3: + { + std::optional PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key/valueid request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + Data.ValueContentId = GetValidIoHash(Tokens[2]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + Data.Namespace = ZenCacheStore::DefaultNamespace; + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey) + { + return false; + } + return true; + } + case 4: + { + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } - IoHash KeyHash; - if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) - { - return false; - } + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey.has_value()) + { + return false; + } - Data.HashKey = KeyHash; + Data.ValueContentId = GetValidIoHash(Tokens[3]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + return true; + } + default: + return false; + } + } - if (ValueSegment.empty()) + bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { + CbFieldView NamespaceField = KeyView["Namespace"sv]; + std::optional Namespace; + if (!NamespaceField) { - return true; + Namespace = ZenCacheStore::DefaultNamespace; } - - if (ValueSegment.size() != IoHash::StringLength) + else { - return false; + if (NamespaceField.HasError()) + { + return false; + } + if (!NamespaceField.IsString()) + { + return false; + } + Namespace = GetValidNamespaceName(NamespaceField.AsString()); } - - IoHash ValueHash; - if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) + if (!Namespace.has_value()) { return false; } - Data.ValueContentId = ValueHash; - - return true; - } - - bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) - { - CbFieldView NamespaceField = KeyView["Namespace"sv]; - std::string_view Namespace = NamespaceField.AsString(ZenCacheStore::DefaultNamespace); - CbFieldView BucketField = KeyView["Bucket"sv]; + CbFieldView BucketField = KeyView["Bucket"sv]; if (BucketField.HasError()) { return false; @@ -216,8 +277,8 @@ namespace { { return false; } - std::string_view Bucket = BucketField.AsString(); - if (!IsValidBucketName(Bucket)) + std::optional Bucket = GetValidBucketName(BucketField.AsString()); + if (!Bucket.has_value()) { return false; } @@ -231,7 +292,7 @@ namespace { return false; } IoHash Hash = HashField.AsHash(); - Key = CacheKey::Create(Namespace, Bucket, Hash); + Key = CacheKey::Create(*Namespace, *Bucket, Hash); return true; } @@ -2325,12 +2386,12 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request) TEST_CASE("z$service.parse.relative.Uri") { - HttpRequestData LegacyBucketRequest; - CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequest)); - CHECK(LegacyBucketRequest.Namespace == ZenCacheStore::DefaultNamespace); - CHECK(LegacyBucketRequest.Bucket == "test"sv); - CHECK(!LegacyBucketRequest.HashKey.has_value()); - CHECK(!LegacyBucketRequest.ValueContentId.has_value()); + HttpRequestData LegacyBucketRequestBecomesNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequestBecomesNamespaceRequest)); + CHECK(LegacyBucketRequestBecomesNamespaceRequest.Namespace == "test"sv); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.Bucket.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.HashKey.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.ValueContentId.has_value()); HttpRequestData LegacyHashKeyRequest; CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); @@ -2348,61 +2409,60 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData V2DefaultNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/default", V2DefaultNamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("default", V2DefaultNamespaceRequest)); CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); HttpRequestData V2NamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace", V2NamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("nicenamespace", V2NamespaceRequest)); CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); CHECK(!V2NamespaceRequest.Bucket.has_value()); CHECK(!V2NamespaceRequest.HashKey.has_value()); CHECK(!V2NamespaceRequest.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithDefaultNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/default/test", V2BucketRequestWithDefaultNamespace)); + CHECK(HttpRequestParseRelativeUri("default/test", V2BucketRequestWithDefaultNamespace)); CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(HttpRequestParseRelativeUri("nicenamespace/test", V2BucketRequestWithNamespace)); CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); HttpRequestData V2HashKeyRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(HttpRequestParseRelativeUri("default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2HashKeyRequest.Bucket == "test"); CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); CHECK(!V2HashKeyRequest.ValueContentId.has_value()); HttpRequestData V2ValueContentIdRequest; - CHECK(HttpRequestParseRelativeUri( - "api/v2/nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", - V2ValueContentIdRequest)); + CHECK( + HttpRequestParseRelativeUri("nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + V2ValueContentIdRequest)); CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); CHECK(V2ValueContentIdRequest.Bucket == "test"sv); CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData Invalid; - CHECK(!HttpRequestParseRelativeUri("api/v2/", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2//", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/bad\2_namespace", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/nice/\2\1bucket", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789a", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); - CHECK(!HttpRequestParseRelativeUri( - "api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", - Invalid)); + CHECK(!HttpRequestParseRelativeUri("", Invalid)); + CHECK(!HttpRequestParseRelativeUri("/", Invalid)); + CHECK(!HttpRequestParseRelativeUri("bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + Invalid)); } #endif -- cgit v1.2.3 From d0440e215136a3b4dbb417f4fe317b8bb223e094 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 17:01:45 +0200 Subject: review feedback --- zenserver/cache/structuredcache.cpp | 53 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 27 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index b6b5b218e..c457c59b2 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -76,8 +76,7 @@ struct PutRequestData }; namespace { - static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; - static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; struct HttpRequestData { @@ -462,19 +461,19 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { switch (Request.RequestVerb()) { case HttpVerb::kHead: case HttpVerb::kGet: { - HandleGetCacheRecord(Request, Ref, PolicyFromURL); + HandleGetCacheRecord(Request, Ref, PolicyFromUrl); } break; case HttpVerb::kPut: - HandlePutCacheRecord(Request, Ref, PolicyFromURL); + HandlePutCacheRecord(Request, Ref, PolicyFromUrl); break; default: break; @@ -482,20 +481,20 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { const ZenContentType AcceptType = Request.AcceptContentType(); - const bool SkipData = EnumHasAllFlags(PolicyFromURL, CachePolicy::SkipData); - const bool PartialRecord = EnumHasAllFlags(PolicyFromURL, CachePolicy::PartialRecord); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); bool Success = false; ZenCacheValue ClientResultValue; - if (!EnumHasAnyFlags(PolicyFromURL, CachePolicy::Query)) + if (!EnumHasAnyFlags(PolicyFromUrl, CachePolicy::Query)) { return Request.WriteResponse(HttpResponseCode::OK); } - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && m_CacheStore.Get(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { Success = true; @@ -575,7 +574,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::OK, ClientResultValue.Value.GetContentType(), ClientResultValue.Value); } } - else if (!EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryRemote)) + else if (!EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote)) { ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); m_CacheStats.MissCount++; @@ -585,12 +584,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request // Issue upstream query asynchronously in order to keep requests flowing without // hogging I/O servicing threads with blocking work - Request.WriteResponseAsync([this, AcceptType, PolicyFromURL, Ref](HttpServerRequest& AsyncRequest) { + Request.WriteResponseAsync([this, AcceptType, PolicyFromUrl, Ref](HttpServerRequest& AsyncRequest) { bool Success = false; - const bool PartialRecord = EnumHasAllFlags(PolicyFromURL, CachePolicy::PartialRecord); - const bool QueryLocal = EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal); - const bool StoreLocal = EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreLocal); - const bool SkipData = EnumHasAllFlags(PolicyFromURL, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); + const bool QueryLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + const bool StoreLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreLocal); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); ZenCacheValue ClientResultValue; metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); @@ -749,7 +748,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } void -HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { IoBuffer Body = Request.ReadPayload(); @@ -767,7 +766,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote)) + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) { m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}}); } @@ -788,7 +787,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); } - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; CbObjectView CacheRecord(Body.Data()); std::vector ValidAttachments; int32_t TotalCount = 0; @@ -834,7 +833,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid package", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); } - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; CbObject CacheRecord = Package.GetObject(); AttachmentCount Count; @@ -917,16 +916,16 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } void -HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { switch (Request.RequestVerb()) { case HttpVerb::kHead: case HttpVerb::kGet: - HandleGetCacheValue(Request, Ref, PolicyFromURL); + HandleGetCacheValue(Request, Ref, PolicyFromUrl); break; case HttpVerb::kPut: - HandlePutCacheValue(Request, Ref, PolicyFromURL); + HandlePutCacheValue(Request, Ref, PolicyFromUrl); break; default: break; @@ -934,13 +933,13 @@ HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { Stopwatch Timer; IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); bool InUpstreamCache = false; - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; { const bool QueryUpstream = !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); @@ -1002,10 +1001,10 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, } void -HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored - ZEN_UNUSED(PolicyFromURL); + ZEN_UNUSED(PolicyFromUrl); Stopwatch Timer; -- cgit v1.2.3 From 3f996c64b7ebc3fc39c2320c2d0a2a49496e2249 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 18 May 2022 09:37:29 +0200 Subject: align bucket naming rules with UE code base --- zenserver/cache/structuredcache.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index c457c59b2..bc6f31dd3 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -86,7 +86,8 @@ namespace { std::optional ValueContentId; }; - const char* ValidNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const char* ValidNamespaceNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const char* ValidBucketNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; std::optional GetValidNamespaceName(std::string_view Name) { @@ -99,7 +100,7 @@ namespace { return {}; } - if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) + if (Name.find_first_not_of(ValidNamespaceNameCharacters) != std::string::npos) { return {}; } @@ -112,7 +113,7 @@ namespace { { return {}; } - if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) + if (Name.find_first_not_of(ValidBucketNameCharacters) != std::string::npos) { return {}; } -- cgit v1.2.3 From a9130d34b5318b0da5d3547c432a8734213fbe9b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 May 2022 11:37:25 +0200 Subject: Keep Namespace out of CacheKey and store it on request level RPC requests now has a Namespace field under Params instead of one Namespace per cache key Fall back to legacy upstream HTTP URI format if default namespace is requested --- zenserver/cache/structuredcache.cpp | 221 ++++++++++++++++++++---------------- 1 file changed, 125 insertions(+), 96 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index bc6f31dd3..a349f13e1 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -70,6 +70,7 @@ struct AttachmentCount struct PutRequestData { + std::string Namespace; CacheKey Key; CbObjectView RecordObject; CacheRecordPolicy Policy; @@ -244,30 +245,27 @@ namespace { } } - bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + std::optional GetRpcRequestNamespace(const CbObjectView Params) { - CbFieldView NamespaceField = KeyView["Namespace"sv]; - std::optional Namespace; + CbFieldView NamespaceField = Params["Namespace"sv]; if (!NamespaceField) { - Namespace = ZenCacheStore::DefaultNamespace; + return std::string(ZenCacheStore::DefaultNamespace); } - else + + if (NamespaceField.HasError()) { - if (NamespaceField.HasError()) - { - return false; - } - if (!NamespaceField.IsString()) - { - return false; - } - Namespace = GetValidNamespaceName(NamespaceField.AsString()); + return {}; } - if (!Namespace.has_value()) + if (!NamespaceField.IsString()) { - return false; + return {}; } + return GetValidNamespaceName(NamespaceField.AsString()); + } + + bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { CbFieldView BucketField = KeyView["Bucket"sv]; if (BucketField.HasError()) { @@ -292,7 +290,7 @@ namespace { return false; } IoHash Hash = HashField.AsHash(); - Key = CacheKey::Create(*Namespace, *Bucket, Hash); + Key = CacheKey::Create(*Bucket, Hash); return true; } @@ -596,7 +594,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); if (GetUpstreamCacheResult UpstreamResult = - m_UpstreamCache.GetCacheRecord({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, AcceptType); + m_UpstreamCache.GetCacheRecord(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, AcceptType); UpstreamResult.Success) { Success = true; @@ -769,7 +767,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) { - m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}}); + m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}); } Request.WriteResponse(HttpResponseCode::Created); @@ -819,7 +817,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, - .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, .ValueContentIds = std::move(ValidAttachments)}); } @@ -904,7 +903,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, .ValueContentIds = std::move(ValidAttachments)}); } @@ -946,7 +946,7 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, if (QueryUpstream) { - if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); + if (auto UpstreamResult = m_UpstreamCache.GetCacheValue(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); UpstreamResult.Success) { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) @@ -1124,8 +1124,13 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req ZEN_ASSERT(BatchObject["Method"sv].AsString() == "PutCacheRecords"sv); - std::string_view PolicyText = Params["DefaultPolicy"].AsString(); - DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; + std::string_view PolicyText = Params["DefaultPolicy"].AsString(); + std::optional Namespace = GetRpcRequestNamespace(Params); + if (!Namespace) + { + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { @@ -1139,7 +1144,7 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req return Request.WriteResponse(HttpResponseCode::BadRequest); } CacheRecordPolicy Policy = LoadCacheRecordPolicy(RequestObject["Policy"sv].AsObjectView(), DefaultPolicy); - PutRequestData PutRequest{std::move(Key), RecordObject, std::move(Policy)}; + PutRequestData PutRequest{*Namespace, std::move(Key), RecordObject, std::move(Policy)}; PutResult Result = PutCacheRecord(PutRequest, &BatchRequest); @@ -1203,7 +1208,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack else { ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", - Request.Key.Namespace, + Request.Namespace, Request.Key.Bucket, Request.Key.Hash, ToString(HttpContentType::kCbPackage), @@ -1225,7 +1230,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack } ZEN_DEBUG("PUT - '{}/{}/{}' {}, attachments '{}/{}/{}' (new/valid/total)", - Request.Key.Namespace, + Request.Namespace, Request.Key.Bucket, Request.Key.Hash, NiceBytes(TransferredSize), @@ -1237,14 +1242,16 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack CacheValue.Value = IoBuffer(Record.GetSize()); Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize())); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue); + m_CacheStore.Put(Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue); const bool IsPartialRecord = Count.Valid != Count.Total; if (EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::StoreRemote) && !IsPartialRecord) { - m_UpstreamCache.EnqueueUpstream( - {.Type = ZenContentType::kCbPackage, .Key = Request.Key, .ValueContentIds = std::move(ValidAttachments)}); + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, + .Namespace = Request.Namespace, + .Key = Request.Key, + .ValueContentIds = std::move(ValidAttachments)}); } return PutResult::Success; } @@ -1277,8 +1284,13 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt bool UsedUpstream = false; }; - std::string_view PolicyText = Params["DefaultPolicy"sv].AsString(); - CachePolicy DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; + std::string_view PolicyText = Params["DefaultPolicy"sv].AsString(); + CachePolicy DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; + std::optional Namespace = GetRpcRequestNamespace(Params); + if (!Namespace) + { + return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); + } std::vector Requests; std::vector UpstreamIndexes; CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); @@ -1322,7 +1334,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt ZenCacheValue RecordCacheValue; if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && - m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, RecordCacheValue)) + m_CacheStore.Get(*Namespace, Key.Bucket, Key.Hash, RecordCacheValue)) { Request.RecordCacheValue = std::move(RecordCacheValue.Value); if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject) @@ -1436,7 +1448,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt } } - const auto OnCacheRecordGetComplete = [this, &ParseValues](CacheRecordGetCompleteParams&& Params) { + const auto OnCacheRecordGetComplete = [this, Namespace, &ParseValues](CacheRecordGetCompleteParams&& Params) { if (!Params.Record) { return; @@ -1453,7 +1465,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt Request.RecordObject = ObjectBuffer; if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal)) { - m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); + m_CacheStore.Put(*Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}); } ParseValues(Request); Request.UsedUpstream = true; @@ -1493,7 +1505,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt { ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}/{}'", Value.ContentId, - Key.Namespace, + *Namespace, Key.Bucket, Key.Hash); } @@ -1510,7 +1522,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt } }; - m_UpstreamCache.GetCacheRecords(UpstreamRequests, std::move(OnCacheRecordGetComplete)); + m_UpstreamCache.GetCacheRecords(*Namespace, UpstreamRequests, std::move(OnCacheRecordGetComplete)); } CbPackage ResponsePackage; @@ -1533,7 +1545,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt } ZEN_DEBUG("HIT - '{}/{}/{}' {}{}{}", - Key.Namespace, + *Namespace, Key.Bucket, Key.Hash, NiceBytes(Request.RecordCacheValue.Size()), @@ -1549,11 +1561,11 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt if (!EnumHasAnyFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::Query)) { // If they requested no query, do not record this as a miss - ZEN_DEBUG("DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash); + ZEN_DEBUG("DISABLEDQUERY - '{}/{}/{}'", *Namespace, Key.Bucket, Key.Hash); } else { - ZEN_DEBUG("MISS - '{}/{}/{}' {}", Key.Namespace, Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv); + ZEN_DEBUG("MISS - '{}/{}/{}' {}", *Namespace, Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv); m_CacheStats.MissCount++; } } @@ -1579,8 +1591,13 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ ZEN_ASSERT(BatchObject["Method"sv].AsString() == "PutCacheValues"sv); - std::string_view PolicyText = Params["DefaultPolicy"].AsString(); - CachePolicy DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; + std::string_view PolicyText = Params["DefaultPolicy"].AsString(); + CachePolicy DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; + std::optional Namespace = GetRpcRequestNamespace(Params); + if (!Namespace) + { + return Request.WriteResponse(HttpResponseCode::BadRequest); + } std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { @@ -1615,21 +1632,21 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ { IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); Value.SetContentType(ZenContentType::kCompressedBinary); - m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Value}); + m_CacheStore.Put(*Namespace, Key.Bucket, Key.Hash, {.Value = Value}); TransferredSize = Chunk.GetCompressedSize(); } Succeeded = true; } else { - ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}/{}' FAILED, value is not compressed", Key.Namespace, Key.Bucket, Key.Hash, RawHash); + ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}/{}' FAILED, value is not compressed", *Namespace, Key.Bucket, Key.Hash, RawHash); return Request.WriteResponse(HttpResponseCode::BadRequest); } } else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { ZenCacheValue ExistingValue; - if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, ExistingValue) && + if (m_CacheStore.Get(*Namespace, Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType())) { Succeeded = true; @@ -1640,11 +1657,11 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ if (Succeeded && EnumHasAllFlags(Policy, CachePolicy::StoreRemote)) { - m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCompressedBinary, .Key = Key}); + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = Key}); } Results.push_back(Succeeded); ZEN_DEBUG("PUTCACHEVALUES - '{}/{}/{}' {}, '{}'", - Key.Namespace, + *Namespace, Key.Bucket, Key.Hash, NiceBytes(TransferredSize), @@ -1679,9 +1696,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http { ZEN_TRACE_CPU("Z$::RpcGetCacheValues"); - CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); - std::string_view PolicyText = Params["DefaultPolicy"sv].AsString(); - CachePolicy DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; + CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); + std::string_view PolicyText = Params["DefaultPolicy"sv].AsString(); + CachePolicy DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; + std::optional Namespace = GetRpcRequestNamespace(Params); + if (!Namespace) + { + return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); + } + struct RequestData { CacheKey Key; @@ -1717,7 +1740,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http ZenCacheValue CacheValue; if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { - if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) + if (m_CacheStore.Get(*Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) { Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value)); } @@ -1725,7 +1748,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http if (Result) { ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", - Key.Namespace, + *Namespace, Key.Bucket, Key.Hash, NiceBytes(Result.GetCompressed().GetSize()), @@ -1740,12 +1763,12 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http else if (!EnumHasAnyFlags(Policy, CachePolicy::Query)) { // If they requested no query, do not record this as a miss - ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash); + ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}/{}'", *Namespace, Key.Bucket, Key.Hash); } else { ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}", - Key.Namespace, + *Namespace, Key.Bucket, Key.Hash, "LOCAL"sv, @@ -1763,13 +1786,14 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http for (size_t Index : RemoteRequestIndexes) { RequestData& Request = Requests[Index]; - RequestedRecordsData.push_back({{Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash}}); + RequestedRecordsData.push_back({Request.Key.Bucket, Request.Key.Hash}); CacheChunkRequests.push_back(&RequestedRecordsData.back()); } Stopwatch Timer; m_UpstreamCache.GetCacheValues( + *Namespace, CacheChunkRequests, - [this, &RequestedRecordsData, &Requests, &RemoteRequestIndexes, &Timer](CacheValueGetCompleteParams&& Params) { + [this, Namespace, &RequestedRecordsData, &Requests, &RemoteRequestIndexes, &Timer](CacheValueGetCompleteParams&& Params) { CacheChunkRequest& ChunkRequest = Params.Request; if (Params.Value) { @@ -1783,9 +1807,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http // that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive // for us to keep the data only on the upstream server. // if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal)) - m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); + m_CacheStore.Put(*Namespace, Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", - ChunkRequest.Key.Namespace, + *Namespace, ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, NiceBytes(Request.Result.GetCompressed().GetSize()), @@ -1797,7 +1821,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http } } ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}", - ChunkRequest.Key.Namespace, + *Namespace, ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, "UPSTREAM"sv, @@ -1888,6 +1912,7 @@ HttpStructuredCacheService::HandleRpcGetCacheChunks(zen::HttpServerRequest& Http ZEN_TRACE_CPU("Z$::RpcGetCacheChunks"); + std::string Namespace; std::vector RecordKeys; // Data about a Record necessary to identify it to the upstream std::vector Records; // Scratch-space data about a Record when fulfilling RecordRequests std::vector RequestKeys; // Data about a ChunkRequest necessary to identify it to the upstream @@ -1897,27 +1922,28 @@ HttpStructuredCacheService::HandleRpcGetCacheChunks(zen::HttpServerRequest& Http std::vector UpstreamChunks; // ChunkRequests that we need to send to the upstream // Parse requests from the CompactBinary body of the RpcRequest and divide it into RecordRequests and ValueRequests - if (!ParseGetCacheChunksRequest(RecordKeys, Records, RequestKeys, Requests, RecordRequests, ValueRequests, RpcRequest)) + if (!ParseGetCacheChunksRequest(Namespace, RecordKeys, Records, RequestKeys, Requests, RecordRequests, ValueRequests, RpcRequest)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } // For each Record request, load the Record if necessary to find the Chunk's ContentId, load its Payloads if we // have it locally, and otherwise append a request for the payload to UpstreamChunks - GetLocalCacheRecords(RecordKeys, Records, RecordRequests, UpstreamChunks); + GetLocalCacheRecords(Namespace, RecordKeys, Records, RecordRequests, UpstreamChunks); // For each Value request, load the Value if we have it locally and otherwise append a request for the payload to UpstreamChunks - GetLocalCacheValues(ValueRequests, UpstreamChunks); + GetLocalCacheValues(Namespace, ValueRequests, UpstreamChunks); // Call GetCacheChunks on the upstream for any payloads we do not have locally - GetUpstreamCacheChunks(UpstreamChunks, RequestKeys, Requests); + GetUpstreamCacheChunks(Namespace, UpstreamChunks, RequestKeys, Requests); // Send the payload and descriptive data about each chunk to the client - WriteGetCacheChunksResponse(Requests, HttpRequest); + WriteGetCacheChunksResponse(Namespace, Requests, HttpRequest); } bool -HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector& RecordKeys, +HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string& Namespace, + std::vector& RecordKeys, std::vector& Records, std::vector& RequestKeys, std::vector& Requests, @@ -1929,11 +1955,20 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector(ChunkRequestsArray.Num()); + CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); + std::string_view DefaultPolicyText = Params["DefaultPolicy"sv].AsString(); + CachePolicy DefaultPolicy = !DefaultPolicyText.empty() ? ParseCachePolicy(DefaultPolicyText) : CachePolicy::Default; + + std::optional NamespaceText = GetRpcRequestNamespace(Params); + if (!NamespaceText) + { + ZEN_WARN("GetCacheChunks: Invalid namespace in ChunkRequest."); + return false; + } + Namespace = *NamespaceText; + + CbArrayView ChunkRequestsArray = Params["ChunkRequests"sv].AsArrayView(); + size_t NumRequests = static_cast(ChunkRequestsArray.Num()); // Note that these reservations allow us to take pointers to the elements while populating them. If the reservation is removed, // we will need to change the pointers to indexes to handle reallocations. @@ -1996,11 +2031,9 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vectorKey.Namespace, PreviousRecordKey->Key.Bucket, PreviousRecordKey->Key.Hash); return false; @@ -2022,7 +2055,8 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector& RecordKeys, +HttpStructuredCacheService::GetLocalCacheRecords(std::string_view Namespace, + std::vector& RecordKeys, std::vector& Records, std::vector& RecordRequests, std::vector& OutUpstreamChunks) @@ -2041,7 +2075,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(RecordKey.Key.Namespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) + if (m_CacheStore.Get(Namespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue)) { Record.Exists = true; Record.CacheValue = std::move(CacheValue.Value); @@ -2058,7 +2092,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (!UpstreamRecordRequests.empty()) { - const auto OnCacheRecordGetComplete = [this, &RecordKeys, &Records](CacheRecordGetCompleteParams&& Params) { + const auto OnCacheRecordGetComplete = [this, Namespace, &RecordKeys, &Records](CacheRecordGetCompleteParams&& Params) { if (!Params.Record) { return; @@ -2076,10 +2110,10 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal)) { - m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); + m_CacheStore.Put(Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}); } }; - m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete)); + m_UpstreamCache.GetCacheRecords(Namespace, UpstreamRecordRequests, std::move(OnCacheRecordGetComplete)); } std::vector UpstreamPayloadRequests; @@ -2163,7 +2197,8 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector& } void -HttpStructuredCacheService::GetLocalCacheValues(std::vector& ValueRequests, +HttpStructuredCacheService::GetLocalCacheValues(std::string_view Namespace, + std::vector& ValueRequests, std::vector& OutUpstreamChunks) { using namespace cache::detail; @@ -2173,7 +2208,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vectorExists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal)) { ZenCacheValue CacheValue; - if (m_CacheStore.Get(Request->Key->Key.Namespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) + if (m_CacheStore.Get(Namespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue)) { if (IsCompressedBinary(CacheValue.Value.GetContentType())) { @@ -2207,7 +2242,8 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vector& UpstreamChunks, +HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view Namespace, + std::vector& UpstreamChunks, std::vector& RequestKeys, std::vector& Requests) { @@ -2215,7 +2251,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vector& Requests, +HttpStructuredCacheService::WriteGetCacheChunksResponse(std::string_view Namespace, + std::vector& Requests, zen::HttpServerRequest& HttpRequest) { using namespace cache::detail; @@ -2290,7 +2327,7 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(std::vectorKey.Namespace, + Namespace, Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId, @@ -2301,19 +2338,11 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(std::vectorKey.Namespace, - Request.Key->Key.Bucket, - Request.Key->Key.Hash, - Request.Key->ValueId); + ZEN_DEBUG("SKIP - '{}/{}/{}/{}'", Namespace, Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId); } else { - ZEN_DEBUG("MISS - '{}/{}/{}/{}'", - Request.Key->Key.Namespace, - Request.Key->Key.Bucket, - Request.Key->Key.Hash, - Request.Key->ValueId); + ZEN_DEBUG("MISS - '{}/{}/{}/{}'", Namespace, Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId); m_CacheStats.MissCount++; } } -- cgit v1.2.3 From bbfd45ff267f03d36e9f842876dedd3f7852f45e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 May 2022 16:02:27 +0200 Subject: fix tests --- zenserver/cache/structuredcache.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index a349f13e1..135572d07 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -2438,8 +2438,8 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData V2DefaultNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("default", V2DefaultNamespaceRequest)); - CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(HttpRequestParseRelativeUri("ue4.ddc", V2DefaultNamespaceRequest)); + CHECK(V2DefaultNamespaceRequest.Namespace == "ue4.ddc"); CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); @@ -2452,8 +2452,8 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(!V2NamespaceRequest.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithDefaultNamespace; - CHECK(HttpRequestParseRelativeUri("default/test", V2BucketRequestWithDefaultNamespace)); - CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(HttpRequestParseRelativeUri("ue4.ddc/test", V2BucketRequestWithDefaultNamespace)); + CHECK(V2BucketRequestWithDefaultNamespace.Namespace == "ue4.ddc"); CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); @@ -2466,7 +2466,7 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); HttpRequestData V2HashKeyRequest; - CHECK(HttpRequestParseRelativeUri("default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2HashKeyRequest.Bucket == "test"); CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); -- cgit v1.2.3 From fadaf985c6a1afeede661e2eb14136b45cb1c6f0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 May 2022 22:37:07 +0200 Subject: fix mac compilation error --- zenserver/cache/structuredcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 135572d07..e11499289 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -1786,7 +1786,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http for (size_t Index : RemoteRequestIndexes) { RequestData& Request = Requests[Index]; - RequestedRecordsData.push_back({Request.Key.Bucket, Request.Key.Hash}); + RequestedRecordsData.push_back({.Key = {Request.Key.Bucket, Request.Key.Hash}}); CacheChunkRequests.push_back(&RequestedRecordsData.back()); } Stopwatch Timer; -- cgit v1.2.3