diff options
| -rw-r--r-- | zenserver/cache/structuredcache.cpp | 602 | ||||
| -rw-r--r-- | zenserver/cache/structuredcache.h | 36 | ||||
| -rw-r--r-- | zenserver/cache/structuredcachestore.cpp | 15 | ||||
| -rw-r--r-- | zenserver/cache/structuredcachestore.h | 3 | ||||
| -rw-r--r-- | zenserver/zenserver.cpp | 1 |
5 files changed, 465 insertions, 192 deletions
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index eed7a4420..c457c59b2 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -35,6 +35,11 @@ #include <gsl/gsl-lite.hpp> +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +# include <zencore/testutils.h> +#endif + namespace zen { using namespace std::literals; @@ -70,6 +75,228 @@ struct PutRequestData CacheRecordPolicy Policy; }; +namespace { + static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + + struct HttpRequestData + { + std::optional<std::string> Namespace; + std::optional<std::string> Bucket; + std::optional<IoHash> HashKey; + std::optional<IoHash> ValueContentId; + }; + + const char* ValidNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + std::optional<std::string> GetValidNamespaceName(std::string_view Name) + { + if (Name.empty()) + { + return {}; + } + if (Name.length() > 64) + { + return {}; + } + + if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) + { + return {}; + } + return ToLower(Name); + } + + std::optional<std::string> GetValidBucketName(std::string_view Name) + { + if (Name.empty()) + { + return {}; + } + if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) + { + return {}; + } + return ToLower(Name); + } + + std::optional<IoHash> GetValidIoHash(std::string_view Hash) + { + if (Hash.length() != IoHash::StringLength) + { + return {}; + } + + IoHash KeyHash; + if (!ParseHexBytes(Hash.data(), Hash.size(), KeyHash.Hash)) + { + return {}; + } + return KeyHash; + } + + bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + { + std::vector<std::string_view> Tokens; + uint32_t TokenCount = zen::ForEachStrTok(Key, '/', [&](const std::string_view& Token) { + Tokens.push_back(Token); + return true; + }); + + switch (TokenCount) + { + case 1: + Data.Namespace = GetValidNamespaceName(Tokens[0]); + return Data.Namespace.has_value(); + case 2: + { + std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + Data.Namespace = ZenCacheStore::DefaultNamespace; + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + return true; + } + case 3: + { + std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key/valueid request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + Data.ValueContentId = GetValidIoHash(Tokens[2]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + Data.Namespace = ZenCacheStore::DefaultNamespace; + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey) + { + return false; + } + return true; + } + case 4: + { + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey.has_value()) + { + return false; + } + + Data.ValueContentId = GetValidIoHash(Tokens[3]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + return true; + } + default: + return false; + } + } + + bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { + CbFieldView NamespaceField = KeyView["Namespace"sv]; + std::optional<std::string> Namespace; + if (!NamespaceField) + { + Namespace = ZenCacheStore::DefaultNamespace; + } + else + { + if (NamespaceField.HasError()) + { + return false; + } + if (!NamespaceField.IsString()) + { + return false; + } + Namespace = GetValidNamespaceName(NamespaceField.AsString()); + } + if (!Namespace.has_value()) + { + return false; + } + CbFieldView BucketField = KeyView["Bucket"sv]; + if (BucketField.HasError()) + { + return false; + } + if (!BucketField.IsString()) + { + return false; + } + std::optional<std::string> Bucket = GetValidBucketName(BucketField.AsString()); + if (!Bucket.has_value()) + { + return false; + } + CbFieldView HashField = KeyView["Hash"sv]; + if (HashField.HasError()) + { + return false; + } + if (!HashField.IsHash()) + { + return false; + } + IoHash Hash = HashField.AsHash(); + Key = CacheKey::Create(*Namespace, *Bucket, Hash); + return true; + } + +} // namespace + ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, @@ -124,44 +351,55 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx) void HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { - CacheRef Ref; - metrics::OperationTiming::Scope $(m_HttpRequests); - if (!ValidateKeyUri(Request, /* out */ Ref)) + std::string_view Key = Request.RelativeUri(); + if (Key == HttpZCacheRPCPrefix) { - std::string_view Key = Request.RelativeUri(); - - if (Key == "$rpc") - { - return HandleRpcRequest(Request); - } - - if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) - { - // Bucket reference - return HandleCacheBucketRequest(Request, Key); - } + return HandleRpcRequest(Request); + } + HttpRequestData RequestData; + if (!HttpRequestParseRelativeUri(Key, RequestData)) + { return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL } - CachePolicy PolicyFromURL = ParseCachePolicy(Request.GetQueryParams()); + if (RequestData.ValueContentId.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + ZEN_ASSERT(RequestData.HashKey.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = RequestData.ValueContentId.value()}; + return HandleCacheValueRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); + } - if (Ref.ValueContentId == IoHash::Zero) + if (RequestData.HashKey.has_value()) { - return HandleCacheRecordRequest(Request, Ref, PolicyFromURL); + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = IoHash::Zero}; + return HandleCacheRecordRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); } - else + + if (RequestData.Bucket.has_value()) { - return HandleCacheValueRequest(Request, Ref, PolicyFromURL); + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheBucketRequest(Request, RequestData.Namespace.value(), RequestData.Bucket.value()); } - return; + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheNamespaceRequest(Request, RequestData.Namespace.value()); } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view) { switch (Request.RequestVerb()) { @@ -173,11 +411,39 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, break; case HttpVerb::kDelete: - // Drop bucket + // Drop namespace + { + // if (m_CacheStore.DropNamespace(Namespace)) + // { + // return Request.WriteResponse(HttpResponseCode::OK); + // } + // else + // { + // return Request.WriteResponse(HttpResponseCode::NotFound); + // } + } + break; + + default: + break; + } +} + +void +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: { - // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability - std::string_view Namespace = ZenCacheStore::DefaultNamespace; + // Query stats + } + break; + case HttpVerb::kDelete: + // Drop bucket + { if (m_CacheStore.DropBucket(Namespace, Bucket)) { return Request.WriteResponse(HttpResponseCode::OK); @@ -195,19 +461,19 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { switch (Request.RequestVerb()) { case HttpVerb::kHead: case HttpVerb::kGet: { - HandleGetCacheRecord(Request, Ref, PolicyFromURL); + HandleGetCacheRecord(Request, Ref, PolicyFromUrl); } break; case HttpVerb::kPut: - HandlePutCacheRecord(Request, Ref, PolicyFromURL); + HandlePutCacheRecord(Request, Ref, PolicyFromUrl); break; default: break; @@ -215,20 +481,20 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { const ZenContentType AcceptType = Request.AcceptContentType(); - const bool SkipData = EnumHasAllFlags(PolicyFromURL, CachePolicy::SkipData); - const bool PartialRecord = EnumHasAllFlags(PolicyFromURL, CachePolicy::PartialRecord); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); bool Success = false; ZenCacheValue ClientResultValue; - if (!EnumHasAnyFlags(PolicyFromURL, CachePolicy::Query)) + if (!EnumHasAnyFlags(PolicyFromUrl, CachePolicy::Query)) { return Request.WriteResponse(HttpResponseCode::OK); } - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && m_CacheStore.Get(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { Success = true; @@ -308,7 +574,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::OK, ClientResultValue.Value.GetContentType(), ClientResultValue.Value); } } - else if (!EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryRemote)) + else if (!EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote)) { ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); m_CacheStats.MissCount++; @@ -318,12 +584,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request // Issue upstream query asynchronously in order to keep requests flowing without // hogging I/O servicing threads with blocking work - Request.WriteResponseAsync([this, AcceptType, PolicyFromURL, Ref](HttpServerRequest& AsyncRequest) { + Request.WriteResponseAsync([this, AcceptType, PolicyFromUrl, Ref](HttpServerRequest& AsyncRequest) { bool Success = false; - const bool PartialRecord = EnumHasAllFlags(PolicyFromURL, CachePolicy::PartialRecord); - const bool QueryLocal = EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal); - const bool StoreLocal = EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreLocal); - const bool SkipData = EnumHasAllFlags(PolicyFromURL, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); + const bool QueryLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + const bool StoreLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreLocal); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); ZenCacheValue ClientResultValue; metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); @@ -482,7 +748,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } void -HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { IoBuffer Body = Request.ReadPayload(); @@ -500,7 +766,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote)) + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) { m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}}); } @@ -521,7 +787,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); } - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; CbObjectView CacheRecord(Body.Data()); std::vector<IoHash> ValidAttachments; int32_t TotalCount = 0; @@ -567,7 +833,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid package", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); } - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; CbObject CacheRecord = Package.GetObject(); AttachmentCount Count; @@ -650,16 +916,16 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } void -HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { switch (Request.RequestVerb()) { case HttpVerb::kHead: case HttpVerb::kGet: - HandleGetCacheValue(Request, Ref, PolicyFromURL); + HandleGetCacheValue(Request, Ref, PolicyFromUrl); break; case HttpVerb::kPut: - HandlePutCacheValue(Request, Ref, PolicyFromURL); + HandlePutCacheValue(Request, Ref, PolicyFromUrl); break; default: break; @@ -667,13 +933,13 @@ HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { Stopwatch Timer; IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); bool InUpstreamCache = false; - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; { const bool QueryUpstream = !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); @@ -735,10 +1001,10 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, } void -HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored - ZEN_UNUSED(PolicyFromURL); + ZEN_UNUSED(PolicyFromUrl); Stopwatch Timer; @@ -782,79 +1048,6 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, Request.WriteResponse(ResponseCode); } -bool -HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& OutRef) -{ - std::string_view Key = Request.RelativeUri(); - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - - if (BucketSplitOffset == std::string_view::npos) - { - return false; - } - - OutRef.Namespace = ToLower(ZenCacheStore::DefaultNamespace); // TODO: Should add namespace to URI and handle if the namespace is - // missing for backwards compatability - OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); - - if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - if (!std::all_of(begin(OutRef.BucketSegment), end(OutRef.BucketSegment), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - std::string_view HashSegment; - std::string_view ValueSegment; - - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - - // We know there is a slash so no need to check for npos return - - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else - { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } - - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } - - if (!ValueSegment.empty() && ValueSegment.size() == IoHash::StringLength) - { - const bool IsOk = ParseHexBytes(ValueSegment.data(), ValueSegment.size(), OutRef.ValueContentId.Hash); - - if (!IsOk) - { - return false; - } - } - else - { - OutRef.ValueContentId = IoHash::Zero; - } - - const bool IsOk = ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); - - if (!IsOk) - { - return false; - } - - return true; -} - void HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) { @@ -935,15 +1128,12 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req std::vector<bool> Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); - CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); + CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } @@ -1112,18 +1302,16 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt for (CbFieldView RequestField : RequestsArray) { - RecordRequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (HashField.HasError() || Key.Bucket.empty()) + RecordRequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + CacheKey& Key = Request.Upstream.Key; + if (!GetRpcRequestCacheKey(KeyObject, Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + Request.DownstreamPolicy = LoadCacheRecordPolicy(RequestObject["Policy"sv].AsObjectView(), DefaultPolicy); const CacheRecordPolicy& Policy = Request.DownstreamPolicy; @@ -1395,17 +1583,15 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ std::vector<bool> Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); CachePolicy Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; IoHash RawHash = RequestObject["RawHash"sv].AsBinaryAttachment(); @@ -1511,18 +1697,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http { Stopwatch Timer; - RequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) + RequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + if (!GetRpcRequestCacheKey(KeyObject, Request.Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); Request.Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; @@ -1768,15 +1951,10 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector<CacheKeyReque CbObjectView RequestObject = RequestView.AsObjectView(); CacheChunkRequest& RequestKey = RequestKeys.emplace_back(); ChunkRequest& Request = Requests.emplace_back(); - Request.Key = &RequestKey; - - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView HashField = KeyObject["Hash"sv]; - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - RequestKey.Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), - KeyObject["Bucket"sv].AsString(), - HashField.AsHash()); - if (RequestKey.Key.Bucket.empty() || HashField.HasError()) + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + Request.Key = &RequestKey; + if (!GetRpcRequestCacheKey(KeyObject, Request.Key->Key)) { ZEN_WARN("GetCacheChunks: Invalid key in ChunkRequest."); return false; @@ -2203,4 +2381,94 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request) Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +#if ZEN_WITH_TESTS + +TEST_CASE("z$service.parse.relative.Uri") +{ + HttpRequestData LegacyBucketRequestBecomesNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequestBecomesNamespaceRequest)); + CHECK(LegacyBucketRequestBecomesNamespaceRequest.Namespace == "test"sv); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.Bucket.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.HashKey.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.ValueContentId.has_value()); + + HttpRequestData LegacyHashKeyRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); + CHECK(LegacyHashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyHashKeyRequest.Bucket == "test"sv); + CHECK(LegacyHashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(!LegacyHashKeyRequest.ValueContentId.has_value()); + + HttpRequestData LegacyValueContentIdRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + LegacyValueContentIdRequest)); + CHECK(LegacyValueContentIdRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyValueContentIdRequest.Bucket == "test"sv); + CHECK(LegacyValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); + + HttpRequestData V2DefaultNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("default", V2DefaultNamespaceRequest)); + CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); + CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); + CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2NamespaceRequest; + CHECK(HttpRequestParseRelativeUri("nicenamespace", V2NamespaceRequest)); + CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); + CHECK(!V2NamespaceRequest.Bucket.has_value()); + CHECK(!V2NamespaceRequest.HashKey.has_value()); + CHECK(!V2NamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithDefaultNamespace; + CHECK(HttpRequestParseRelativeUri("default/test", V2BucketRequestWithDefaultNamespace)); + CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); + CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithNamespace; + CHECK(HttpRequestParseRelativeUri("nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); + CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); + CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); + + HttpRequestData V2HashKeyRequest; + CHECK(HttpRequestParseRelativeUri("default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(V2HashKeyRequest.Bucket == "test"); + CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(!V2HashKeyRequest.ValueContentId.has_value()); + + HttpRequestData V2ValueContentIdRequest; + CHECK( + HttpRequestParseRelativeUri("nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + V2ValueContentIdRequest)); + CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); + CHECK(V2ValueContentIdRequest.Bucket == "test"sv); + CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); + + HttpRequestData Invalid; + CHECK(!HttpRequestParseRelativeUri("", Invalid)); + CHECK(!HttpRequestParseRelativeUri("/", Invalid)); + CHECK(!HttpRequestParseRelativeUri("bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + Invalid)); +} + +#endif + +void +z$service_forcelink() +{ +} + } // namespace zen diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 8285d517d..5f248edd1 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -99,23 +99,23 @@ private: Invalid, }; - [[nodiscard]] bool ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef); - void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandleCacheValueRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandleRpcRequest(zen::HttpServerRequest& Request); - void HandleRpcPutCacheRecords(zen::HttpServerRequest& Request, const CbPackage& BatchRequest); - void HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); - void HandleRpcPutCacheValues(zen::HttpServerRequest& Request, const CbPackage& BatchRequest); - void HandleRpcGetCacheValues(zen::HttpServerRequest& Request, CbObjectView BatchRequest); - void HandleRpcGetCacheChunks(zen::HttpServerRequest& Request, CbObjectView BatchRequest); - void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); - virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; - virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; - PutResult PutCacheRecord(PutRequestData& Request, const CbPackage* Package); + void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleCacheValueRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleRpcRequest(zen::HttpServerRequest& Request); + void HandleRpcPutCacheRecords(zen::HttpServerRequest& Request, const CbPackage& BatchRequest); + void HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); + void HandleRpcPutCacheValues(zen::HttpServerRequest& Request, const CbPackage& BatchRequest); + void HandleRpcGetCacheValues(zen::HttpServerRequest& Request, CbObjectView BatchRequest); + void HandleRpcGetCacheChunks(zen::HttpServerRequest& Request, CbObjectView BatchRequest); + void HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view Namespace); + void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket); + virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; + PutResult PutCacheRecord(PutRequestData& Request, const CbPackage* Package); /** HandleRpcGetCacheChunks Helper: Parse the Body object into RecordValue Requests and Value Requests. */ bool ParseGetCacheChunksRequest(std::vector<CacheKeyRequest>& RecordKeys, @@ -160,4 +160,6 @@ IsCompressedBinary(ZenContentType Type) return Type == ZenContentType::kBinary || Type == ZenContentType::kCompressedBinary; } +void z$service_forcelink(); + } // namespace zen diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index c21945702..16130a98b 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2128,8 +2128,6 @@ ZenCacheDiskLayer::TotalSize() const //////////////////////////// ZenCacheStore -static constexpr std::string_view ZenCacheNamespaceDirPrefix = "ns_"; - ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc) { CreateDirectories(BasePath); @@ -2142,9 +2140,9 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor for (const std::filesystem::path& DirPath : DirContent.Directories) { std::string DirName = PathToUtf8(DirPath.stem()); - if (DirName.starts_with(ZenCacheNamespaceDirPrefix)) + if (DirName.starts_with(NamespaceDiskPrefix)) { - Namespaces.push_back(DirName.substr(3)); + Namespaces.push_back(DirName.substr(NamespaceDiskPrefix.length())); continue; } LegacyBuckets.push_back(DirName); @@ -2155,7 +2153,8 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor if (std::find(Namespaces.begin(), Namespaces.end(), DefaultNamespace) == Namespaces.end()) { ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size()); - std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, DefaultNamespace); + + std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", NamespaceDiskPrefix, DefaultNamespace); CreateDirectories(DefaultNamespaceFolder); // Move any non-namespace folders into the default namespace folder @@ -2170,13 +2169,14 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor ZEN_ERROR("Unable to move '{}' to '{}', reason '{}'", LegacyFolder, NewPath, Ec.message()); } } + Namespaces.push_back(std::string(DefaultNamespace)); } for (const std::string& NamespaceName : Namespaces) { m_Namespaces[NamespaceName] = - std::make_unique<ZenCacheNamespace>(Gc, BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, NamespaceName)); + std::make_unique<ZenCacheNamespace>(Gc, BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName)); } } @@ -2284,8 +2284,9 @@ ZenCacheStore::StorageSize() const ////////////////////////////////////////////////////////////////////////// +} // namespace zen + #if ZEN_WITH_TESTS -} namespace zen { diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 881285bc9..232e8b9a8 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -356,7 +356,8 @@ private: class ZenCacheStore final : public GcStorage, public GcContributor { public: - static constexpr std::string_view DefaultNamespace = ""; + static constexpr std::string_view DefaultNamespace = "default"; + static constexpr std::string_view NamespaceDiskPrefix = "ns_"; ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath); ~ZenCacheStore(); diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index abaec888a..9e6c67d34 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -1155,6 +1155,7 @@ test_main(int argc, char** argv) zen::zenhttp_forcelinktests(); zen::zenstore_forcelinktests(); zen::z$_forcelink(); + zen::z$service_forcelink(); zen::logging::InitializeLogging(); spdlog::set_level(spdlog::level::debug); |