From 1b235fc947589dfbac0d09024947c37171c5dc7f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 13:04:59 +0200 Subject: Add support for /api/v2/ URI requests with namespace support --- zenserver/cache/structuredcache.cpp | 177 +++++++++++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 25 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index eed7a4420..691da36fa 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -121,47 +121,148 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx) m_CacheStore.Scrub(Ctx); } -void -HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) +static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; +static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + +struct HttpRequestData { - CacheRef Ref; + std::optional Namespace; + std::optional Bucket; + std::optional HashKey; + std::optional ValueContentId; +}; - metrics::OperationTiming::Scope $(m_HttpRequests); +static bool +HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) +{ + std::string_view Namespace = ZenCacheStore::DefaultNamespace; + if (Key.starts_with(HttpZCacheAPIV2Prefix)) + { + std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/', HttpZCacheAPIV2Prefix.length()); + if (NamespaceSplitOffset == std::string_view::npos) + { + // Namespace reference + if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Namespace = ToLower(Key); + return true; + } + Data.Namespace = Key.substr(0, NamespaceSplitOffset); + Key = Key.substr(NamespaceSplitOffset + 1); + } + + std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); + if (BucketSplitOffset == std::string_view::npos) + { + if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Bucket = ToLower(Key); + return true; + } + + std::string_view HashSegment; + std::string_view ValueSegment; + + std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); + if (ValueSplitOffset == BucketSplitOffset) + { + // Basic cache record lookup + HashSegment = Key.substr(BucketSplitOffset + 1); + } + else + { + // Cache record + valueid lookup + HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); + ValueSegment = Key.substr(ValueSplitOffset + 1); + } - if (!ValidateKeyUri(Request, /* out */ Ref)) + if (HashSegment.size() != IoHash::StringLength) { - std::string_view Key = Request.RelativeUri(); + return false; + } + + IoHash KeyHash; + if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) + { + return false; + } - if (Key == "$rpc") + Data.HashKey = KeyHash; + + if (!ValueSegment.empty()) + { + if (ValueSegment.size() != IoHash::StringLength) { - return HandleRpcRequest(Request); + return false; } - if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) + IoHash ValueHash; + if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) { - // Bucket reference - return HandleCacheBucketRequest(Request, Key); + return false; } + Data.ValueContentId = ValueHash; + } + + return true; +} + +void +HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) +{ + metrics::OperationTiming::Scope $(m_HttpRequests); + + std::string_view Key = Request.RelativeUri(); + if (Key == HttpZCacheRPCPrefix) + { + return HandleRpcRequest(Request); + } + HttpRequestData RequestData; + if (!HttpRequestParseRelativeUri(Key, RequestData)) + { return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL } - CachePolicy PolicyFromURL = ParseCachePolicy(Request.GetQueryParams()); + if (RequestData.ValueContentId.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + ZEN_ASSERT(RequestData.HashKey.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = RequestData.ValueContentId.value()}; + return HandleCacheValueRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); + } - if (Ref.ValueContentId == IoHash::Zero) + if (RequestData.HashKey.has_value()) { - return HandleCacheRecordRequest(Request, Ref, PolicyFromURL); + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = IoHash::Zero}; + return HandleCacheRecordRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); } - else + + if (RequestData.Bucket.has_value()) { - return HandleCacheValueRequest(Request, Ref, PolicyFromURL); + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheBucketRequest(Request, RequestData.Namespace.value(), RequestData.Bucket.value()); } - return; + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheNamespaceRequest(Request, RequestData.Namespace.value()); } void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view) { switch (Request.RequestVerb()) { @@ -173,11 +274,39 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, break; case HttpVerb::kDelete: - // Drop bucket + // Drop namespace { - // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability - std::string_view Namespace = ZenCacheStore::DefaultNamespace; + // if (m_CacheStore.DropNamespace(Namespace)) + // { + // return Request.WriteResponse(HttpResponseCode::OK); + // } + // else + // { + // return Request.WriteResponse(HttpResponseCode::NotFound); + // } + } + break; + default: + break; + } +} + +void +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + // Query stats + } + break; + + case HttpVerb::kDelete: + // Drop bucket + { if (m_CacheStore.DropBucket(Namespace, Bucket)) { return Request.WriteResponse(HttpResponseCode::OK); @@ -783,9 +912,8 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, } bool -HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& OutRef) +HttpStructuredCacheService::ValidateKeyUri(std::string_view Namespace, std::string_view Key, CacheRef& OutRef) { - std::string_view Key = Request.RelativeUri(); std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); if (BucketSplitOffset == std::string_view::npos) @@ -793,8 +921,7 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& return false; } - OutRef.Namespace = ToLower(ZenCacheStore::DefaultNamespace); // TODO: Should add namespace to URI and handle if the namespace is - // missing for backwards compatability + OutRef.Namespace = ToLower(Namespace); OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) -- cgit v1.2.3 From 1274f92cf7ce890b7aa1fc9354503e2508c185eb Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 13:48:45 +0200 Subject: Tests for HttpRequestParseRelativeUri --- zenserver/cache/structuredcache.cpp | 168 ++++++++++++++++++++++++++++++------ 1 file changed, 142 insertions(+), 26 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 691da36fa..9299911cf 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -35,6 +35,11 @@ #include +#if ZEN_WITH_TESTS +# include +# include +#endif + namespace zen { using namespace std::literals; @@ -135,32 +140,51 @@ struct HttpRequestData static bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) { - std::string_view Namespace = ZenCacheStore::DefaultNamespace; if (Key.starts_with(HttpZCacheAPIV2Prefix)) { - std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/', HttpZCacheAPIV2Prefix.length()); + Key = Key.substr(HttpZCacheAPIV2Prefix.length()); + // Namespace reference + if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) + { + return false; + } + Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); + std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); + + std::string_view Namespace = Key.substr(0, NamespaceSplitOffset); + if (!std::all_of(begin(Namespace), end(Namespace), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Namespace = ToLower(Namespace); + if (NamespaceSplitOffset == std::string_view::npos) { - // Namespace reference - if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Namespace = ToLower(Key); return true; } - Data.Namespace = Key.substr(0, NamespaceSplitOffset); - Key = Key.substr(NamespaceSplitOffset + 1); + + Key = Key.substr(NamespaceSplitOffset + 1); + } + else + { + Data.Namespace = ZenCacheStore::DefaultNamespace; } std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); + std::string_view Bucket = Key.substr(0, BucketSplitOffset); + if (Bucket.empty()) + { + return false; + } + if (!std::all_of(begin(Bucket), end(Bucket), [](const char c) { return std::isalnum(c); })) + { + return false; + } + Data.Bucket = ToLower(Bucket); + if (BucketSplitOffset == std::string_view::npos) { - if (!std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Bucket = ToLower(Key); + // Bucket reference return true; } @@ -193,20 +217,22 @@ HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) Data.HashKey = KeyHash; - if (!ValueSegment.empty()) + if (ValueSegment.empty()) { - if (ValueSegment.size() != IoHash::StringLength) - { - return false; - } + return true; + } - IoHash ValueHash; - if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) - { - return false; - } - Data.ValueContentId = ValueHash; + if (ValueSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash ValueHash; + if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) + { + return false; } + Data.ValueContentId = ValueHash; return true; } @@ -2330,4 +2356,94 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request) Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +#if ZEN_WITH_TESTS + +TEST_CASE("z$service.parse.relative.Uri") +{ + HttpRequestData LegacyBucketRequest; + CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequest)); + CHECK(LegacyBucketRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyBucketRequest.Bucket == "test"sv); + CHECK(!LegacyBucketRequest.HashKey.has_value()); + CHECK(!LegacyBucketRequest.ValueContentId.has_value()); + + HttpRequestData LegacyHashKeyRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); + CHECK(LegacyHashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyHashKeyRequest.Bucket == "test"sv); + CHECK(LegacyHashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(!LegacyHashKeyRequest.ValueContentId.has_value()); + + HttpRequestData LegacyValueContentIdRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + LegacyValueContentIdRequest)); + CHECK(LegacyValueContentIdRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(LegacyValueContentIdRequest.Bucket == "test"sv); + CHECK(LegacyValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); + + HttpRequestData V2DefaultNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_", V2DefaultNamespaceRequest)); + CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); + CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); + CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2NamespaceRequest; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace", V2NamespaceRequest)); + CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); + CHECK(!V2NamespaceRequest.Bucket.has_value()); + CHECK(!V2NamespaceRequest.HashKey.has_value()); + CHECK(!V2NamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithDefaultNamespace; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test", V2BucketRequestWithDefaultNamespace)); + CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); + CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithNamespace; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); + CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); + CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); + + HttpRequestData V2HashKeyRequest; + CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); + CHECK(V2HashKeyRequest.Bucket == "test"); + CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(!V2HashKeyRequest.ValueContentId.has_value()); + + HttpRequestData V2ValueContentIdRequest; + CHECK(HttpRequestParseRelativeUri( + "api/v2/ns_nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + V2ValueContentIdRequest)); + CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); + CHECK(V2ValueContentIdRequest.Bucket == "test"sv); + CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); + CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); + + HttpRequestData Invalid; + CHECK(!HttpRequestParseRelativeUri("api/v2/bla", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2//", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri( + "api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + Invalid)); +} + +#endif + +void +z$service_forcelink() +{ +} + } // namespace zen -- cgit v1.2.3 From 4090ba3269bbebeb7dc772bd15b632560a7202b8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 15:23:57 +0200 Subject: Add validation to namespace names that follows Jupiters rules. Add unified validation of Namespace, Bucket and Hash for rpc requests. cleanup --- zenserver/cache/structuredcache.cpp | 417 +++++++++++++++++------------------- 1 file changed, 192 insertions(+), 225 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 9299911cf..06114ed1e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -75,6 +75,169 @@ struct PutRequestData CacheRecordPolicy Policy; }; +namespace { + static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; + static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + + struct HttpRequestData + { + std::optional Namespace; + std::optional Bucket; + std::optional HashKey; + std::optional ValueContentId; + }; + + bool IsValidNamespaceName(std::string_view Name) + { + if (Name == ZenCacheStore::DefaultNamespace) + { + return true; + } + if (Name.empty()) + { + return false; + } + return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + } + + bool IsValidBucketName(std::string_view Name) + { + if (Name.empty()) + { + return false; + } + return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + } + + bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + { + if (Key.starts_with(HttpZCacheAPIV2Prefix)) + { + Key = Key.substr(HttpZCacheAPIV2Prefix.length()); + // Namespace reference + if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) + { + return false; + } + Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); + std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); + + std::string Namespace = ToLower(Key.substr(0, NamespaceSplitOffset)); + if (!IsValidNamespaceName(Namespace)) + { + return false; + } + + Data.Namespace = Namespace; + + if (NamespaceSplitOffset == std::string_view::npos) + { + return true; + } + Key = Key.substr(NamespaceSplitOffset + 1); + } + else + { + Data.Namespace = ZenCacheStore::DefaultNamespace; + } + + std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); + std::string Bucket = ToLower(Key.substr(0, BucketSplitOffset)); + if (!IsValidBucketName(Bucket)) + { + return false; + } + Data.Bucket = Bucket; + + if (BucketSplitOffset == std::string_view::npos) + { + // Bucket reference + return true; + } + + std::string_view HashSegment; + std::string_view ValueSegment; + + std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); + if (ValueSplitOffset == BucketSplitOffset) + { + // Basic cache record lookup + HashSegment = Key.substr(BucketSplitOffset + 1); + } + else + { + // Cache record + valueid lookup + HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); + ValueSegment = Key.substr(ValueSplitOffset + 1); + } + + if (HashSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash KeyHash; + if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) + { + return false; + } + + Data.HashKey = KeyHash; + + if (ValueSegment.empty()) + { + return true; + } + + if (ValueSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash ValueHash; + if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) + { + return false; + } + Data.ValueContentId = ValueHash; + + return true; + } + + bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { + CbFieldView NamespaceField = KeyView["Namespace"sv]; + std::string_view Namespace = NamespaceField.AsString(ZenCacheStore::DefaultNamespace); + CbFieldView BucketField = KeyView["Bucket"sv]; + if (BucketField.HasError()) + { + return false; + } + if (!BucketField.IsString()) + { + return false; + } + std::string_view Bucket = BucketField.AsString(); + if (!IsValidBucketName(Bucket)) + { + return false; + } + CbFieldView HashField = KeyView["Hash"sv]; + if (HashField.HasError()) + { + return false; + } + if (!HashField.IsHash()) + { + return false; + } + IoHash Hash = HashField.AsHash(); + Key = CacheKey::Create(Namespace, Bucket, Hash); + return true; + } + +} // namespace + ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, @@ -126,117 +289,6 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx) m_CacheStore.Scrub(Ctx); } -static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; -static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; - -struct HttpRequestData -{ - std::optional Namespace; - std::optional Bucket; - std::optional HashKey; - std::optional ValueContentId; -}; - -static bool -HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) -{ - if (Key.starts_with(HttpZCacheAPIV2Prefix)) - { - Key = Key.substr(HttpZCacheAPIV2Prefix.length()); - // Namespace reference - if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) - { - return false; - } - Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); - std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); - - std::string_view Namespace = Key.substr(0, NamespaceSplitOffset); - if (!std::all_of(begin(Namespace), end(Namespace), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Namespace = ToLower(Namespace); - - if (NamespaceSplitOffset == std::string_view::npos) - { - return true; - } - - Key = Key.substr(NamespaceSplitOffset + 1); - } - else - { - Data.Namespace = ZenCacheStore::DefaultNamespace; - } - - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - std::string_view Bucket = Key.substr(0, BucketSplitOffset); - if (Bucket.empty()) - { - return false; - } - if (!std::all_of(begin(Bucket), end(Bucket), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Bucket = ToLower(Bucket); - - if (BucketSplitOffset == std::string_view::npos) - { - // Bucket reference - return true; - } - - std::string_view HashSegment; - std::string_view ValueSegment; - - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else - { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } - - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } - - IoHash KeyHash; - if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) - { - return false; - } - - Data.HashKey = KeyHash; - - if (ValueSegment.empty()) - { - return true; - } - - if (ValueSegment.size() != IoHash::StringLength) - { - return false; - } - - IoHash ValueHash; - if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) - { - return false; - } - Data.ValueContentId = ValueHash; - - return true; -} - void HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { @@ -937,77 +989,6 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, Request.WriteResponse(ResponseCode); } -bool -HttpStructuredCacheService::ValidateKeyUri(std::string_view Namespace, std::string_view Key, CacheRef& OutRef) -{ - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - - if (BucketSplitOffset == std::string_view::npos) - { - return false; - } - - OutRef.Namespace = ToLower(Namespace); - OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); - - if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - if (!std::all_of(begin(OutRef.BucketSegment), end(OutRef.BucketSegment), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - std::string_view HashSegment; - std::string_view ValueSegment; - - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - - // We know there is a slash so no need to check for npos return - - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else - { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } - - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } - - if (!ValueSegment.empty() && ValueSegment.size() == IoHash::StringLength) - { - const bool IsOk = ParseHexBytes(ValueSegment.data(), ValueSegment.size(), OutRef.ValueContentId.Hash); - - if (!IsOk) - { - return false; - } - } - else - { - OutRef.ValueContentId = IoHash::Zero; - } - - const bool IsOk = ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); - - if (!IsOk) - { - return false; - } - - return true; -} - void HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) { @@ -1088,15 +1069,12 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); - CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); + CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } @@ -1265,18 +1243,16 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt for (CbFieldView RequestField : RequestsArray) { - RecordRequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (HashField.HasError() || Key.Bucket.empty()) + RecordRequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + CacheKey& Key = Request.Upstream.Key; + if (!GetRpcRequestCacheKey(KeyObject, Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + Request.DownstreamPolicy = LoadCacheRecordPolicy(RequestObject["Policy"sv].AsObjectView(), DefaultPolicy); const CacheRecordPolicy& Policy = Request.DownstreamPolicy; @@ -1548,17 +1524,15 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); CachePolicy Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; IoHash RawHash = RequestObject["RawHash"sv].AsBinaryAttachment(); @@ -1664,18 +1638,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http { Stopwatch Timer; - RequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) + RequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + if (!GetRpcRequestCacheKey(KeyObject, Request.Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); Request.Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; @@ -1921,15 +1892,10 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vectorKey)) { ZEN_WARN("GetCacheChunks: Invalid key in ChunkRequest."); return false; @@ -2432,6 +2398,7 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(!HttpRequestParseRelativeUri("api/v2/ns_bad\2_namespace", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_nice/\2\1bucket", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); CHECK(!HttpRequestParseRelativeUri( -- cgit v1.2.3 From c9a2cc8afa11f3dfea05b91f0758ba3d0cae0784 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 15:33:18 +0200 Subject: keep compatability for valid bucket names --- zenserver/cache/structuredcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 06114ed1e..83f2f41d9 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -106,7 +106,7 @@ namespace { { return false; } - return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + return std::all_of(begin(Name), end(Name), [](const char c) { return std::isalnum(c); }); } bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) -- cgit v1.2.3 From e9466f9684d5479029d73fd2a60327d1daa89192 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 15:33:58 +0200 Subject: Validate max length for namespace name --- zenserver/cache/structuredcache.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 83f2f41d9..02cecf3e1 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -97,6 +97,10 @@ namespace { { return false; } + if (Name.length() > 64) + { + return false; + } return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; } -- cgit v1.2.3 From 4be7a0cb5b83732e780a9247636a27172f2fb3e1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 12:56:14 +0200 Subject: use "default" as the default namespace remove ns_ prefix for namespaces on disk and in requests --- zenserver/cache/structuredcache.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 02cecf3e1..9f0ff7408 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -2353,35 +2353,35 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData V2DefaultNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_", V2DefaultNamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("api/v2/default", V2DefaultNamespaceRequest)); CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); HttpRequestData V2NamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace", V2NamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace", V2NamespaceRequest)); CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); CHECK(!V2NamespaceRequest.Bucket.has_value()); CHECK(!V2NamespaceRequest.HashKey.has_value()); CHECK(!V2NamespaceRequest.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithDefaultNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test", V2BucketRequestWithDefaultNamespace)); + CHECK(HttpRequestParseRelativeUri("api/v2/default/test", V2BucketRequestWithDefaultNamespace)); CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace/test", V2BucketRequestWithNamespace)); CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); HttpRequestData V2HashKeyRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/ns_/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(HttpRequestParseRelativeUri("api/v2/default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2HashKeyRequest.Bucket == "test"); CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); @@ -2389,7 +2389,7 @@ TEST_CASE("z$service.parse.relative.Uri") HttpRequestData V2ValueContentIdRequest; CHECK(HttpRequestParseRelativeUri( - "api/v2/ns_nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + "api/v2/nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", V2ValueContentIdRequest)); CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); CHECK(V2ValueContentIdRequest.Bucket == "test"sv); @@ -2397,16 +2397,16 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData Invalid; - CHECK(!HttpRequestParseRelativeUri("api/v2/bla", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2//", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_bad\2_namespace", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_nice/\2\1bucket", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789a", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); CHECK(!HttpRequestParseRelativeUri( - "api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + "api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", Invalid)); } -- cgit v1.2.3 From 2f69a30c936bc475bf85daded3706dc67f2e8a0f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 13:10:10 +0200 Subject: use ns_ prefix on disk only --- zenserver/cache/structuredcache.cpp | 5 ----- 1 file changed, 5 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 9f0ff7408..6d3211c0b 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -119,11 +119,6 @@ namespace { { Key = Key.substr(HttpZCacheAPIV2Prefix.length()); // Namespace reference - if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) - { - return false; - } - Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); std::string Namespace = ToLower(Key.substr(0, NamespaceSplitOffset)); -- cgit v1.2.3 From c442837aff6212c711e959a44fba7c070bcdcaf1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 15:55:07 +0200 Subject: drop api/v2 prefix for non-legacy requests --- zenserver/cache/structuredcache.cpp | 284 ++++++++++++++++++++++-------------- 1 file changed, 172 insertions(+), 112 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 6d3211c0b..b6b5b218e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -87,127 +87,188 @@ namespace { std::optional ValueContentId; }; - bool IsValidNamespaceName(std::string_view Name) + const char* ValidNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + std::optional GetValidNamespaceName(std::string_view Name) { - if (Name == ZenCacheStore::DefaultNamespace) - { - return true; - } if (Name.empty()) { - return false; + return {}; } if (Name.length() > 64) { - return false; + return {}; } - return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; - } - bool IsValidBucketName(std::string_view Name) - { - if (Name.empty()) + if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) { - return false; + return {}; } - return std::all_of(begin(Name), end(Name), [](const char c) { return std::isalnum(c); }); + return ToLower(Name); } - bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + std::optional GetValidBucketName(std::string_view Name) { - if (Key.starts_with(HttpZCacheAPIV2Prefix)) + if (Name.empty()) { - Key = Key.substr(HttpZCacheAPIV2Prefix.length()); - // Namespace reference - std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); - - std::string Namespace = ToLower(Key.substr(0, NamespaceSplitOffset)); - if (!IsValidNamespaceName(Namespace)) - { - return false; - } - - Data.Namespace = Namespace; - - if (NamespaceSplitOffset == std::string_view::npos) - { - return true; - } - Key = Key.substr(NamespaceSplitOffset + 1); + return {}; } - else + if (Name.find_first_not_of(ValidNameCharacters) != std::string::npos) { - Data.Namespace = ZenCacheStore::DefaultNamespace; + return {}; } + return ToLower(Name); + } - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - std::string Bucket = ToLower(Key.substr(0, BucketSplitOffset)); - if (!IsValidBucketName(Bucket)) + std::optional GetValidIoHash(std::string_view Hash) + { + if (Hash.length() != IoHash::StringLength) { - return false; + return {}; } - Data.Bucket = Bucket; - if (BucketSplitOffset == std::string_view::npos) + IoHash KeyHash; + if (!ParseHexBytes(Hash.data(), Hash.size(), KeyHash.Hash)) { - // Bucket reference - return true; + return {}; } + return KeyHash; + } - std::string_view HashSegment; - std::string_view ValueSegment; + bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + { + std::vector Tokens; + uint32_t TokenCount = zen::ForEachStrTok(Key, '/', [&](const std::string_view& Token) { + Tokens.push_back(Token); + return true; + }); - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else + switch (TokenCount) { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } + case 1: + Data.Namespace = GetValidNamespaceName(Tokens[0]); + return Data.Namespace.has_value(); + case 2: + { + std::optional PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + Data.Namespace = ZenCacheStore::DefaultNamespace; + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + return true; + } + case 3: + { + std::optional PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key/valueid request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + Data.ValueContentId = GetValidIoHash(Tokens[2]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + Data.Namespace = ZenCacheStore::DefaultNamespace; + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey) + { + return false; + } + return true; + } + case 4: + { + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } - IoHash KeyHash; - if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) - { - return false; - } + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey.has_value()) + { + return false; + } - Data.HashKey = KeyHash; + Data.ValueContentId = GetValidIoHash(Tokens[3]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + return true; + } + default: + return false; + } + } - if (ValueSegment.empty()) + bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { + CbFieldView NamespaceField = KeyView["Namespace"sv]; + std::optional Namespace; + if (!NamespaceField) { - return true; + Namespace = ZenCacheStore::DefaultNamespace; } - - if (ValueSegment.size() != IoHash::StringLength) + else { - return false; + if (NamespaceField.HasError()) + { + return false; + } + if (!NamespaceField.IsString()) + { + return false; + } + Namespace = GetValidNamespaceName(NamespaceField.AsString()); } - - IoHash ValueHash; - if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) + if (!Namespace.has_value()) { return false; } - Data.ValueContentId = ValueHash; - - return true; - } - - bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) - { - CbFieldView NamespaceField = KeyView["Namespace"sv]; - std::string_view Namespace = NamespaceField.AsString(ZenCacheStore::DefaultNamespace); - CbFieldView BucketField = KeyView["Bucket"sv]; + CbFieldView BucketField = KeyView["Bucket"sv]; if (BucketField.HasError()) { return false; @@ -216,8 +277,8 @@ namespace { { return false; } - std::string_view Bucket = BucketField.AsString(); - if (!IsValidBucketName(Bucket)) + std::optional Bucket = GetValidBucketName(BucketField.AsString()); + if (!Bucket.has_value()) { return false; } @@ -231,7 +292,7 @@ namespace { return false; } IoHash Hash = HashField.AsHash(); - Key = CacheKey::Create(Namespace, Bucket, Hash); + Key = CacheKey::Create(*Namespace, *Bucket, Hash); return true; } @@ -2325,12 +2386,12 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request) TEST_CASE("z$service.parse.relative.Uri") { - HttpRequestData LegacyBucketRequest; - CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequest)); - CHECK(LegacyBucketRequest.Namespace == ZenCacheStore::DefaultNamespace); - CHECK(LegacyBucketRequest.Bucket == "test"sv); - CHECK(!LegacyBucketRequest.HashKey.has_value()); - CHECK(!LegacyBucketRequest.ValueContentId.has_value()); + HttpRequestData LegacyBucketRequestBecomesNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequestBecomesNamespaceRequest)); + CHECK(LegacyBucketRequestBecomesNamespaceRequest.Namespace == "test"sv); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.Bucket.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.HashKey.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.ValueContentId.has_value()); HttpRequestData LegacyHashKeyRequest; CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); @@ -2348,61 +2409,60 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData V2DefaultNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/default", V2DefaultNamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("default", V2DefaultNamespaceRequest)); CHECK(V2DefaultNamespaceRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); HttpRequestData V2NamespaceRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace", V2NamespaceRequest)); + CHECK(HttpRequestParseRelativeUri("nicenamespace", V2NamespaceRequest)); CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); CHECK(!V2NamespaceRequest.Bucket.has_value()); CHECK(!V2NamespaceRequest.HashKey.has_value()); CHECK(!V2NamespaceRequest.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithDefaultNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/default/test", V2BucketRequestWithDefaultNamespace)); + CHECK(HttpRequestParseRelativeUri("default/test", V2BucketRequestWithDefaultNamespace)); CHECK(V2BucketRequestWithDefaultNamespace.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); HttpRequestData V2BucketRequestWithNamespace; - CHECK(HttpRequestParseRelativeUri("api/v2/nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(HttpRequestParseRelativeUri("nicenamespace/test", V2BucketRequestWithNamespace)); CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); HttpRequestData V2HashKeyRequest; - CHECK(HttpRequestParseRelativeUri("api/v2/default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(HttpRequestParseRelativeUri("default/test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); CHECK(V2HashKeyRequest.Bucket == "test"); CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); CHECK(!V2HashKeyRequest.ValueContentId.has_value()); HttpRequestData V2ValueContentIdRequest; - CHECK(HttpRequestParseRelativeUri( - "api/v2/nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", - V2ValueContentIdRequest)); + CHECK( + HttpRequestParseRelativeUri("nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + V2ValueContentIdRequest)); CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); CHECK(V2ValueContentIdRequest.Bucket == "test"sv); CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); HttpRequestData Invalid; - CHECK(!HttpRequestParseRelativeUri("api/v2/", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2//", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/bad\2_namespace", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/nice/\2\1bucket", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789a", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); - CHECK(!HttpRequestParseRelativeUri( - "api/v2/namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", - Invalid)); + CHECK(!HttpRequestParseRelativeUri("", Invalid)); + CHECK(!HttpRequestParseRelativeUri("/", Invalid)); + CHECK(!HttpRequestParseRelativeUri("bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + Invalid)); } #endif -- cgit v1.2.3 From d0440e215136a3b4dbb417f4fe317b8bb223e094 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 May 2022 17:01:45 +0200 Subject: review feedback --- zenserver/cache/structuredcache.cpp | 53 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 27 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index b6b5b218e..c457c59b2 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -76,8 +76,7 @@ struct PutRequestData }; namespace { - static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; - static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; struct HttpRequestData { @@ -462,19 +461,19 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { switch (Request.RequestVerb()) { case HttpVerb::kHead: case HttpVerb::kGet: { - HandleGetCacheRecord(Request, Ref, PolicyFromURL); + HandleGetCacheRecord(Request, Ref, PolicyFromUrl); } break; case HttpVerb::kPut: - HandlePutCacheRecord(Request, Ref, PolicyFromURL); + HandlePutCacheRecord(Request, Ref, PolicyFromUrl); break; default: break; @@ -482,20 +481,20 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { const ZenContentType AcceptType = Request.AcceptContentType(); - const bool SkipData = EnumHasAllFlags(PolicyFromURL, CachePolicy::SkipData); - const bool PartialRecord = EnumHasAllFlags(PolicyFromURL, CachePolicy::PartialRecord); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); bool Success = false; ZenCacheValue ClientResultValue; - if (!EnumHasAnyFlags(PolicyFromURL, CachePolicy::Query)) + if (!EnumHasAnyFlags(PolicyFromUrl, CachePolicy::Query)) { return Request.WriteResponse(HttpResponseCode::OK); } - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && m_CacheStore.Get(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { Success = true; @@ -575,7 +574,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::OK, ClientResultValue.Value.GetContentType(), ClientResultValue.Value); } } - else if (!EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryRemote)) + else if (!EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote)) { ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); m_CacheStats.MissCount++; @@ -585,12 +584,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request // Issue upstream query asynchronously in order to keep requests flowing without // hogging I/O servicing threads with blocking work - Request.WriteResponseAsync([this, AcceptType, PolicyFromURL, Ref](HttpServerRequest& AsyncRequest) { + Request.WriteResponseAsync([this, AcceptType, PolicyFromUrl, Ref](HttpServerRequest& AsyncRequest) { bool Success = false; - const bool PartialRecord = EnumHasAllFlags(PolicyFromURL, CachePolicy::PartialRecord); - const bool QueryLocal = EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal); - const bool StoreLocal = EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreLocal); - const bool SkipData = EnumHasAllFlags(PolicyFromURL, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); + const bool QueryLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + const bool StoreLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreLocal); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); ZenCacheValue ClientResultValue; metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); @@ -749,7 +748,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } void -HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { IoBuffer Body = Request.ReadPayload(); @@ -767,7 +766,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); - if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote)) + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) { m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}}); } @@ -788,7 +787,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); } - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; CbObjectView CacheRecord(Body.Data()); std::vector ValidAttachments; int32_t TotalCount = 0; @@ -834,7 +833,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid package", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); } - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; CbObject CacheRecord = Package.GetObject(); AttachmentCount Count; @@ -917,16 +916,16 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } void -HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { switch (Request.RequestVerb()) { case HttpVerb::kHead: case HttpVerb::kGet: - HandleGetCacheValue(Request, Ref, PolicyFromURL); + HandleGetCacheValue(Request, Ref, PolicyFromUrl); break; case HttpVerb::kPut: - HandlePutCacheValue(Request, Ref, PolicyFromURL); + HandlePutCacheValue(Request, Ref, PolicyFromUrl); break; default: break; @@ -934,13 +933,13 @@ HttpStructuredCacheService::HandleCacheValueRequest(HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { Stopwatch Timer; IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); bool InUpstreamCache = false; - CachePolicy Policy = PolicyFromURL; + CachePolicy Policy = PolicyFromUrl; { const bool QueryUpstream = !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); @@ -1002,10 +1001,10 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request, } void -HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL) +HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) { // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored - ZEN_UNUSED(PolicyFromURL); + ZEN_UNUSED(PolicyFromUrl); Stopwatch Timer; -- cgit v1.2.3