From 4090ba3269bbebeb7dc772bd15b632560a7202b8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 May 2022 15:23:57 +0200 Subject: Add validation to namespace names that follows Jupiters rules. Add unified validation of Namespace, Bucket and Hash for rpc requests. cleanup --- zenserver/cache/structuredcache.cpp | 417 +++++++++++++++++------------------- 1 file changed, 192 insertions(+), 225 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 9299911cf..06114ed1e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -75,6 +75,169 @@ struct PutRequestData CacheRecordPolicy Policy; }; +namespace { + static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; + static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + + struct HttpRequestData + { + std::optional Namespace; + std::optional Bucket; + std::optional HashKey; + std::optional ValueContentId; + }; + + bool IsValidNamespaceName(std::string_view Name) + { + if (Name == ZenCacheStore::DefaultNamespace) + { + return true; + } + if (Name.empty()) + { + return false; + } + return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + } + + bool IsValidBucketName(std::string_view Name) + { + if (Name.empty()) + { + return false; + } + return Name.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-_.") == std::string::npos; + } + + bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + { + if (Key.starts_with(HttpZCacheAPIV2Prefix)) + { + Key = Key.substr(HttpZCacheAPIV2Prefix.length()); + // Namespace reference + if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) + { + return false; + } + Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); + std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); + + std::string Namespace = ToLower(Key.substr(0, NamespaceSplitOffset)); + if (!IsValidNamespaceName(Namespace)) + { + return false; + } + + Data.Namespace = Namespace; + + if (NamespaceSplitOffset == std::string_view::npos) + { + return true; + } + Key = Key.substr(NamespaceSplitOffset + 1); + } + else + { + Data.Namespace = ZenCacheStore::DefaultNamespace; + } + + std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); + std::string Bucket = ToLower(Key.substr(0, BucketSplitOffset)); + if (!IsValidBucketName(Bucket)) + { + return false; + } + Data.Bucket = Bucket; + + if (BucketSplitOffset == std::string_view::npos) + { + // Bucket reference + return true; + } + + std::string_view HashSegment; + std::string_view ValueSegment; + + std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); + if (ValueSplitOffset == BucketSplitOffset) + { + // Basic cache record lookup + HashSegment = Key.substr(BucketSplitOffset + 1); + } + else + { + // Cache record + valueid lookup + HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); + ValueSegment = Key.substr(ValueSplitOffset + 1); + } + + if (HashSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash KeyHash; + if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) + { + return false; + } + + Data.HashKey = KeyHash; + + if (ValueSegment.empty()) + { + return true; + } + + if (ValueSegment.size() != IoHash::StringLength) + { + return false; + } + + IoHash ValueHash; + if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) + { + return false; + } + Data.ValueContentId = ValueHash; + + return true; + } + + bool GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { + CbFieldView NamespaceField = KeyView["Namespace"sv]; + std::string_view Namespace = NamespaceField.AsString(ZenCacheStore::DefaultNamespace); + CbFieldView BucketField = KeyView["Bucket"sv]; + if (BucketField.HasError()) + { + return false; + } + if (!BucketField.IsString()) + { + return false; + } + std::string_view Bucket = BucketField.AsString(); + if (!IsValidBucketName(Bucket)) + { + return false; + } + CbFieldView HashField = KeyView["Hash"sv]; + if (HashField.HasError()) + { + return false; + } + if (!HashField.IsHash()) + { + return false; + } + IoHash Hash = HashField.AsHash(); + Key = CacheKey::Create(Namespace, Bucket, Hash); + return true; + } + +} // namespace + ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, @@ -126,117 +289,6 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx) m_CacheStore.Scrub(Ctx); } -static constexpr std::string_view HttpZCacheAPIV2Prefix = "api/v2/"sv; -static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv; - -struct HttpRequestData -{ - std::optional Namespace; - std::optional Bucket; - std::optional HashKey; - std::optional ValueContentId; -}; - -static bool -HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) -{ - if (Key.starts_with(HttpZCacheAPIV2Prefix)) - { - Key = Key.substr(HttpZCacheAPIV2Prefix.length()); - // Namespace reference - if (!Key.starts_with(ZenCacheStore::NamespacePrefix)) - { - return false; - } - Key = Key.substr(ZenCacheStore::NamespacePrefix.length()); - std::string_view::size_type NamespaceSplitOffset = Key.find_first_of('/'); - - std::string_view Namespace = Key.substr(0, NamespaceSplitOffset); - if (!std::all_of(begin(Namespace), end(Namespace), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Namespace = ToLower(Namespace); - - if (NamespaceSplitOffset == std::string_view::npos) - { - return true; - } - - Key = Key.substr(NamespaceSplitOffset + 1); - } - else - { - Data.Namespace = ZenCacheStore::DefaultNamespace; - } - - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - std::string_view Bucket = Key.substr(0, BucketSplitOffset); - if (Bucket.empty()) - { - return false; - } - if (!std::all_of(begin(Bucket), end(Bucket), [](const char c) { return std::isalnum(c); })) - { - return false; - } - Data.Bucket = ToLower(Bucket); - - if (BucketSplitOffset == std::string_view::npos) - { - // Bucket reference - return true; - } - - std::string_view HashSegment; - std::string_view ValueSegment; - - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else - { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } - - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } - - IoHash KeyHash; - if (!ParseHexBytes(HashSegment.data(), HashSegment.size(), KeyHash.Hash)) - { - return false; - } - - Data.HashKey = KeyHash; - - if (ValueSegment.empty()) - { - return true; - } - - if (ValueSegment.size() != IoHash::StringLength) - { - return false; - } - - IoHash ValueHash; - if (!ParseHexBytes(ValueSegment.data(), ValueSegment.size(), ValueHash.Hash)) - { - return false; - } - Data.ValueContentId = ValueHash; - - return true; -} - void HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { @@ -937,77 +989,6 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request, Request.WriteResponse(ResponseCode); } -bool -HttpStructuredCacheService::ValidateKeyUri(std::string_view Namespace, std::string_view Key, CacheRef& OutRef) -{ - std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); - - if (BucketSplitOffset == std::string_view::npos) - { - return false; - } - - OutRef.Namespace = ToLower(Namespace); - OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset)); - - if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - if (!std::all_of(begin(OutRef.BucketSegment), end(OutRef.BucketSegment), [](const char c) { return std::isalnum(c); })) - { - return false; - } - - std::string_view HashSegment; - std::string_view ValueSegment; - - std::string_view::size_type ValueSplitOffset = Key.find_last_of('/'); - - // We know there is a slash so no need to check for npos return - - if (ValueSplitOffset == BucketSplitOffset) - { - // Basic cache record lookup - HashSegment = Key.substr(BucketSplitOffset + 1); - } - else - { - // Cache record + valueid lookup - HashSegment = Key.substr(BucketSplitOffset + 1, ValueSplitOffset - BucketSplitOffset - 1); - ValueSegment = Key.substr(ValueSplitOffset + 1); - } - - if (HashSegment.size() != IoHash::StringLength) - { - return false; - } - - if (!ValueSegment.empty() && ValueSegment.size() == IoHash::StringLength) - { - const bool IsOk = ParseHexBytes(ValueSegment.data(), ValueSegment.size(), OutRef.ValueContentId.Hash); - - if (!IsOk) - { - return false; - } - } - else - { - OutRef.ValueContentId = IoHash::Zero; - } - - const bool IsOk = ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); - - if (!IsOk) - { - return false; - } - - return true; -} - void HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) { @@ -1088,15 +1069,12 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); - CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView(); + CbObjectView KeyView = RecordObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } @@ -1265,18 +1243,16 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt for (CbFieldView RequestField : RequestsArray) { - RecordRequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - CacheKey& Key = Request.Upstream.Key; - Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (HashField.HasError() || Key.Bucket.empty()) + RecordRequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + CacheKey& Key = Request.Upstream.Key; + if (!GetRpcRequestCacheKey(KeyObject, Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + Request.DownstreamPolicy = LoadCacheRecordPolicy(RequestObject["Policy"sv].AsObjectView(), DefaultPolicy); const CacheRecordPolicy& Policy = Request.DownstreamPolicy; @@ -1548,17 +1524,15 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ std::vector Results; for (CbFieldView RequestField : Params["Requests"sv]) { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyView["Namespace"sv]; - CbFieldView BucketField = KeyView["Bucket"sv]; - CbFieldView HashField = KeyView["Hash"sv]; - CacheKey Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty()) + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyView = RequestObject["Key"sv].AsObjectView(); + + CacheKey Key; + if (!GetRpcRequestCacheKey(KeyView, Key)) { return Request.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); CachePolicy Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; IoHash RawHash = RequestObject["RawHash"sv].AsBinaryAttachment(); @@ -1664,18 +1638,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http { Stopwatch Timer; - RequestData& Request = Requests.emplace_back(); - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CbFieldView NamespaceField = KeyObject["Namespace"sv]; - CbFieldView BucketField = KeyObject["Bucket"sv]; - CbFieldView HashField = KeyObject["Hash"sv]; - Request.Key = - CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash()); - if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty()) + RequestData& Request = Requests.emplace_back(); + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + + if (!GetRpcRequestCacheKey(KeyObject, Request.Key)) { return HttpRequest.WriteResponse(HttpResponseCode::BadRequest); } + PolicyText = RequestObject["Policy"sv].AsString(); Request.Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; @@ -1921,15 +1892,10 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vectorKey)) { ZEN_WARN("GetCacheChunks: Invalid key in ChunkRequest."); return false; @@ -2432,6 +2398,7 @@ TEST_CASE("z$service.parse.relative.Uri") CHECK(!HttpRequestParseRelativeUri("api/v2/ns_bad\2_namespace", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_nice/\2\1bucket", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); CHECK(!HttpRequestParseRelativeUri("api/v2/ns_namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); CHECK(!HttpRequestParseRelativeUri( -- cgit v1.2.3