From bd85a74a9d15fd676a6677fbd4d5ab4e3dcb0d42 Mon Sep 17 00:00:00 2001 From: mattpetersepic <58296718+mattpetersepic@users.noreply.github.com> Date: Tue, 25 Jan 2022 06:57:47 -0700 Subject: Cachepolicy (#36) * Copy CachePolicy implementation from UE5/Release-5.0. Add backwards compatability for clients and upstreams that are using the old protocol. * Add RefPtr templated move operator and constructor, so that RefPtr A = std::move(RefPtr()) will do a move. * Fix broken CachePolicy tests and add tests for new Save/Load. * Remove TODO comments * CachePolicy Save/Load Fixes from codereview * Fix comment to match code change. * Remove backwards compatibility for CachePolicy change. Convert policy string tokens to PascalCase. Fix tests for new policy text. Change ParseCachePolicy to assert string is non-empty and always succeed. * Fix release build: use ZEN_WITH_TESTS define --- zenserver/cache/structuredcache.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'zenserver/cache/structuredcache.h') diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 1bf3940e7..a7ecba845 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -41,7 +41,7 @@ enum class CachePolicy : uint32_t; * * Additionally, attachments may be addressed as: * - * {BucketId}/{KeyHash}/{PayloadHash} + * {BucketId}/{KeyHash}/{ValueHash} * * Where the two initial components are the same as for the main endpoint * @@ -84,12 +84,12 @@ private: }; [[nodiscard]] bool ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef); - void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); - void HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); - void HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); - void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); - void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); - void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); + void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); + void HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); + void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); + void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); + void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); void HandleRpcRequest(zen::HttpServerRequest& Request); void HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); void HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest); -- cgit v1.2.3 From 9f60e3a53ea082e4dd2553d50c4c03fa9e9a42c8 Mon Sep 17 00:00:00 2001 From: mattpetersepic <58296718+mattpetersepic@users.noreply.github.com> Date: Thu, 27 Jan 2022 09:16:05 -0700 Subject: Add batched CacheRecord put rpc (#38) * Add batched CacheRecord put rpc --- zenserver/cache/structuredcache.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'zenserver/cache/structuredcache.h') diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index a7ecba845..5b6b10898 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -19,6 +19,7 @@ namespace zen { class CasStore; class CidStore; class CbObjectView; +struct PutRequestData; class ScrubContext; class UpstreamCache; class ZenCacheStore; @@ -82,6 +83,12 @@ private: std::atomic_uint64_t UpstreamHitCount{}; std::atomic_uint64_t MissCount{}; }; + enum class PutResult + { + Success, + Fail, + Invalid, + }; [[nodiscard]] bool ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef); void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); @@ -91,11 +98,13 @@ private: void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); void HandleRpcRequest(zen::HttpServerRequest& Request); + void HandleRpcPutCacheRecords(zen::HttpServerRequest& Request, const CbPackage& BatchRequest); void HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); void HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; + PutResult PutCacheRecord(PutRequestData& Request, const CbPackage* Package); spdlog::logger& Log() { return m_Log; } spdlog::logger& m_Log; -- cgit v1.2.3 From fb1afc87a436ff6374daeaef5f7682f098a85cd7 Mon Sep 17 00:00:00 2001 From: mattpetersepic <58296718+mattpetersepic@users.noreply.github.com> Date: Thu, 27 Jan 2022 10:52:07 -0700 Subject: Rename Paylod to Value to match the client side. Rename PayloadId to ValueContentId where its a hash instead of an oid. --- zenserver/cache/structuredcache.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'zenserver/cache/structuredcache.h') diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 5b6b10898..88bf6cda1 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -74,7 +74,7 @@ private: { std::string BucketSegment; IoHash HashKey; - IoHash PayloadId; + IoHash ValueContentId; }; struct CacheStats @@ -94,13 +94,13 @@ private: void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); void HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); void HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); - void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); + void HandleCacheValueRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); + void HandleGetCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); + void HandlePutCacheValue(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL); void HandleRpcRequest(zen::HttpServerRequest& Request); void HandleRpcPutCacheRecords(zen::HttpServerRequest& Request, const CbPackage& BatchRequest); void HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); - void HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest); + void HandleRpcGetCacheValues(zen::HttpServerRequest& Request, CbObjectView BatchRequest); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; -- cgit v1.2.3