cache requests API (#134)

1 files changed, 264 insertions, 0 deletions

diff --git a/zenutil/include/zenutil/cache/cacherequests.h b/zenutil/include/zenutil/cache/cacherequests.h
new file mode 100644
index 000000000..aeaa5beee
--- /dev/null
+++ b/zenutil/include/zenutil/cache/cacherequests.h
@@ -0,0 +1,264 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compress.h>
+
+#include "cachekey.h"
+#include "cachepolicy.h"
+
+#include <functional>
+
+namespace zen {
+
+class CbPackage;
+class CbObjectWriter;
+class CbObjectView;
+
+namespace cacherequests {
+	// I'd really like to get rid of std::optional<CacheRecordPolicy> (or really the class CacheRecordPolicy)
+	//
+	// CacheRecordPolicy has a record level policy but it can also contain policies for individual
+	// values inside the record.
+	//
+	// However, when we do a "PutCacheRecords" we already list the individual Values with their Id
+	// so we can just as well use an optional plain CachePolicy for each value.
+	//
+	// In "GetCacheRecords" we do not currently as for the individual values but you can add
+	// a policy on a per-value level in the std::optional<CacheRecordPolicy> Policy for each record.
+	//
+	// But as we already need to know the Ids of the values we want to set the policy for
+	// it would be simpler to add an array of requested values which each has an optional policy.
+	//
+	// We could add:
+	// struct GetCacheRecordValueRequest
+	// {
+	//     Oid						  Id;
+	//     std::optional<CachePolicy> Policy;
+	// };
+	//
+	// and change GetCacheRecordRequest to
+	// struct GetCacheRecordRequest
+	// {
+	// 	CacheKey				                Key = CacheKey::Empty;
+	//  std::vector<GetCacheRecordValueRequest> ValueRequests;
+	// 	std::optional<CachePolicy>              Policy;
+	// };
+	//
+	// This way we don't need the complex CacheRecordPolicy class and the request becomes
+	// more uniform and easier to understand.
+	//
+	// Would need to decide what the ValueRequests actually mean:
+	// Do they dictate which values to fetch or just a change of the policy?
+	// If they dictate the values to fetch you need to know all the value ids to set them
+	// and that is unlikely what we want - we want to be able to get a cache record with
+	// all its values without knowing all the Ids, right?
+	//
+
+	//////////////////////////////////////////////////////////////////////////
+	// Put 1..n structured cache records with optional attachments
+
+	struct PutCacheRecordRequestValue
+	{
+		Oid				 Id		 = Oid::Zero;
+		IoHash			 RawHash = IoHash::Zero;  // If Body is not set, this must be set and the value must already exist in cache
+		CompressedBuffer Body	 = CompressedBuffer::Null;
+	};
+
+	struct PutCacheRecordRequest
+	{
+		CacheKey								Key = CacheKey::Empty;
+		std::function<IoBuffer()>				RecordBody;	 // Leave unset for PutCacheRecordRequest::Format
+		std::vector<PutCacheRecordRequestValue> Values;
+		std::optional<CacheRecordPolicy>		Policy;
+	};
+
+	struct PutCacheRecordsRequest
+	{
+		CachePolicy						   DefaultPolicy = CachePolicy::Default;
+		std::string						   Namespace;
+		std::vector<PutCacheRecordRequest> Requests;
+
+		bool Parse(const CbPackage& Package);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	struct PutCacheRecordsResult
+	{
+		std::vector<bool> Success;
+
+		bool Parse(const CbPackage& Package);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	//////////////////////////////////////////////////////////////////////////
+	// Get 1..n structured cache records with optional attachments
+
+	struct GetCacheRecordRequest
+	{
+		CacheKey						 Key = CacheKey::Empty;
+		std::optional<CacheRecordPolicy> Policy;
+	};
+
+	struct GetCacheRecordsRequest
+	{
+		CachePolicy						   DefaultPolicy = CachePolicy::Default;
+		std::string						   Namespace;
+		std::vector<GetCacheRecordRequest> Requests;
+
+		bool Parse(const CbPackage& RpcRequest);
+		bool Parse(const CbObjectView& RpcRequest);
+		bool Format(CbPackage& OutPackage) const;
+		bool Format(CbObjectWriter& Writer) const;
+	};
+
+	struct GetCacheRecordResultValue
+	{
+		Oid				 Id		 = Oid::Zero;
+		IoHash			 RawHash = IoHash::Zero;
+		uint64_t		 RawSize = 0;
+		CompressedBuffer Body	 = CompressedBuffer::Null;
+	};
+
+	struct GetCacheRecordResult
+	{
+		CacheKey							   Key = CacheKey::Empty;
+		std::function<IoBuffer()>			   RecordBody;
+		std::vector<GetCacheRecordResultValue> Values;
+	};
+
+	struct GetCacheRecordsResult
+	{
+		std::vector<std::optional<GetCacheRecordResult>> Results;
+
+		bool Parse(const CbPackage& Package);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	//////////////////////////////////////////////////////////////////////////
+	// Put 1..n unstructured cache objects
+
+	struct PutCacheValueRequest
+	{
+		CacheKey				   Key	   = CacheKey::Empty;
+		IoHash					   RawHash = IoHash::Zero;
+		CompressedBuffer		   Body	   = CompressedBuffer::Null;  // If not set the value is expected to already exist in cache store
+		std::optional<CachePolicy> Policy;
+	};
+
+	struct PutCacheValuesRequest
+	{
+		CachePolicy						  DefaultPolicy = CachePolicy::Default;
+		std::string						  Namespace;
+		std::vector<PutCacheValueRequest> Requests;
+
+		bool Parse(const CbPackage& Package);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	struct PutCacheValuesResult
+	{
+		std::vector<bool> Success;
+
+		bool Parse(const CbPackage& Package);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	//////////////////////////////////////////////////////////////////////////
+	// Get 1..n unstructured cache objects (stored data may be structured or unstructured)
+
+	struct GetCacheValueRequest
+	{
+		CacheKey				   Key = CacheKey::Empty;
+		std::optional<CachePolicy> Policy;
+	};
+
+	struct GetCacheValuesRequest
+	{
+		CachePolicy						  DefaultPolicy = CachePolicy::Default;
+		std::string						  Namespace;
+		std::vector<GetCacheValueRequest> Requests;
+
+		bool Parse(const CbObjectView& BatchObject);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	struct CacheValueResult
+	{
+		uint64_t		 RawSize = 0;
+		IoHash			 RawHash = IoHash::Zero;
+		CompressedBuffer Body	 = CompressedBuffer::Null;
+	};
+
+	struct CacheValuesResult
+	{
+		std::vector<CacheValueResult> Results;
+
+		bool Parse(const CbPackage& Package);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	//////////////////////////////////////////////////////////////////////////
+	// Get 1..n cache record values (attachments) for 1..n records
+
+	struct GetCacheChunkRequest
+	{
+		CacheKey				   Key;
+		Oid						   ValueId	 = Oid::Zero;  // Set if ChunkId is not known at request time
+		IoHash					   ChunkId	 = IoHash::Zero;
+		uint64_t				   RawOffset = 0ull;
+		uint64_t				   RawSize	 = ~uint64_t(0);
+		std::optional<CachePolicy> Policy;
+	};
+
+	struct GetCacheChunksRequest
+	{
+		CachePolicy						  DefaultPolicy = CachePolicy::Default;
+		std::string						  Namespace;
+		std::vector<GetCacheChunkRequest> Requests;
+
+		bool Parse(const CbObjectView& BatchObject);
+		bool Format(CbPackage& OutPackage) const;
+	};
+
+	typedef CacheValuesResult GetCacheChunksResult;
+
+	//////////////////////////////////////////////////////////////////////////
+
+	struct HttpRequestData
+	{
+		std::optional<std::string> Namespace;
+		std::optional<std::string> Bucket;
+		std::optional<IoHash>	   HashKey;
+		std::optional<IoHash>	   ValueContentId;
+	};
+
+	bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data);
+
+	// Temporarily public
+	std::optional<std::string> GetRequestNamespace(const CbObjectView& Params);
+	bool					   GetRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key);
+
+	//////////////////////////////////////////////////////////////////////////
+
+	//	struct CacheRecordValue
+	//	{
+	//		Oid		 Id		 = Oid::Zero;
+	//		IoHash	 RawHash = IoHash::Zero;
+	//		uint64_t RawSize = 0;
+	//	};
+	//
+	//	struct CacheRecord
+	//	{
+	//		CacheKey					  Key = CacheKey::Empty;
+	//		std::vector<CacheRecordValue> Values;
+	//
+	//		bool Parse(CbObjectView& Reader);
+	//		bool Format(CbObjectWriter& Writer) const;
+	//	};
+
+}  // namespace cacherequests
+
+void cacherequests_forcelink();	 // internal
+
+}  // namespace zen