cacherequests helpers test only (#551)

* don't use cacherequests utils in cache_cmd.cpp
* make zenutil/cacherequests code into test code helpers only

6 files changed, 376 insertions, 3 deletions

diff --git a/src/zenstore/include/zenstore/cache/cache.h b/src/zenstore/include/zenstore/cache/cache.h
new file mode 100644
index 000000000..4e72d1b05
--- /dev/null
+++ b/src/zenstore/include/zenstore/cache/cache.h
@@ -0,0 +1,52 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenstore/cache/cachekey.h>
+#include <zenstore/cache/cachepolicy.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+struct CacheRequestContext
+{
+	Oid		 SessionId{Oid::Zero};
+	uint32_t RequestId{0};
+};
+
+std::optional<std::string> GetValidNamespaceName(std::string_view Name);
+std::optional<std::string> GetValidBucketName(std::string_view Name);
+std::optional<IoHash>	   GetValidIoHash(std::string_view Hash);
+
+struct HttpCacheRequestData
+{
+	std::optional<std::string> Namespace;
+	std::optional<std::string> Bucket;
+	std::optional<IoHash>	   HashKey;
+	std::optional<IoHash>	   ValueContentId;
+};
+
+bool HttpCacheRequestParseRelativeUri(std::string_view Key, std::string_view DefaultNamespace, HttpCacheRequestData& Data);
+
+// Temporarily public
+std::optional<std::string> GetCacheRequestNamespace(const CbObjectView& Params);
+bool					   GetCacheRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key);
+
+}  // namespace zen
+
+template<>
+struct fmt::formatter<zen::CacheRequestContext> : formatter<string_view>
+{
+	template<typename FormatContext>
+	auto format(const zen::CacheRequestContext& Context, FormatContext& ctx) const
+	{
+		zen::ExtendableStringBuilder<64> String;
+		Context.SessionId.ToString(String);
+		String << ".";
+		String << Context.RequestId;
+		return formatter<string_view>::format(String.ToView(), ctx);
+	}
+};
diff --git a/src/zenstore/include/zenstore/cache/cachekey.h b/src/zenstore/include/zenstore/cache/cachekey.h
new file mode 100644
index 000000000..ae79f2a3d
--- /dev/null
+++ b/src/zenstore/include/zenstore/cache/cachekey.h
@@ -0,0 +1,92 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+
+#include <zenstore/cache/cachepolicy.h>
+
+namespace zen {
+
+struct CacheKey
+{
+	std::string Bucket;
+	IoHash		Hash;
+
+	static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; }
+
+	// This should be used whenever the bucket name has already been validated to avoid redundant ToLower calls
+	static CacheKey CreateValidated(std::string&& BucketValidated, const IoHash& Hash)
+	{
+		return {.Bucket = std::move(BucketValidated), .Hash = Hash};
+	}
+
+	auto operator<=>(const CacheKey& that) const
+	{
+		if (auto b = caseSensitiveCompareStrings(Bucket, that.Bucket); b != std::strong_ordering::equal)
+		{
+			return b;
+		}
+		return Hash <=> that.Hash;
+	}
+
+	auto operator==(const CacheKey& that) const { return (*this <=> that) == std::strong_ordering::equal; }
+
+	static const CacheKey Empty;
+};
+
+struct CacheChunkRequest
+{
+	CacheKey	Key;
+	IoHash		ChunkId;
+	Oid			ValueId;
+	uint64_t	RawOffset = 0ull;
+	uint64_t	RawSize	  = ~uint64_t(0);
+	CachePolicy Policy	  = CachePolicy::Default;
+};
+
+struct CacheKeyRequest
+{
+	CacheKey		  Key;
+	CacheRecordPolicy Policy;
+};
+
+struct CacheValueRequest
+{
+	CacheKey	Key;
+	CachePolicy Policy = CachePolicy::Default;
+};
+
+inline bool
+operator<(const CacheChunkRequest& A, const CacheChunkRequest& B)
+{
+	if (A.Key < B.Key)
+	{
+		return true;
+	}
+	if (B.Key < A.Key)
+	{
+		return false;
+	}
+	if (A.ChunkId < B.ChunkId)
+	{
+		return true;
+	}
+	if (B.ChunkId < A.ChunkId)
+	{
+		return false;
+	}
+	if (A.ValueId < B.ValueId)
+	{
+		return true;
+	}
+	if (B.ValueId < A.ValueId)
+	{
+		return false;
+	}
+	return A.RawOffset < B.RawOffset;
+}
+
+}  // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/cachepolicy.h b/src/zenstore/include/zenstore/cache/cachepolicy.h
new file mode 100644
index 000000000..7773cd3d1
--- /dev/null
+++ b/src/zenstore/include/zenstore/cache/cachepolicy.h
@@ -0,0 +1,229 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/refcount.h>
+#include <zencore/compactbinary.h>
+#include <zencore/enumflags.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+
+#include <gsl/gsl-lite.hpp>
+#include <span>
+namespace zen::Private {
+class ICacheRecordPolicyShared;
+}
+namespace zen {
+
+class CbObjectView;
+class CbWriter;
+
+class OptionalCacheRecordPolicy;
+
+enum class CachePolicy : uint32_t
+{
+	/** A value with no flags. Disables access to the cache unless combined with other flags. */
+	None = 0,
+
+	/** Allow a cache request to query local caches. */
+	QueryLocal = 1 << 0,
+	/** Allow a cache request to query remote caches. */
+	QueryRemote = 1 << 1,
+	/** Allow a cache request to query any caches. */
+	Query = QueryLocal | QueryRemote,
+
+	/** Allow cache requests to query and store records and values in local caches. */
+	StoreLocal = 1 << 2,
+	/** Allow cache records and values to be stored in remote caches. */
+	StoreRemote = 1 << 3,
+	/** Allow cache records and values to be stored in any caches. */
+	Store = StoreLocal | StoreRemote,
+
+	/** Allow cache requests to query and store records and values in local caches. */
+	Local = QueryLocal | StoreLocal,
+	/** Allow cache requests to query and store records and values in remote caches. */
+	Remote = QueryRemote | StoreRemote,
+
+	/** Allow cache requests to query and store records and values in any caches. */
+	Default = Query | Store,
+
+	/** Skip fetching the data for values. */
+	SkipData = 1 << 4,
+
+	/** Skip fetching the metadata for record requests. */
+	SkipMeta = 1 << 5,
+
+	/**
+	 * Partial output will be provided with the error status when a required value is missing.
+	 *
+	 * This is meant for cases when the missing values can be individually recovered, or rebuilt,
+	 * without rebuilding the whole record. The cache automatically adds this flag when there are
+	 * other cache stores that it may be able to recover missing values from.
+	 *
+	 * Missing values will be returned in the records, but with only the hash and size.
+	 *
+	 * Applying this flag for a put of a record allows a partial record to be stored.
+	 */
+	PartialRecord = 1 << 6,
+
+	/**
+	 * Keep records in the cache for at least the duration of the session.
+	 *
+	 * This is a hint that the record may be accessed again in this session. This is mainly meant
+	 * to be used when subsequent accesses will not tolerate a cache miss.
+	 */
+	KeepAlive = 1 << 7,
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy);
+/** Append a non-empty text version of the policy to the builder. */
+StringBuilderBase& operator<<(StringBuilderBase& Builder, CachePolicy Policy);
+/** Parse non-empty text written by operator<< into a policy. */
+CachePolicy ParseCachePolicy(std::string_view Text);
+/** Return input converted into the equivalent policy that the upstream should use when forwarding a put or get to an upstream server. */
+CachePolicy ConvertToUpstream(CachePolicy Policy);
+
+inline CachePolicy
+Union(CachePolicy A, CachePolicy B)
+{
+	constexpr CachePolicy InvertedFlags = CachePolicy::SkipData | CachePolicy::SkipMeta;
+	return (A & ~(InvertedFlags)) | (B & ~(InvertedFlags)) | (A & B & InvertedFlags);
+}
+
+/** A value ID and the cache policy to use for that value. */
+struct CacheValuePolicy
+{
+	Oid			Id;
+	CachePolicy Policy = CachePolicy::Default;
+
+	/** Flags that are valid on a value policy. */
+	static constexpr CachePolicy PolicyMask = CachePolicy::Default | CachePolicy::SkipData;
+};
+
+/** Interface for the private implementation of the cache record policy. */
+class Private::ICacheRecordPolicyShared : public RefCounted
+{
+public:
+	virtual ~ICacheRecordPolicyShared()														 = default;
+	virtual void							  AddValuePolicy(const CacheValuePolicy& Policy) = 0;
+	virtual std::span<const CacheValuePolicy> GetValuePolicies() const						 = 0;
+};
+
+/**
+ * Flags to control the behavior of cache record requests, with optional overrides by value.
+ *
+ * Examples:
+ * - A base policy of None with value policy overrides of Default will fetch those values if they
+ *   exist in the record, and skip data for any other values.
+ * - A base policy of Default, with value policy overrides of (Query | SkipData), will skip those
+ *   values, but still check if they exist, and will load any other values.
+ */
+class CacheRecordPolicy
+{
+public:
+	/** Construct a cache record policy that uses the default policy. */
+	CacheRecordPolicy() = default;
+
+	/** Construct a cache record policy with a uniform policy for the record and every value. */
+	inline CacheRecordPolicy(CachePolicy BasePolicy)
+	: RecordPolicy(BasePolicy)
+	, DefaultValuePolicy(BasePolicy & CacheValuePolicy::PolicyMask)
+	{
+	}
+
+	/** Returns true if the record and every value use the same cache policy. */
+	inline bool IsUniform() const { return !Shared; }
+
+	/** Returns the cache policy to use for the record. */
+	inline CachePolicy GetRecordPolicy() const { return RecordPolicy; }
+
+	/** Returns the base cache policy that this was constructed from. */
+	inline CachePolicy GetBasePolicy() const { return DefaultValuePolicy | (RecordPolicy & ~CacheValuePolicy::PolicyMask); }
+
+	/** Returns the cache policy to use for the value. */
+	CachePolicy GetValuePolicy(const Oid& Id) const;
+
+	/** Returns the array of cache policy overrides for values, sorted by ID. */
+	inline std::span<const CacheValuePolicy> GetValuePolicies() const
+	{
+		return Shared ? Shared->GetValuePolicies() : std::span<const CacheValuePolicy>();
+	}
+
+	/** Saves the cache record policy to a compact binary object. */
+	void Save(CbWriter& Writer) const;
+
+	/** Loads a cache record policy from an object. */
+	static OptionalCacheRecordPolicy Load(CbObjectView Object);
+
+	/** Return *this converted into the equivalent policy that the upstream should use when forwarding a put or get to an upstream server.
+	 */
+	CacheRecordPolicy ConvertToUpstream() const;
+
+private:
+	friend class CacheRecordPolicyBuilder;
+	friend class OptionalCacheRecordPolicy;
+
+	CachePolicy										RecordPolicy	   = CachePolicy::Default;
+	CachePolicy										DefaultValuePolicy = CachePolicy::Default;
+	RefPtr<const Private::ICacheRecordPolicyShared> Shared;
+};
+
+/** A cache record policy builder is used to construct a cache record policy. */
+class CacheRecordPolicyBuilder
+{
+public:
+	/** Construct a policy builder that uses the default policy as its base policy. */
+	CacheRecordPolicyBuilder() = default;
+
+	/** Construct a policy builder that uses the provided policy for the record and values with no override. */
+	inline explicit CacheRecordPolicyBuilder(CachePolicy Policy) : BasePolicy(Policy) {}
+
+	/** Adds a cache policy override for a value. */
+	void		AddValuePolicy(const CacheValuePolicy& Value);
+	inline void AddValuePolicy(const Oid& Id, CachePolicy Policy) { AddValuePolicy({Id, Policy}); }
+
+	/** Build a cache record policy, which makes this builder subsequently unusable. */
+	CacheRecordPolicy Build();
+
+private:
+	CachePolicy								  BasePolicy = CachePolicy::Default;
+	RefPtr<Private::ICacheRecordPolicyShared> Shared;
+};
+
+/**
+ * A cache record policy that can be null.
+ *
+ * @see CacheRecordPolicy
+ */
+class OptionalCacheRecordPolicy : private CacheRecordPolicy
+{
+public:
+	inline OptionalCacheRecordPolicy() : CacheRecordPolicy(~CachePolicy::None) {}
+
+	inline OptionalCacheRecordPolicy(CacheRecordPolicy&& InOutput) : CacheRecordPolicy(std::move(InOutput)) {}
+	inline OptionalCacheRecordPolicy(const CacheRecordPolicy& InOutput) : CacheRecordPolicy(InOutput) {}
+	inline OptionalCacheRecordPolicy& operator=(CacheRecordPolicy&& InOutput)
+	{
+		CacheRecordPolicy::operator=(std::move(InOutput));
+		return *this;
+	}
+	inline OptionalCacheRecordPolicy& operator=(const CacheRecordPolicy& InOutput)
+	{
+		CacheRecordPolicy::operator=(InOutput);
+		return *this;
+	}
+
+	/** Returns the cache record policy. The caller must check for null before using this accessor. */
+	inline const CacheRecordPolicy& Get() const& { return *this; }
+	inline CacheRecordPolicy		Get() && { return std::move(*this); }
+
+	inline bool		IsNull() const { return RecordPolicy == ~CachePolicy::None; }
+	inline bool		IsValid() const { return !IsNull(); }
+	inline explicit operator bool() const { return !IsNull(); }
+
+	inline void Reset() { *this = OptionalCacheRecordPolicy(); }
+};
+
+void cachepolicy_forcelink();
+
+}  // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/cacherpc.h b/src/zenstore/include/zenstore/cache/cacherpc.h
index e0c8738ca..eb40befa0 100644
--- a/src/zenstore/include/zenstore/cache/cacherpc.h
+++ b/src/zenstore/include/zenstore/cache/cacherpc.h
@@ -4,9 +4,9 @@
 
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
+#include <zenstore/cache/cache.h>
 #include <zenstore/cache/cacheshared.h>
 #include <zenstore/cache/structuredcachestore.h>
-#include <zenutil/cache/cache.h>
 
 #include <atomic>
 #include <string_view>
diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h
index c51d7312c..1ba469431 100644
--- a/src/zenstore/include/zenstore/cache/structuredcachestore.h
+++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h
@@ -5,9 +5,9 @@
 #include <zencore/compactbinary.h>
 #include <zencore/iohash.h>
 #include <zencore/stats.h>
+#include <zenstore/cache/cache.h>
 #include <zenstore/cache/cachedisklayer.h>
 #include <zenstore/gc.h>
-#include <zenutil/cache/cache.h>
 #include <zenutil/statsreporter.h>
 
 #include <atomic>
diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
index 152031c3a..2f3b6b0d7 100644
--- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
+++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
@@ -8,7 +8,7 @@
 #include <zencore/iohash.h>
 #include <zencore/stats.h>
 #include <zencore/zencore.h>
-#include <zenutil/cache/cache.h>
+#include <zenstore/cache/cache.h>
 
 #include <functional>
 #include <memory>