Merged main.

1 files changed, 194 insertions, 105 deletions

diff --git a/zenutil/cache/cachepolicy.cpp b/zenutil/cache/cachepolicy.cpp
index e1c31d885..3bf7a0c67 100644
--- a/zenutil/cache/cachepolicy.cpp
+++ b/zenutil/cache/cachepolicy.cpp
@@ -4,64 +4,115 @@
 
 #include <zencore/compactbinary.h>
 #include <zencore/compactbinarybuilder.h>
+#include <zencore/enumflags.h>
 #include <zencore/string.h>
 
+#include <algorithm>
+#include <unordered_map>
+
 namespace zen {
 
 using namespace std::literals;
 
-namespace detail { namespace cacheopt {
-	constexpr std::string_view Local	   = "local"sv;
-	constexpr std::string_view Remote	   = "remote"sv;
-	constexpr std::string_view Data		   = "data"sv;
-	constexpr std::string_view Meta		   = "meta"sv;
-	constexpr std::string_view Value	   = "value"sv;
-	constexpr std::string_view Attachments = "attachments"sv;
-}}	// namespace detail::cacheopt
+namespace detail::CachePolicyImpl {
+	constexpr char			   DelimiterChar = ',';
+	constexpr std::string_view None			 = "None"sv;
+	constexpr std::string_view QueryLocal	 = "QueryLocal"sv;
+	constexpr std::string_view QueryRemote	 = "QueryRemote"sv;
+	constexpr std::string_view Query		 = "Query"sv;
+	constexpr std::string_view StoreLocal	 = "StoreLocal"sv;
+	constexpr std::string_view StoreRemote	 = "StoreRemote"sv;
+	constexpr std::string_view Store		 = "Store"sv;
+	constexpr std::string_view SkipMeta		 = "SkipMeta"sv;
+	constexpr std::string_view SkipData		 = "SkipData"sv;
+	constexpr std::string_view PartialRecord = "PartialRecord"sv;
+	constexpr std::string_view KeepAlive	 = "KeepAlive"sv;
+	constexpr std::string_view Local		 = "Local"sv;
+	constexpr std::string_view Remote		 = "Remote"sv;
+	constexpr std::string_view Default		 = "Default"sv;
+	constexpr std::string_view Disable		 = "Disable"sv;
 
-CachePolicy
-ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default)
-{
-	if (QueryPolicy.empty())
-	{
-		return Default;
-	}
+	using TextToPolicyMap			   = std::unordered_map<std::string_view, CachePolicy>;
+	const TextToPolicyMap TextToPolicy = {{None, CachePolicy::None},
+										  {QueryLocal, CachePolicy::QueryLocal},
+										  {QueryRemote, CachePolicy::QueryRemote},
+										  {Query, CachePolicy::Query},
+										  {StoreLocal, CachePolicy::StoreLocal},
+										  {StoreRemote, CachePolicy::StoreRemote},
+										  {Store, CachePolicy::Store},
+										  {SkipMeta, CachePolicy::SkipMeta},
+										  {SkipData, CachePolicy::SkipData},
+										  {PartialRecord, CachePolicy::PartialRecord},
+										  {KeepAlive, CachePolicy::KeepAlive},
+										  {Local, CachePolicy::Local},
+										  {Remote, CachePolicy::Remote},
+										  {Default, CachePolicy::Default},
+										  {Disable, CachePolicy::Disable}};
 
-	CachePolicy Result = CachePolicy::None;
+	using PolicyTextPair = std::pair<CachePolicy, std::string_view>;
+	const PolicyTextPair FlagsToString[]{
+		// Order of these Flags is important: we want the aliases before the atomic values,
+		// and the bigger aliases first, to reduce the number of tokens we add
+		{CachePolicy::Default, Default},
+		{CachePolicy::Remote, Remote},
+		{CachePolicy::Local, Local},
+		{CachePolicy::Store, Store},
+		{CachePolicy::Query, Query},
 
-	ForEachStrTok(QueryPolicy, ',', [&Result](const std::string_view& Token) {
-		if (Token == detail::cacheopt::Local)
-		{
-			Result |= CachePolicy::QueryLocal;
-		}
-		if (Token == detail::cacheopt::Remote)
+		// Order of Atomics doesn't matter, so arbitrarily we list them in enum order
+		{CachePolicy::QueryLocal, QueryLocal},
+		{CachePolicy::QueryRemote, QueryRemote},
+		{CachePolicy::StoreLocal, StoreLocal},
+		{CachePolicy::StoreRemote, StoreRemote},
+		{CachePolicy::SkipMeta, SkipMeta},
+		{CachePolicy::SkipData, SkipData},
+		{CachePolicy::PartialRecord, PartialRecord},
+		{CachePolicy::KeepAlive, KeepAlive},
+
+		// None must come at the end of the array, to write out only if no others exist
+		{CachePolicy::None, None},
+	};
+	constexpr CachePolicy KnownFlags =
+		CachePolicy::Default | CachePolicy::SkipMeta | CachePolicy::SkipData | CachePolicy::KeepAlive | CachePolicy::PartialRecord;
+}  // namespace detail::CachePolicyImpl
+
+StringBuilderBase&
+AppendToBuilderImpl(StringBuilderBase& Builder, CachePolicy Policy)
+{
+	// Remove any bits we don't recognize; write None if there are not any bits we recognize
+	Policy = Policy & detail::CachePolicyImpl::KnownFlags;
+	for (const detail::CachePolicyImpl::PolicyTextPair& Pair : detail::CachePolicyImpl::FlagsToString)
+	{
+		if (EnumHasAllFlags(Policy, Pair.first))
 		{
-			Result |= CachePolicy::QueryRemote;
+			EnumRemoveFlags(Policy, Pair.first);
+			Builder << Pair.second << detail::CachePolicyImpl::DelimiterChar;
+			if (Policy == CachePolicy::None)
+			{
+				break;
+			}
 		}
-		return true;
-	});
-
-	return Result;
+	}
+	Builder.RemoveSuffix(1);  // Text will have been added by CachePolicy::None if not by anything else
+	return Builder;
+}
+StringBuilderBase&
+operator<<(StringBuilderBase& Builder, CachePolicy Policy)
+{
+	return AppendToBuilderImpl(Builder, Policy);
 }
 
 CachePolicy
-ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default)
+ParseCachePolicy(std::string_view Text)
 {
-	if (StorePolicy.empty())
-	{
-		return Default;
-	}
+	ZEN_ASSERT(!Text.empty());	// Empty string is not valid input to ParseCachePolicy
 
 	CachePolicy Result = CachePolicy::None;
-
-	ForEachStrTok(StorePolicy, ',', [&Result](const std::string_view& Token) {
-		if (Token == detail::cacheopt::Local)
-		{
-			Result |= CachePolicy::StoreLocal;
-		}
-		if (Token == detail::cacheopt::Remote)
+	ForEachStrTok(Text, detail::CachePolicyImpl::DelimiterChar, [&Result](const std::string_view& Token) {
+		auto it = detail::CachePolicyImpl::TextToPolicy.find(Token);
+		if (it != detail::CachePolicyImpl::TextToPolicy.end())
 		{
-			Result |= CachePolicy::StoreRemote;
+			Result |= it->second;
 		}
 		return true;
 	});
@@ -69,101 +120,139 @@ ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default)
 	return Result;
 }
 
-CachePolicy
-ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default)
-{
-	if (SkipPolicy.empty())
+namespace Private {
+
+	class CacheRecordPolicyShared final : public ICacheRecordPolicyShared
 	{
-		return Default;
-	}
+	public:
+		inline std::span<const CacheValuePolicy> GetValuePolicies() const final { return Values; }
 
-	CachePolicy Result = CachePolicy::None;
+		inline void AddValuePolicy(const CacheValuePolicy& Policy) final { Values.push_back(Policy); }
 
-	ForEachStrTok(SkipPolicy, ',', [&Result](const std::string_view& Token) {
-		if (Token == detail::cacheopt::Meta)
-		{
-			Result |= CachePolicy::SkipMeta;
-		}
-		if (Token == detail::cacheopt::Value)
+		inline void Build() final
 		{
-			Result |= CachePolicy::SkipValue;
+			std::sort(Values.begin(), Values.end(), [](const CacheValuePolicy& A, const CacheValuePolicy& B) { return A.Id < B.Id; });
 		}
-		if (Token == detail::cacheopt::Attachments)
-		{
-			Result |= CachePolicy::SkipAttachments;
-		}
-		if (Token == detail::cacheopt::Data)
-		{
-			Result |= CachePolicy::SkipData;
-		}
-		return true;
-	});
 
-	return Result;
-}
+	private:
+		std::vector<CacheValuePolicy> Values;
+	};
 
-CacheRecordPolicy::CacheRecordPolicy(const CachePolicy RecordPolicy, const CachePolicy PayloadPolicy)
-: m_RecordPolicy(RecordPolicy)
-, m_DefaultPayloadPolicy(PayloadPolicy)
-{
-}
+}  // namespace Private
 
 CachePolicy
-CacheRecordPolicy::GetPayloadPolicy(const Oid& PayloadId) const
+CacheRecordPolicy::GetValuePolicy(const Oid& Id) const
 {
-	if (const auto It = m_PayloadPolicies.find(PayloadId); It != m_PayloadPolicies.end())
+	if (Shared)
 	{
-		return It->second;
+		if (std::span<const CacheValuePolicy> Values = Shared->GetValuePolicies(); !Values.empty())
+		{
+			auto Iter =
+				std::lower_bound(Values.begin(), Values.end(), Id, [](const CacheValuePolicy& A, const Oid& B) { return A.Id < B; });
+			if (Iter != Values.end() && Iter->Id == Id)
+			{
+				return Iter->Policy;
+			}
+		}
 	}
+	return DefaultValuePolicy;
+}
 
-	return m_DefaultPayloadPolicy;
+void
+CacheRecordPolicy::Save(CbWriter& Writer) const
+{
+	Writer.BeginObject();
+	{
+		// The RecordPolicy is calculated from the ValuePolicies and does not need to be saved separately.
+		Writer << "DefaultValuePolicy"sv << WriteToString<128>(GetDefaultValuePolicy());
+		if (!IsUniform())
+		{
+			// FCacheRecordPolicyBuilder guarantees IsUniform -> non-empty GetValuePolicies. Small size penalty here if not.
+			Writer.BeginArray("ValuePolicies"sv);
+			{
+				for (const CacheValuePolicy& ValuePolicy : GetValuePolicies())
+				{
+					// FCacheRecordPolicyBuilder is responsible for ensuring that each ValuePolicy != DefaultValuePolicy
+					// If it lets any duplicates through we will incur a small serialization size penalty here
+					Writer.BeginObject();
+					Writer << "Id"sv << ValuePolicy.Id;
+					Writer << "Policy"sv << WriteToString<128>(ValuePolicy.Policy);
+					Writer.EndObject();
+				}
+			}
+			Writer.EndArray();
+		}
+	}
+	Writer.EndObject();
 }
 
-bool
-CacheRecordPolicy::Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy)
+CacheRecordPolicy
+CacheRecordPolicy::Load(CbObjectView Object, CachePolicy DefaultPolicy)
 {
-	using namespace std::literals;
+	std::string_view PolicyText			= Object["DefaultValuePolicy"sv].AsString();
+	CachePolicy		 DefaultValuePolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy;
 
-	const uint32_t RecordPolicy = RecordPolicyObject["RecordPolicy"sv].AsUInt32(static_cast<uint32_t>(CachePolicy::Default));
-	const uint32_t DefaultPayloadPolicy =
-		RecordPolicyObject["DefaultPayloadPolicy"sv].AsUInt32(static_cast<uint32_t>(CachePolicy::Default));
+	CacheRecordPolicyBuilder Builder(DefaultValuePolicy);
+	for (CbFieldView ValueObjectField : Object["ValuePolicies"sv])
+	{
+		CbObjectView ValueObject = ValueObjectField.AsObjectView();
+		const Oid	 ValueId	 = ValueObject["Id"sv].AsObjectId();
+		PolicyText				 = ValueObject["Policy"sv].AsString();
+		CachePolicy ValuePolicy	 = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultValuePolicy;
+		// FCacheRecordPolicyBuilder should guarantee that FValueId(ValueId).IsValid and ValuePolicy != DefaultValuePolicy
+		// If it lets any through we will have unused data in the record we create.
+		Builder.AddValuePolicy(ValueId, ValuePolicy);
+	}
 
-	OutRecordPolicy.m_RecordPolicy		   = static_cast<CachePolicy>(RecordPolicy);
-	OutRecordPolicy.m_DefaultPayloadPolicy = static_cast<CachePolicy>(DefaultPayloadPolicy);
+	return Builder.Build();
+}
 
-	for (CbFieldView PayloadPolicyView : RecordPolicyObject["PayloadPolicies"sv])
+CacheRecordPolicy
+CacheRecordPolicy::ConvertToUpstream() const
+{
+	auto DownstreamToUpstream = [](CachePolicy P) {
+		// Remote|Local -> Set Remote
+		// Delete Skip Flags
+		// Maintain Remaining Flags
+		return (EnumHasAllFlags(P, CachePolicy::QueryRemote) ? CachePolicy::QueryLocal : CachePolicy::None) |
+			   (EnumHasAllFlags(P, CachePolicy::StoreRemote) ? CachePolicy::StoreLocal : CachePolicy::None) |
+			   (P & ~(CachePolicy::SkipData | CachePolicy::SkipMeta));
+	};
+	CacheRecordPolicyBuilder Builder(DownstreamToUpstream(GetDefaultValuePolicy()));
+	for (const CacheValuePolicy& ValuePolicy : GetValuePolicies())
 	{
-		CbObjectView   PayloadPolicyObject = PayloadPolicyView.AsObjectView();
-		const Oid	   PayloadId		   = PayloadPolicyObject["Id"sv].AsObjectId();
-		const uint32_t PayloadPolicy	   = PayloadPolicyObject["Policy"sv].AsUInt32();
-
-		if (PayloadId != Oid::Zero && PayloadPolicy != 0)
-		{
-			OutRecordPolicy.m_PayloadPolicies.emplace(PayloadId, static_cast<CachePolicy>(PayloadPolicy));
-		}
+		Builder.AddValuePolicy(ValuePolicy.Id, DownstreamToUpstream(ValuePolicy.Policy));
 	}
-
-	return true;
+	return Builder.Build();
 }
 
 void
-CacheRecordPolicy::Save(const CacheRecordPolicy& Policy, CbWriter& Writer)
+CacheRecordPolicyBuilder::AddValuePolicy(const CacheValuePolicy& Policy)
 {
-	Writer << "RecordPolicy"sv << static_cast<uint32_t>(Policy.GetRecordPolicy());
-	Writer << "DefaultPayloadPolicy"sv << static_cast<uint32_t>(Policy.GetDefaultPayloadPolicy());
+	if (!Shared)
+	{
+		Shared = new Private::CacheRecordPolicyShared;
+	}
+	Shared->AddValuePolicy(Policy);
+}
 
-	if (!Policy.m_PayloadPolicies.empty())
+CacheRecordPolicy
+CacheRecordPolicyBuilder::Build()
+{
+	CacheRecordPolicy Policy(BasePolicy);
+	if (Shared)
 	{
-		Writer.BeginArray("PayloadPolicies"sv);
-		for (const auto& Kv : Policy.m_PayloadPolicies)
+		Shared->Build();
+		const auto								PolicyOr = [](CachePolicy A, CachePolicy B) { return A | (B & ~CachePolicy::SkipData); };
+		const std::span<const CacheValuePolicy> Values	 = Shared->GetValuePolicies();
+		Policy.RecordPolicy								 = BasePolicy;
+		for (const CacheValuePolicy& ValuePolicy : Values)
 		{
-			Writer.BeginObject();
-			Writer.AddObjectId("Id"sv, Kv.first);
-			Writer << "Policy"sv << static_cast<uint32_t>(Kv.second);
-			Writer.EndObject();
+			Policy.RecordPolicy = PolicyOr(Policy.RecordPolicy, ValuePolicy.Policy);
 		}
-		Writer.EndArray();
+		Policy.Shared = std::move(Shared);
 	}
+	return Policy;
 }
 
 }  // namespace zen