aboutsummaryrefslogtreecommitdiff
path: root/zenutil
diff options
context:
space:
mode:
Diffstat (limited to 'zenutil')
-rw-r--r--zenutil/cache/cachekey.cpp9
-rw-r--r--zenutil/cache/cachepolicy.cpp167
-rw-r--r--zenutil/include/zenutil/cache/cache.h6
-rw-r--r--zenutil/include/zenutil/cache/cachekey.h83
-rw-r--r--zenutil/include/zenutil/cache/cachepolicy.h109
-rw-r--r--zenutil/zenutil.vcxproj5
-rw-r--r--zenutil/zenutil.vcxproj.filters20
7 files changed, 399 insertions, 0 deletions
diff --git a/zenutil/cache/cachekey.cpp b/zenutil/cache/cachekey.cpp
new file mode 100644
index 000000000..545b47f11
--- /dev/null
+++ b/zenutil/cache/cachekey.cpp
@@ -0,0 +1,9 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cache/cachekey.h>
+
+namespace zen {
+
+const CacheKey CacheKey::Empty = CacheKey{.Bucket = std::string(), .Hash = IoHash()};
+
+} // namespace zen
diff --git a/zenutil/cache/cachepolicy.cpp b/zenutil/cache/cachepolicy.cpp
new file mode 100644
index 000000000..f718bf841
--- /dev/null
+++ b/zenutil/cache/cachepolicy.cpp
@@ -0,0 +1,167 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cache/cachepolicy.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/string.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+namespace detail { namespace cacheopt {
+ constexpr std::string_view Local = "local"sv;
+ constexpr std::string_view Remote = "remote"sv;
+ constexpr std::string_view Data = "data"sv;
+ constexpr std::string_view Meta = "meta"sv;
+ constexpr std::string_view Value = "value"sv;
+ constexpr std::string_view Attachments = "attachments"sv;
+}} // namespace detail::cacheopt
+
+CachePolicy
+ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default)
+{
+ if (QueryPolicy.empty())
+ {
+ return Default;
+ }
+
+ CachePolicy Result = CachePolicy::None;
+
+ ForEachStrTok(QueryPolicy, ',', [&Result](const std::string_view& Token) {
+ if (Token == detail::cacheopt::Local)
+ {
+ Result |= CachePolicy::QueryLocal;
+ }
+ if (Token == detail::cacheopt::Remote)
+ {
+ Result |= CachePolicy::QueryRemote;
+ }
+ return true;
+ });
+
+ return Result;
+}
+
+CachePolicy
+ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default)
+{
+ if (StorePolicy.empty())
+ {
+ return Default;
+ }
+
+ CachePolicy Result = CachePolicy::None;
+
+ ForEachStrTok(StorePolicy, ',', [&Result](const std::string_view& Token) {
+ if (Token == detail::cacheopt::Local)
+ {
+ Result |= CachePolicy::StoreLocal;
+ }
+ if (Token == detail::cacheopt::Remote)
+ {
+ Result |= CachePolicy::StoreRemote;
+ }
+ return true;
+ });
+
+ return Result;
+}
+
+CachePolicy
+ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default)
+{
+ if (SkipPolicy.empty())
+ {
+ return Default;
+ }
+
+ CachePolicy Result = CachePolicy::None;
+
+ ForEachStrTok(SkipPolicy, ',', [&Result](const std::string_view& Token) {
+ if (Token == detail::cacheopt::Meta)
+ {
+ Result |= CachePolicy::SkipMeta;
+ }
+ if (Token == detail::cacheopt::Value)
+ {
+ Result |= CachePolicy::SkipValue;
+ }
+ if (Token == detail::cacheopt::Attachments)
+ {
+ Result |= CachePolicy::SkipAttachments;
+ }
+ if (Token == detail::cacheopt::Data)
+ {
+ Result |= CachePolicy::SkipData;
+ }
+ return true;
+ });
+
+ return Result;
+}
+
+CacheRecordPolicy::CacheRecordPolicy(const CachePolicy RecordPolicy, const CachePolicy PayloadPolicy)
+: m_RecordPolicy(RecordPolicy)
+, m_DefaultPayloadPolicy(PayloadPolicy)
+{
+}
+
+CachePolicy
+CacheRecordPolicy::GetPayloadPolicy(const Oid& PayloadId) const
+{
+ if (const auto It = m_PayloadPolicies.find(PayloadId); It != m_PayloadPolicies.end())
+ {
+ return It->second;
+ }
+
+ return m_DefaultPayloadPolicy;
+}
+
+bool
+CacheRecordPolicy::Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy)
+{
+ using namespace std::literals;
+
+ const uint32_t RecordPolicy = RecordPolicyObject["RecordPolicy"sv].AsUInt32(static_cast<uint32_t>(CachePolicy::Default));
+ const uint32_t DefaultPayloadPolicy =
+ RecordPolicyObject["DefaultPayloadPolicy"sv].AsUInt32(static_cast<uint32_t>(CachePolicy::Default));
+
+ OutRecordPolicy.m_RecordPolicy = static_cast<CachePolicy>(RecordPolicy);
+ OutRecordPolicy.m_DefaultPayloadPolicy = static_cast<CachePolicy>(DefaultPayloadPolicy);
+
+ for (CbFieldView PayloadPolicyView : RecordPolicyObject["PayloadPolicies"sv])
+ {
+ CbObjectView PayloadPolicyObject = PayloadPolicyView.AsObjectView();
+ const Oid PayloadId = PayloadPolicyObject["Id"sv].AsObjectId();
+ const uint32_t PayloadPolicy = PayloadPolicyObject["Policy"sv].AsUInt32();
+
+ if (PayloadId != Oid::Zero && PayloadPolicy != 0)
+ {
+ OutRecordPolicy.m_PayloadPolicies.emplace(PayloadId, static_cast<CachePolicy>(PayloadPolicy));
+ }
+ }
+
+ return true;
+}
+
+void
+CacheRecordPolicy::Save(const CacheRecordPolicy& Policy, CbWriter& Writer)
+{
+ Writer << "RecordPolicy"sv << static_cast<uint32_t>(Policy.GetRecordPolicy());
+ Writer << "DefaultPayloadPolicy"sv << static_cast<uint32_t>(Policy.GetDefaultPayloadPolicy());
+
+ if (!Policy.m_PayloadPolicies.empty())
+ {
+ Writer.BeginArray("PayloadPolicies"sv);
+ for (const auto& Kv : Policy.m_PayloadPolicies)
+ {
+ Writer.AddObjectId("Id"sv, Kv.first);
+ Writer << "Policy"sv << static_cast<uint32_t>(Kv.second);
+ }
+ Writer.EndArray();
+ }
+}
+
+} // namespace zen
diff --git a/zenutil/include/zenutil/cache/cache.h b/zenutil/include/zenutil/cache/cache.h
new file mode 100644
index 000000000..1a1dd9386
--- /dev/null
+++ b/zenutil/include/zenutil/cache/cache.h
@@ -0,0 +1,6 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenutil/cache/cachekey.h>
+#include <zenutil/cache/cachepolicy.h>
diff --git a/zenutil/include/zenutil/cache/cachekey.h b/zenutil/include/zenutil/cache/cachekey.h
new file mode 100644
index 000000000..fb36c7759
--- /dev/null
+++ b/zenutil/include/zenutil/cache/cachekey.h
@@ -0,0 +1,83 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+
+#include <zenutil/cache/cachepolicy.h>
+
+namespace zen {
+
+struct CacheKey
+{
+ std::string Bucket;
+ IoHash Hash;
+
+ static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; }
+
+ static const CacheKey Empty;
+};
+
+inline bool
+operator==(const CacheKey& A, const CacheKey& B)
+{
+ return A.Bucket == B.Bucket && A.Hash == B.Hash;
+}
+
+inline bool
+operator!=(const CacheKey& A, const CacheKey& B)
+{
+ return A.Bucket != B.Bucket || A.Hash != B.Hash;
+}
+
+inline bool
+operator<(const CacheKey& A, const CacheKey& B)
+{
+ const std::string& BucketA = A.Bucket;
+ const std::string& BucketB = B.Bucket;
+ return BucketA == BucketB ? A.Hash < B.Hash : BucketA < BucketB;
+}
+
+struct CacheChunkRequest
+{
+ CacheKey Key;
+ IoHash ChunkId;
+ Oid PayloadId;
+ uint64_t RawOffset = 0ull;
+ uint64_t RawSize = ~uint64_t(0);
+ CachePolicy Policy = CachePolicy::Default;
+};
+
+inline bool
+operator<(const CacheChunkRequest& A, const CacheChunkRequest& B)
+{
+ if (A.Key < B.Key)
+ {
+ return true;
+ }
+ if (B.Key < A.Key)
+ {
+ return false;
+ }
+ if (A.ChunkId < B.ChunkId)
+ {
+ return true;
+ }
+ if (B.ChunkId < A.ChunkId)
+ {
+ return false;
+ }
+ if (A.PayloadId < B.PayloadId)
+ {
+ return true;
+ }
+ if (B.PayloadId < A.PayloadId)
+ {
+ return false;
+ }
+ return A.RawOffset < B.RawOffset;
+}
+
+} // namespace zen
diff --git a/zenutil/include/zenutil/cache/cachepolicy.h b/zenutil/include/zenutil/cache/cachepolicy.h
new file mode 100644
index 000000000..5cf19238e
--- /dev/null
+++ b/zenutil/include/zenutil/cache/cachepolicy.h
@@ -0,0 +1,109 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/string.h>
+#include <zencore/uid.h>
+
+#include <gsl/gsl-lite.hpp>
+#include <unordered_map>
+
+namespace zen {
+
+class CbObjectView;
+class CbWriter;
+
+enum class CachePolicy : uint32_t
+{
+ /** A value without any flags set. */
+ None = 0,
+
+ /** Allow a cache request to query local caches. */
+ QueryLocal = 1 << 0,
+ /** Allow a cache request to query remote caches. */
+ QueryRemote = 1 << 1,
+ /** Allow a cache request to query any caches. */
+ Query = QueryLocal | QueryRemote,
+
+ /** Allow cache records and values to be stored in local caches. */
+ StoreLocal = 1 << 2,
+ /** Allow cache records and values to be stored in remote caches. */
+ StoreRemote = 1 << 3,
+ /** Allow cache records and values to be stored in any caches. */
+ Store = StoreLocal | StoreRemote,
+
+ /** Skip fetching the metadata for record requests. */
+ SkipMeta = 1 << 4,
+ /** Skip fetching the value for record, chunk, or value requests. */
+ SkipValue = 1 << 5,
+ /** Skip fetching the attachments for record requests. */
+ SkipAttachments = 1 << 6,
+ /**
+ * Skip fetching the data for any requests.
+ *
+ * Put requests with skip flags may assume that record existence implies payload existence.
+ */
+ SkipData = SkipMeta | SkipValue | SkipAttachments,
+
+ /**
+ * Keep records in the cache for at least the duration of the session.
+ *
+ * This is a hint that the record may be accessed again in this session. This is mainly meant
+ * to be used when subsequent accesses will not tolerate a cache miss.
+ */
+ KeepAlive = 1 << 7,
+
+ /**
+ * Partial output will be provided with the error status when a required payload is missing.
+ *
+ * This is meant for cases when the missing payloads can be individually recovered or rebuilt
+ * without rebuilding the whole record. The cache automatically adds this flag when there are
+ * other cache stores that it may be able to recover missing payloads from.
+ *
+ * Requests for records would return records where the missing payloads have a hash and size,
+ * but no data. Requests for chunks or values would return the hash and size, but no data.
+ */
+ PartialOnError = 1 << 8,
+
+ /** Allow cache requests to query and store records and values in local caches. */
+ Local = QueryLocal | StoreLocal,
+ /** Allow cache requests to query and store records and values in remote caches. */
+ Remote = QueryRemote | StoreRemote,
+
+ /** Allow cache requests to query and store records and values in any caches. */
+ Default = Query | Store,
+
+ /** Do not allow cache requests to query or store records and values in any caches. */
+ Disable = None,
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy);
+
+CachePolicy ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default = CachePolicy::Query);
+
+CachePolicy ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default = CachePolicy::Store);
+
+CachePolicy ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default = CachePolicy::None);
+
+class CacheRecordPolicy
+{
+public:
+ CacheRecordPolicy() = default;
+ CacheRecordPolicy(const CachePolicy RecordPolicy, const CachePolicy DefaultPayloadPolicy = CachePolicy::Default);
+
+ CachePolicy GetRecordPolicy() const { return m_RecordPolicy; }
+ CachePolicy GetPayloadPolicy(const Oid& PayloadId) const;
+ CachePolicy GetDefaultPayloadPolicy() const { return m_DefaultPayloadPolicy; }
+
+ static bool Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy);
+ static void Save(const CacheRecordPolicy& Policy, CbWriter& Writer);
+
+private:
+ using PayloadPolicyMap = std::unordered_map<Oid, CachePolicy, Oid::Hasher>;
+
+ CachePolicy m_RecordPolicy = CachePolicy::Default;
+ CachePolicy m_DefaultPayloadPolicy = CachePolicy::Default;
+ PayloadPolicyMap m_PayloadPolicies;
+};
+
+} // namespace zen
diff --git a/zenutil/zenutil.vcxproj b/zenutil/zenutil.vcxproj
index 3bf6111f7..f5db7c5b0 100644
--- a/zenutil/zenutil.vcxproj
+++ b/zenutil/zenutil.vcxproj
@@ -97,9 +97,14 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
+ <ClCompile Include="cache\cachekey.cpp" />
+ <ClCompile Include="cache\cachepolicy.cpp" />
<ClCompile Include="zenserverprocess.cpp" />
</ItemGroup>
<ItemGroup>
+ <ClInclude Include="include\zenutil\cache\cache.h" />
+ <ClInclude Include="include\zenutil\cache\cachekey.h" />
+ <ClInclude Include="include\zenutil\cache\cachepolicy.h" />
<ClInclude Include="include\zenutil\zenserverprocess.h" />
</ItemGroup>
<ItemGroup>
diff --git a/zenutil/zenutil.vcxproj.filters b/zenutil/zenutil.vcxproj.filters
index 9952e7159..368a827c2 100644
--- a/zenutil/zenutil.vcxproj.filters
+++ b/zenutil/zenutil.vcxproj.filters
@@ -2,11 +2,31 @@
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="zenserverprocess.cpp" />
+ <ClCompile Include="cache\cachekey.cpp">
+ <Filter>cache</Filter>
+ </ClCompile>
+ <ClCompile Include="cache\cachepolicy.cpp">
+ <Filter>cache</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="include\zenutil\zenserverprocess.h" />
+ <ClInclude Include="include\zenutil\cache\cache.h">
+ <Filter>cache</Filter>
+ </ClInclude>
+ <ClInclude Include="include\zenutil\cache\cachekey.h">
+ <Filter>cache</Filter>
+ </ClInclude>
+ <ClInclude Include="include\zenutil\cache\cachepolicy.h">
+ <Filter>cache</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="xmake.lua" />
</ItemGroup>
+ <ItemGroup>
+ <Filter Include="cache">
+ <UniqueIdentifier>{a441c536-6a01-4ac4-85a0-2667c95027d0}</UniqueIdentifier>
+ </Filter>
+ </ItemGroup>
</Project> \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-22 04:21:48 +0000