aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2022-01-25 15:16:04 +0100
committerStefan Boberg <[email protected]>2022-01-25 15:16:04 +0100
commit080b73be664064d13eb88e53cd627ef859aa7da8 (patch)
tree1614a7637a33551e46dcb53ebb0e92d02a26b4f7
parentImplemented support for storing compressed buffers as values in structured ca... (diff)
parentCachepolicy (#36) (diff)
downloadzen-080b73be664064d13eb88e53cd627ef859aa7da8.tar.xz
zen-080b73be664064d13eb88e53cd627ef859aa7da8.zip
Merge branch 'main' of https://github.com/EpicGames/zen
-rw-r--r--scripts/generateprojects.bat1
-rw-r--r--zencore/include/zencore/refcount.h17
-rw-r--r--zencore/include/zencore/string.h22
-rw-r--r--zenserver-test/cachepolicy-tests.cpp164
-rw-r--r--zenserver-test/zenserver-test.cpp187
-rw-r--r--zenserver/auth/authservice.cpp2
-rw-r--r--zenserver/cache/structuredcache.cpp297
-rw-r--r--zenserver/cache/structuredcache.h14
-rw-r--r--zenserver/config.cpp22
-rw-r--r--zenserver/upstream/upstreamcache.cpp20
-rw-r--r--zenstore/gc.cpp5
-rw-r--r--zenutil/cache/cachepolicy.cpp284
-rw-r--r--zenutil/include/zenutil/cache/cachepolicy.h137
13 files changed, 667 insertions, 505 deletions
diff --git a/scripts/generateprojects.bat b/scripts/generateprojects.bat
new file mode 100644
index 000000000..cc6732aaa
--- /dev/null
+++ b/scripts/generateprojects.bat
@@ -0,0 +1 @@
+xmake project -k vsxmake2022 -y
diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h
index 0324b94cc..afee8536f 100644
--- a/zencore/include/zencore/refcount.h
+++ b/zencore/include/zencore/refcount.h
@@ -94,10 +94,27 @@ public:
}
return *this;
}
+ template<typename OtherType>
+ inline RefPtr& operator=(RefPtr<OtherType>&& Rhs) noexcept
+ {
+ if ((RefPtr*)&Rhs != this)
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ }
+ return *this;
+ }
inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+ template<typename OtherType>
+ explicit inline RefPtr(RefPtr<OtherType>&& Rhs) noexcept : m_Ref(Rhs.m_Ref)
+ {
+ Rhs.m_Ref = nullptr;
+ }
private:
T* m_Ref = nullptr;
+ friend class RefPtr;
};
/**
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index 1e8907906..4c378730f 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -413,6 +413,17 @@ private:
char m_StringBuffer[N];
};
+template<size_t N>
+class WriteToString : public ExtendableStringBuilder<N>
+{
+public:
+ template<typename... ArgTypes>
+ explicit WriteToString(ArgTypes&&... Args)
+ {
+ (*this << ... << std::forward<ArgTypes>(Args));
+ }
+};
+
//////////////////////////////////////////////////////////////////////////
extern template class StringBuilderImpl<wchar_t>;
@@ -454,6 +465,17 @@ private:
wchar_t m_Buffer[N];
};
+template<size_t N>
+class WriteToWideString : public ExtendableWideStringBuilder<N>
+{
+public:
+ template<typename... ArgTypes>
+ explicit WriteToWideString(ArgTypes&&... Args)
+ {
+ (*this << ... << Forward<ArgTypes>(Args));
+ }
+};
+
//////////////////////////////////////////////////////////////////////////
void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out);
diff --git a/zenserver-test/cachepolicy-tests.cpp b/zenserver-test/cachepolicy-tests.cpp
new file mode 100644
index 000000000..686ff818c
--- /dev/null
+++ b/zenserver-test/cachepolicy-tests.cpp
@@ -0,0 +1,164 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/zencore.h>
+
+#if ZEN_WITH_TESTS
+
+# include <zencore/compactbinary.h>
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/string.h>
+# include <zencore/testing.h>
+# include <zencore/uid.h>
+# include <zenutil/cache/cachepolicy.h>
+
+namespace zen::tests {
+
+using namespace std::literals;
+
+TEST_CASE("cachepolicy")
+{
+ SUBCASE("atomics serialization")
+ {
+ CachePolicy SomeAtomics[] = {CachePolicy::None,
+ CachePolicy::QueryLocal,
+ CachePolicy::StoreRemote,
+ CachePolicy::SkipData,
+ CachePolicy::KeepAlive,
+ CachePolicy::Disable};
+ for (CachePolicy Atomic : SomeAtomics)
+ {
+ CHECK(ParseCachePolicy(WriteToString<128>(Atomic)) == Atomic);
+ }
+ // Also verify that we ignore unrecognized bits
+ for (CachePolicy Atomic : SomeAtomics)
+ {
+ CHECK(ParseCachePolicy(WriteToString<128>(Atomic | (CachePolicy)0x10000000)) == Atomic);
+ }
+ }
+ SUBCASE("aliases serialization")
+ {
+ CachePolicy SomeAliases[] = {CachePolicy::Query, CachePolicy::Local};
+ for (CachePolicy Alias : SomeAliases)
+ {
+ CHECK(ParseCachePolicy(WriteToString<128>(Alias)) == Alias);
+ }
+ // Also verify that we ignore unrecognized bits
+ for (CachePolicy Alias : SomeAliases)
+ {
+ CHECK(ParseCachePolicy(WriteToString<128>(Alias | (CachePolicy)0x10000000)) == Alias);
+ }
+ }
+ SUBCASE("aliases take priority over atomics")
+ {
+ CHECK(WriteToString<128>(CachePolicy::Default).ToView() == "Default"sv);
+ CHECK(WriteToString<128>(CachePolicy::Query).ToView() == "Query"sv);
+ CHECK(WriteToString<128>(CachePolicy::Local).ToView() == "Local"sv);
+ }
+ SUBCASE("policies requiring multiple strings work")
+ {
+ char Delimiter = ',';
+ CachePolicy Combination = CachePolicy::SkipData | CachePolicy::QueryLocal;
+ CHECK(WriteToString<128>(Combination).ToView().find(Delimiter) != std::string_view::npos);
+ CHECK(ParseCachePolicy(WriteToString<128>(Combination)) == Combination);
+ }
+ SUBCASE("parsing invalid text")
+ {
+ CHECK(ParseCachePolicy(",,,") == CachePolicy::None);
+ CHECK(ParseCachePolicy("fee,fie,foo,fum") == CachePolicy::None);
+ CHECK(ParseCachePolicy("fee,KeepAlive,foo,fum") == CachePolicy::KeepAlive);
+ }
+}
+
+TEST_CASE("cacherecordpolicy")
+{
+ SUBCASE("policy with no values")
+ {
+ CachePolicy Policy = CachePolicy::SkipData | CachePolicy::QueryLocal;
+ CacheRecordPolicy RecordPolicy;
+ CacheRecordPolicyBuilder Builder(Policy);
+ RecordPolicy = Builder.Build();
+ SUBCASE("construct")
+ {
+ CHECK(RecordPolicy.IsUniform());
+ CHECK(RecordPolicy.GetRecordPolicy() == Policy);
+ CHECK(RecordPolicy.GetDefaultValuePolicy() == Policy);
+ CHECK(RecordPolicy.GetValuePolicy(Oid::NewOid()) == Policy);
+ CHECK(RecordPolicy.GetValuePolicies().size() == 0);
+ }
+ SUBCASE("saveload")
+ {
+ CbWriter Writer;
+ RecordPolicy.Save(Writer);
+ CbObject Saved = Writer.Save()->AsObject();
+ CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved);
+ CHECK(Loaded.IsUniform());
+ CHECK(Loaded.GetRecordPolicy() == Policy);
+ CHECK(Loaded.GetDefaultValuePolicy() == Policy);
+ CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == Policy);
+ CHECK(Loaded.GetValuePolicies().size() == 0);
+ }
+ }
+
+ SUBCASE("policy with values")
+ {
+ CachePolicy DefaultPolicy = CachePolicy::StoreRemote | CachePolicy::QueryLocal;
+ CachePolicy PartialOverlap = CachePolicy::StoreRemote;
+ CachePolicy NoOverlap = CachePolicy::QueryRemote;
+ CachePolicy UnionPolicy = DefaultPolicy | PartialOverlap | NoOverlap;
+
+ CacheRecordPolicy RecordPolicy;
+ CacheRecordPolicyBuilder Builder(DefaultPolicy);
+ Oid PartialOid = Oid::NewOid();
+ Oid NoOverlapOid = Oid::NewOid();
+ Oid OtherOid = Oid::NewOid();
+ Builder.AddValuePolicy(PartialOid, PartialOverlap);
+ Builder.AddValuePolicy(NoOverlapOid, NoOverlap);
+ RecordPolicy = Builder.Build();
+ SUBCASE("construct")
+ {
+ CHECK(!RecordPolicy.IsUniform());
+ CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy);
+ CHECK(RecordPolicy.GetDefaultValuePolicy() == DefaultPolicy);
+ CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap);
+ CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap);
+ CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultPolicy);
+ CHECK(RecordPolicy.GetValuePolicies().size() == 2);
+ }
+ SUBCASE("saveload")
+ {
+ CbWriter Writer;
+ RecordPolicy.Save(Writer);
+ CbObject Saved = Writer.Save()->AsObject();
+ CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved);
+ CHECK(!RecordPolicy.IsUniform());
+ CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy);
+ CHECK(RecordPolicy.GetDefaultValuePolicy() == DefaultPolicy);
+ CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap);
+ CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap);
+ CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultPolicy);
+ CHECK(RecordPolicy.GetValuePolicies().size() == 2);
+ }
+ }
+
+ SUBCASE("parsing invalid text")
+ {
+ CacheRecordPolicy Loaded = CacheRecordPolicy::Load(CbObject());
+ CHECK(Loaded.IsUniform());
+ CHECK(Loaded.GetRecordPolicy() == CachePolicy::Default);
+ CHECK(Loaded.GetDefaultValuePolicy() == CachePolicy::Default);
+ CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == CachePolicy::Default);
+ CHECK(Loaded.GetValuePolicies().size() == 0);
+
+ CachePolicy Policy = CachePolicy::SkipData;
+ Loaded = CacheRecordPolicy::Load(CbObject(), Policy);
+ CHECK(Loaded.IsUniform());
+ CHECK(Loaded.GetRecordPolicy() == Policy);
+ CHECK(Loaded.GetDefaultValuePolicy() == Policy);
+ CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == Policy);
+ CHECK(Loaded.GetValuePolicies().size() == 0);
+ }
+}
+
+} // namespace zen::tests
+
+#endif
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 75aae6321..c1d48502e 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -1536,13 +1536,13 @@ TEST_CASE("zcache.policy")
}
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=QueryLocal,Store", LocalCfg.BaseUri, Bucket, Key)},
cpr::Header{{"Accept", "application/octet-stream"}});
CHECK(Result.status_code == 404);
}
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
cpr::Header{{"Accept", "application/octet-stream"}});
CHECK(Result.status_code == 200);
}
@@ -1564,7 +1564,7 @@ TEST_CASE("zcache.policy")
// Store binary cache value locally
{
- cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,StoreLocal", LocalCfg.BaseUri, Bucket, Key)},
cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
cpr::Header{{"Content-Type", "application/octet-stream"}});
CHECK(Result.status_code == 201);
@@ -1599,7 +1599,7 @@ TEST_CASE("zcache.policy")
// Store binary cache value locally and upstream
{
- cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
cpr::Header{{"Content-Type", "application/octet-stream"}});
CHECK(Result.status_code == 201);
@@ -1643,13 +1643,13 @@ TEST_CASE("zcache.policy")
}
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=QueryLocal,Store", LocalCfg.BaseUri, Bucket, Key)},
cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
CHECK(Result.status_code == 404);
}
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
CHECK(Result.status_code == 200);
}
@@ -1673,7 +1673,7 @@ TEST_CASE("zcache.policy")
// Store packge locally
{
- cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,StoreLocal", LocalCfg.BaseUri, Bucket, Key)},
cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
CHECK(Result.status_code == 201);
@@ -1710,7 +1710,7 @@ TEST_CASE("zcache.policy")
// Store package locally and upstream
{
- cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
CHECK(Result.status_code == 201);
@@ -1729,129 +1729,6 @@ TEST_CASE("zcache.policy")
}
}
- SUBCASE("skip - 'attachments' does not return attachments")
- {
- ZenConfig LocalCfg = ZenConfig::New();
- ZenServerInstance LocalInst(TestEnv);
- const auto Bucket = "texture"sv;
-
- LocalCfg.Spawn(LocalInst);
-
- zen::IoHash Key;
- zen::IoHash PayloadId;
-
- // Store package locally
- {
- zen::CbPackage Package = GeneratePackage(Key, PayloadId);
- auto Buf = ToBuffer(Package);
-
- CHECK(Package.GetAttachments().size() != 0);
- cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}", LocalCfg.BaseUri, Bucket, Key)},
- cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
- cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
- CHECK(Result.status_code == 201);
- }
-
- {
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=attachments", LocalCfg.BaseUri, Bucket, Key)},
- cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
- CHECK(Result.status_code == 200);
-
- zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
- zen::CbPackage Package;
- const bool Ok = Package.TryLoad(Body);
- CHECK(Ok);
-
- CHECK(Ok);
-
- CbObject CacheRecord = Package.GetObject();
- std::vector<IoHash> AttachmentKeys;
-
- CacheRecord.IterateAttachments(
- [&AttachmentKeys](CbFieldView AttachmentKey) { AttachmentKeys.push_back(AttachmentKey.AsHash()); });
-
- CHECK(AttachmentKeys.size() != 0);
- CHECK(Package.GetAttachments().size() == 0);
- }
-
- {
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}", LocalCfg.BaseUri, Bucket, Key)},
- cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
- CHECK(Result.status_code == 200);
-
- zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
- zen::CbPackage Package;
- const bool Ok = Package.TryLoad(Body);
- CHECK(Ok);
-
- CHECK(Ok);
- CHECK(Package.GetAttachments().size() != 0);
- }
- }
-
- SUBCASE("skip - 'attachments' does not return attachments when retrieved from upstream")
- {
- ZenConfig UpstreamCfg = ZenConfig::New(13338);
- ZenServerInstance UpstreamInst(TestEnv);
- ZenConfig LocalCfg = ZenConfig::NewWithUpstream(13338);
- ZenServerInstance LocalInst(TestEnv);
- const auto Bucket = "texture"sv;
-
- UpstreamCfg.Spawn(UpstreamInst);
- LocalCfg.Spawn(LocalInst);
-
- zen::IoHash Key;
- zen::IoHash PayloadId;
-
- // Store package upstream
- {
- zen::CbPackage Package = GeneratePackage(Key, PayloadId);
- auto Buf = ToBuffer(Package);
-
- CHECK(Package.GetAttachments().size() != 0);
- cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}", UpstreamCfg.BaseUri, Bucket, Key)},
- cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
- cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
- CHECK(Result.status_code == 201);
- }
-
- {
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=attachments", LocalCfg.BaseUri, Bucket, Key)},
- cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
- CHECK(Result.status_code == 200);
-
- zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
- zen::CbPackage Package;
- const bool Ok = Package.TryLoad(Body);
- CHECK(Ok);
-
- CHECK(Ok);
-
- CbObject CacheRecord = Package.GetObject();
- std::vector<IoHash> AttachmentKeys;
-
- CacheRecord.IterateAttachments(
- [&AttachmentKeys](CbFieldView AttachmentKey) { AttachmentKeys.push_back(AttachmentKey.AsHash()); });
-
- CHECK(AttachmentKeys.size() != 0);
- CHECK(Package.GetAttachments().size() == 0);
- }
-
- {
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}", LocalCfg.BaseUri, Bucket, Key)},
- cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
- CHECK(Result.status_code == 200);
-
- zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
- zen::CbPackage Package;
- const bool Ok = Package.TryLoad(Body);
- CHECK(Ok);
-
- CHECK(Ok);
- CHECK(Package.GetAttachments().size() != 0);
- }
- }
-
SUBCASE("skip - 'data' returns empty cache record/payload")
{
ZenConfig Cfg = ZenConfig::New();
@@ -1875,7 +1752,7 @@ TEST_CASE("zcache.policy")
// Get package
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)},
cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
CHECK(IsHttpSuccessCode(Result.status_code));
CHECK(Result.text.size() == 0);
@@ -1883,7 +1760,7 @@ TEST_CASE("zcache.policy")
// Get record
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)},
cpr::Header{{"Accept", "application/x-ue-cbobject"}});
CHECK(IsHttpSuccessCode(Result.status_code));
CHECK(Result.text.size() == 0);
@@ -1891,8 +1768,9 @@ TEST_CASE("zcache.policy")
// Get payload
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key, PayloadId)},
- cpr::Header{{"Accept", "application/x-ue-comp"}});
+ cpr::Response Result =
+ cpr::Get(cpr::Url{fmt::format("{}/{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key, PayloadId)},
+ cpr::Header{{"Accept", "application/x-ue-comp"}});
CHECK(IsHttpSuccessCode(Result.status_code));
CHECK(Result.text.size() == 0);
}
@@ -1919,7 +1797,7 @@ TEST_CASE("zcache.policy")
// Get package
{
- cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key)},
+ cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)},
cpr::Header{{"Accept", "application/octet-stream"}});
CHECK(IsHttpSuccessCode(Result.status_code));
CHECK(Result.text.size() == 0);
@@ -2011,9 +1889,8 @@ TEST_CASE("zcache.rpc")
}
Request.EndArray();
- Request.BeginObject("Policy");
- CacheRecordPolicy::Save(Policy, Request);
- Request.EndObject();
+ Request.SetName("Policy"sv);
+ Policy.Save(Request);
Request.EndObject();
@@ -2134,38 +2011,6 @@ TEST_CASE("zcache.rpc")
}
}
- SUBCASE("policy - 'SkipAttachments' does not return any record attachments")
- {
- std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
- const uint16_t PortNumber = 13337;
- const auto BaseUri = fmt::format("http://localhost:{}/z$", PortNumber);
-
- ZenServerInstance Inst(TestEnv);
- Inst.SetTestDir(TestDir);
- Inst.SpawnServer(PortNumber);
- Inst.WaitUntilReady();
-
- CacheRecordPolicy Policy(CachePolicy::QueryLocal | CachePolicy::SkipAttachments);
- std::vector<zen::CacheKey> Keys = PutCacheRecords(BaseUri, ""sv, "mastodon"sv, 4);
- GetCacheRecordResult Result = GetCacheRecords(BaseUri, Keys, Policy);
-
- CHECK(Result.Records.size() == Keys.size());
-
- std::span<const zen::CbAttachment> Attachments = Result.Response.GetAttachments();
- CHECK(Attachments.empty());
-
- for (size_t Index = 0; CbFieldView RecordView : Result.Records)
- {
- const CacheKey& ExpectedKey = Keys[Index++];
-
- CbObjectView RecordObj = RecordView.AsObjectView();
- CbObjectView KeyObj = RecordObj["CacheKey"sv].AsObjectView();
- const CacheKey Key = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash());
-
- CHECK(Key == ExpectedKey);
- }
- }
-
SUBCASE("policy - 'QueryLocal' does not query upstream")
{
using namespace utils;
diff --git a/zenserver/auth/authservice.cpp b/zenserver/auth/authservice.cpp
index c6def15b4..eecad45bf 100644
--- a/zenserver/auth/authservice.cpp
+++ b/zenserver/auth/authservice.cpp
@@ -11,7 +11,7 @@ HttpAuthService::HttpAuthService()
{
m_Router.RegisterRoute(
"token",
- [this](HttpRouterRequest& RouterRequest) {
+ [](HttpRouterRequest& RouterRequest) {
HttpServerRequest& ServerRequest = RouterRequest.ServerRequest();
ServerRequest.WriteResponse(HttpResponseCode::OK);
},
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 5918d5178..8854ff3d1 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -7,6 +7,7 @@
#include <zencore/compactbinarypackage.h>
#include <zencore/compactbinaryvalidation.h>
#include <zencore/compress.h>
+#include <zencore/enumflags.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/scopeguard.h>
@@ -42,11 +43,8 @@ using namespace std::literals;
CachePolicy
ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams)
{
- const CachePolicy QueryPolicy = zen::ParseQueryCachePolicy(QueryParams.GetValue("query"sv));
- const CachePolicy StorePolicy = zen::ParseStoreCachePolicy(QueryParams.GetValue("store"sv));
- const CachePolicy SkipPolicy = zen::ParseSkipCachePolicy(QueryParams.GetValue("skip"sv));
-
- return QueryPolicy | StorePolicy | SkipPolicy;
+ std::string_view PolicyText = QueryParams.GetValue("Policy"sv);
+ return !PolicyText.empty() ? zen::ParseCachePolicy(PolicyText) : CachePolicy::Default;
}
struct AttachmentCount
@@ -134,16 +132,15 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL
}
- const auto QueryParams = Request.GetQueryParams();
- CachePolicy Policy = ParseCachePolicy(QueryParams);
+ CachePolicy PolicyFromURL = ParseCachePolicy(Request.GetQueryParams());
if (Ref.PayloadId == IoHash::Zero)
{
- return HandleCacheRecordRequest(Request, Ref, Policy);
+ return HandleCacheRecordRequest(Request, Ref, PolicyFromURL);
}
else
{
- return HandleCachePayloadRequest(Request, Ref, Policy);
+ return HandleCachePayloadRequest(Request, Ref, PolicyFromURL);
}
return;
@@ -180,19 +177,19 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
}
void
-HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
+HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL)
{
switch (Request.RequestVerb())
{
case HttpVerb::kHead:
case HttpVerb::kGet:
{
- HandleGetCacheRecord(Request, Ref, Policy);
+ HandleGetCacheRecord(Request, Ref, PolicyFromURL);
}
break;
case HttpVerb::kPut:
- HandlePutCacheRecord(Request, Ref, Policy);
+ HandlePutCacheRecord(Request, Ref, PolicyFromURL);
break;
default:
break;
@@ -200,13 +197,12 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request,
}
void
-HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
+HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL)
{
- const ZenContentType AcceptType = Request.AcceptContentType();
- const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData;
- const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments;
- const bool PartialOnError = (Policy & CachePolicy::PartialOnError) == CachePolicy::PartialOnError;
- const bool QueryUpstream = (Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote;
+ const ZenContentType AcceptType = Request.AcceptContentType();
+ const bool SkipData = EnumHasAllFlags(PolicyFromURL, CachePolicy::SkipData);
+ const bool PartialRecord = EnumHasAllFlags(PolicyFromURL, CachePolicy::PartialRecord);
+ const bool QueryUpstream = EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryRemote);
bool Success = false;
ZenCacheValue LocalCacheValue;
@@ -221,28 +217,18 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
uint32_t MissingCount = 0;
CbObjectView CacheRecord(LocalCacheValue.Value.Data());
- CacheRecord.IterateAttachments([this, SkipAttachments, &MissingCount, &Package](CbFieldView AttachmentHash) {
- if (SkipAttachments && MissingCount == 0)
+ CacheRecord.IterateAttachments([this, &MissingCount, &Package](CbFieldView AttachmentHash) {
+ if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
{
- if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash()))
- {
- MissingCount++;
- }
+ Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
}
else
{
- if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
- {
- Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
- }
- else
- {
- MissingCount++;
- }
+ MissingCount++;
}
});
- Success = MissingCount == 0 || PartialOnError;
+ Success = MissingCount == 0 || PartialRecord;
if (Success)
{
@@ -286,7 +272,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
// Issue upstream query asynchronously in order to keep requests flowing without
// hogging I/O servicing threads with blocking work
- Request.WriteResponseAsync([this, AcceptType, SkipData, SkipAttachments, PartialOnError, Ref](HttpServerRequest& AsyncRequest) {
+ Request.WriteResponseAsync([this, AcceptType, SkipData, PartialRecord, Ref](HttpServerRequest& AsyncRequest) {
bool Success = false;
ZenCacheValue UpstreamCacheValue;
@@ -328,7 +314,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
CbObject CacheRecord = Package.GetObject();
AttachmentCount Count;
- CacheRecord.IterateAttachments([this, &Package, &Ref, &Count, SkipAttachments](CbFieldView HashView) {
+ CacheRecord.IterateAttachments([this, &Package, &Ref, &Count](CbFieldView HashView) {
if (const CbAttachment* Attachment = Package.FindAttachment(HashView.AsHash()))
{
if (CompressedBuffer Compressed = Attachment->AsCompressedBinary())
@@ -342,7 +328,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
else
{
- ZEN_WARN("Uncompressed payload '{}' from upstream cache record '{}/{}'",
+ ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'",
HashView.AsHash(),
Ref.BucketSegment,
Ref.HashKey);
@@ -351,16 +337,13 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(HashView.AsHash()))
{
- if (!SkipAttachments)
- {
- Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
- }
+ Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
Count.Valid++;
}
Count.Total++;
});
- if ((Count.Valid == Count.Total) || PartialOnError)
+ if ((Count.Valid == Count.Total) || PartialRecord)
{
ZenCacheValue CacheValue;
CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
@@ -368,12 +351,6 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue);
- if (SkipAttachments)
- {
- Package.Reset();
- Package.SetObject(CacheRecord);
- }
-
BinaryWriter MemStream;
Package.Save(MemStream);
@@ -427,7 +404,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
void
-HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
+HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL)
{
IoBuffer Body = Request.ReadPayload();
@@ -436,8 +413,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
return Request.WriteResponse(HttpResponseCode::BadRequest);
}
- const HttpContentType ContentType = Request.RequestContentType();
- const bool StoreUpstream = (Policy & CachePolicy::StoreRemote) == CachePolicy::StoreRemote;
+ const HttpContentType ContentType = Request.RequestContentType();
Body.SetContentType(ContentType);
@@ -446,7 +422,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType));
m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body});
- if (StoreUpstream)
+ if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote))
{
m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kBinary, .Key = {Ref.BucketSegment, Ref.HashKey}});
}
@@ -463,6 +439,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv);
}
+ CachePolicy Policy = PolicyFromURL;
CbObjectView CacheRecord(Body.Data());
std::vector<IoHash> ValidAttachments;
int32_t TotalCount = 0;
@@ -489,7 +466,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
const bool IsPartialRecord = TotalCount != static_cast<int32_t>(ValidAttachments.size());
- if (StoreUpstream && !IsPartialRecord)
+ if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
m_UpstreamCache.EnqueueUpstream(
{.Type = ZenContentType::kCbObject, .Key = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(ValidAttachments)});
@@ -506,6 +483,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid package", Ref.BucketSegment, Ref.HashKey, ToString(ContentType));
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv);
}
+ CachePolicy Policy = PolicyFromURL;
CbObject CacheRecord = Package.GetObject();
AttachmentCount Count;
@@ -570,7 +548,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
const bool IsPartialRecord = Count.Valid != Count.Total;
- if (StoreUpstream && !IsPartialRecord)
+ if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
m_UpstreamCache.EnqueueUpstream(
{.Type = ZenContentType::kCbPackage, .Key = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(ValidAttachments)});
@@ -585,18 +563,16 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
}
void
-HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
+HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL)
{
switch (Request.RequestVerb())
{
case HttpVerb::kHead:
case HttpVerb::kGet:
- {
- HandleGetCachePayload(Request, Ref, Policy);
- }
+ HandleGetCachePayload(Request, Ref, PolicyFromURL);
break;
case HttpVerb::kPut:
- HandlePutCachePayload(Request, Ref, Policy);
+ HandlePutCachePayload(Request, Ref, PolicyFromURL);
break;
default:
break;
@@ -604,11 +580,12 @@ HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request
}
void
-HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
+HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL)
{
- IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId);
- bool InUpstreamCache = false;
- const bool QueryUpstream = !Payload && (Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote;
+ IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId);
+ bool InUpstreamCache = false;
+ CachePolicy Policy = PolicyFromURL;
+ const bool QueryUpstream = !Payload && EnumHasAllFlags(Policy, CachePolicy::QueryRemote);
if (QueryUpstream)
{
@@ -621,7 +598,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
}
else
{
- ZEN_WARN("got uncompressed upstream cache payload");
+ ZEN_WARN("got uncompressed upstream cache value");
}
}
}
@@ -647,7 +624,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
m_CacheStats.UpstreamHitCount++;
}
- if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData)
+ if (EnumHasAllFlags(Policy, CachePolicy::SkipData))
{
Request.WriteResponse(HttpResponseCode::OK);
}
@@ -658,10 +635,10 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
}
void
-HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
+HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL)
{
- // Note: Individual cache payloads are not propagated upstream until a valid cache record has been stored
- ZEN_UNUSED(Policy);
+ // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored
+ ZEN_UNUSED(PolicyFromURL);
IoBuffer Body = Request.ReadPayload();
@@ -681,7 +658,7 @@ HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Reques
if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Ref.PayloadId)
{
- return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Payload ID does not match attachment hash"sv);
+ return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "ValueId does not match attachment hash"sv);
}
CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
@@ -731,7 +708,7 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef&
}
else
{
- // Cache record + payload lookup
+ // Cache record + valueid lookup
HashSegment = Key.substr(BucketSplitOffset + 1, PayloadSplitOffset - BucketSplitOffset - 1);
PayloadSegment = Key.substr(PayloadSplitOffset + 1);
}
@@ -787,7 +764,7 @@ HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request)
{
HandleRpcGetCacheRecords(AsyncRequest, RpcRequest);
}
- else if (Method == "GetCachePayloads"sv)
+ else if (Method == "GetCacheValues"sv)
{
HandleRpcGetCachePayloads(AsyncRequest, RpcRequest);
}
@@ -818,11 +795,10 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req
ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheRecords"sv);
- CacheRecordPolicy::Load(Params["Policy"sv].AsObjectView(), Policy);
+ Policy = CacheRecordPolicy::Load(Params["Policy"sv].AsObjectView());
- const bool PartialOnError = Policy.HasRecordPolicy(CachePolicy::PartialOnError);
- const bool SkipAttachments = Policy.HasRecordPolicy(CachePolicy::SkipAttachments);
- const bool QueryRemote = Policy.HasRecordPolicy(CachePolicy::QueryRemote);
+ const bool PartialRecord = EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::PartialRecord);
+ const bool QueryRemote = EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryRemote);
for (CbFieldView KeyView : Params["CacheKeys"sv])
{
@@ -845,30 +821,20 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req
if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue))
{
CbObjectView CacheRecord(CacheValue.Value.Data());
- CacheRecord.IterateAttachments([this, SkipAttachments, &MissingCount, &RpcResponse](CbFieldView AttachmentHash) {
- if (SkipAttachments && MissingCount == 0)
+ CacheRecord.IterateAttachments([this, &MissingCount, &RpcResponse](CbFieldView AttachmentHash) {
+ if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
{
- if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash()))
- {
- MissingCount++;
- }
+ ZEN_ASSERT(Chunk.GetSize() > 0);
+ RpcResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
}
else
{
- if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
- {
- ZEN_ASSERT(Chunk.GetSize() > 0);
- RpcResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
- }
- else
- {
- MissingCount++;
- }
+ MissingCount++;
}
});
}
- if (CacheValue.Value && (MissingCount == 0 || PartialOnError))
+ if (CacheValue.Value && (MissingCount == 0 || PartialRecord))
{
ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL) {}",
Key.Bucket,
@@ -895,80 +861,76 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req
if (!UpstreamRequests.empty())
{
- const auto OnCacheRecordGetComplete =
- [this, &CacheValues, &RpcResponse, PartialOnError, SkipAttachments](CacheRecordGetCompleteParams&& Params) {
- ZEN_ASSERT(Params.KeyIndex < CacheValues.size());
+ const auto OnCacheRecordGetComplete = [this, &CacheValues, &RpcResponse, PartialRecord](CacheRecordGetCompleteParams&& Params) {
+ ZEN_ASSERT(Params.KeyIndex < CacheValues.size());
- IoBuffer CacheValue;
- AttachmentCount Count;
+ IoBuffer CacheValue;
+ AttachmentCount Count;
- if (Params.Record)
- {
- Params.Record.IterateAttachments([this, &RpcResponse, SkipAttachments, &Params, &Count](CbFieldView HashView) {
- if (const CbAttachment* Attachment = Params.Package.FindAttachment(HashView.AsHash()))
+ if (Params.Record)
+ {
+ Params.Record.IterateAttachments([this, &RpcResponse, &Params, &Count](CbFieldView HashView) {
+ if (const CbAttachment* Attachment = Params.Package.FindAttachment(HashView.AsHash()))
+ {
+ if (CompressedBuffer Compressed = Attachment->AsCompressedBinary())
{
- if (CompressedBuffer Compressed = Attachment->AsCompressedBinary())
+ auto InsertResult = m_CidStore.AddChunk(Compressed);
+ if (InsertResult.New)
{
- auto InsertResult = m_CidStore.AddChunk(Compressed);
- if (InsertResult.New)
- {
- Count.New++;
- }
- Count.Valid++;
-
- if (!SkipAttachments)
- {
- RpcResponse.AddAttachment(CbAttachment(Compressed));
- }
- }
- else
- {
- ZEN_DEBUG("Uncompressed payload '{}' from upstream cache record '{}/{}'",
- HashView.AsHash(),
- Params.Key.Bucket,
- Params.Key.Hash);
- Count.Invalid++;
+ Count.New++;
}
+ Count.Valid++;
+
+ RpcResponse.AddAttachment(CbAttachment(Compressed));
}
- else if (m_CidStore.ContainsChunk(HashView.AsHash()))
+ else
{
- Count.Valid++;
+ ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}'",
+ HashView.AsHash(),
+ Params.Key.Bucket,
+ Params.Key.Hash);
+ Count.Invalid++;
}
- Count.Total++;
- });
-
- if ((Count.Valid == Count.Total) || PartialOnError)
+ }
+ else if (m_CidStore.ContainsChunk(HashView.AsHash()))
{
- CacheValue = CbObject::Clone(Params.Record).GetBuffer().AsIoBuffer();
+ Count.Valid++;
}
- }
+ Count.Total++;
+ });
- if (CacheValue)
- {
- ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (new/valid/total) (UPSTREAM)",
- Params.Key.Bucket,
- Params.Key.Hash,
- NiceBytes(CacheValue.GetSize()),
- ToString(HttpContentType::kCbPackage),
- Count.New,
- Count.Valid,
- Count.Total);
-
- CacheValue.SetContentType(ZenContentType::kCbObject);
-
- CacheValues[Params.KeyIndex] = CacheValue;
- m_CacheStore.Put(Params.Key.Bucket, Params.Key.Hash, {.Value = CacheValue});
-
- m_CacheStats.HitCount++;
- m_CacheStats.UpstreamHitCount++;
- }
- else
+ if ((Count.Valid == Count.Total) || PartialRecord)
{
- const bool IsPartial = Count.Valid != Count.Total;
- ZEN_DEBUG("MISS - '{}/{}' {}", Params.Key.Bucket, Params.Key.Hash, IsPartial ? "(partial)"sv : ""sv);
- m_CacheStats.MissCount++;
+ CacheValue = CbObject::Clone(Params.Record).GetBuffer().AsIoBuffer();
}
- };
+ }
+
+ if (CacheValue)
+ {
+ ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (new/valid/total) (UPSTREAM)",
+ Params.Key.Bucket,
+ Params.Key.Hash,
+ NiceBytes(CacheValue.GetSize()),
+ ToString(HttpContentType::kCbPackage),
+ Count.New,
+ Count.Valid,
+ Count.Total);
+
+ CacheValue.SetContentType(ZenContentType::kCbObject);
+
+ CacheValues[Params.KeyIndex] = CacheValue;
+ m_CacheStore.Put(Params.Key.Bucket, Params.Key.Hash, {.Value = CacheValue});
+
+ m_CacheStats.HitCount++;
+ m_CacheStats.UpstreamHitCount++;
+ }
+ else
+ {
+ const bool IsPartial = Count.Valid != Count.Total;
+ ZEN_DEBUG("MISS - '{}/{}' {}", Params.Key.Bucket, Params.Key.Hash, IsPartial ? "(partial)"sv : ""sv);
+ m_CacheStats.MissCount++;
+ }
+ };
m_UpstreamCache.GetCacheRecords(CacheKeys, UpstreamRequests, Policy, std::move(OnCacheRecordGetComplete));
}
@@ -1005,7 +967,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re
{
ZEN_TRACE_CPU("Z$::RpcGetCachePayloads");
- ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCachePayloads"sv);
+ ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheValues"sv);
std::vector<CacheChunkRequest> ChunkRequests;
std::vector<size_t> UpstreamRequests;
@@ -1014,19 +976,20 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re
for (CbFieldView RequestView : Params["ChunkRequests"sv])
{
- CbObjectView RequestObject = RequestView.AsObjectView();
- CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
- const CacheKey Key = CacheKey::Create(KeyObject["Bucket"sv].AsString(), KeyObject["Hash"sv].AsHash());
- const IoHash ChunkId = RequestObject["ChunkId"sv].AsHash();
- const Oid PayloadId = RequestObject["PayloadId"sv].AsObjectId();
- const uint64_t RawOffset = RequestObject["RawOffset"sv].AsUInt64();
- const uint64_t RawSize = RequestObject["RawSize"sv].AsUInt64();
- const uint32_t ChunkPolicy = RequestObject["Policy"sv].AsUInt32();
+ CbObjectView RequestObject = RequestView.AsObjectView();
+ CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
+ const CacheKey Key = CacheKey::Create(KeyObject["Bucket"sv].AsString(), KeyObject["Hash"sv].AsHash());
+ const IoHash ChunkId = RequestObject["ChunkId"sv].AsHash();
+ const Oid PayloadId = RequestObject["ValueId"sv].AsObjectId();
+ const uint64_t RawOffset = RequestObject["RawOffset"sv].AsUInt64();
+ const uint64_t RawSize = RequestObject["RawSize"sv].AsUInt64();
+ std::string_view PolicyText = RequestObject["Policy"sv].AsString();
+ const CachePolicy ChunkPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
// Note we could use emplace_back here but [Apple] LLVM-12's C++ library
// can't infer a constructor like other platforms (or can't handle an
// initializer list like others do).
- ChunkRequests.push_back({Key, ChunkId, PayloadId, RawOffset, RawSize, static_cast<CachePolicy>(ChunkPolicy)});
+ ChunkRequests.push_back({Key, ChunkId, PayloadId, RawOffset, RawSize, ChunkPolicy});
}
if (ChunkRequests.empty())
@@ -1036,13 +999,13 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re
Chunks.resize(ChunkRequests.size());
- // Unreal uses a 12 byte ID to address cache record payloads. When the uncompressed hash (ChunkId)
- // is missing, load the cache record and try to find the raw hash from the payload ID.
+ // Unreal uses a 12 byte ID to address cache record values. When the uncompressed hash (ChunkId)
+ // is missing, load the cache record and try to find the raw hash from the ValueId.
{
const auto GetChunkIdFromPayloadId = [](CbObjectView Record, const Oid& PayloadId) -> IoHash {
if (PayloadId)
{
- // A valid PayloadId indicates that the caller is searching for a Payload in a Record
+ // A valid ValueId indicates that the caller is searching for a Value in a Record
// that was Put with ICacheStore::Put
for (CbFieldView ValueView : Record["Values"sv])
{
@@ -1079,7 +1042,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re
}
else
{
- // An invalid PayloadId indicates that the caller is requesting a Value that
+ // An invalid ValueId indicates that the caller is requesting a Value that
// was Put with ICacheStore::PutValue
return Record["RawHash"sv].AsHash();
}
@@ -1115,8 +1078,8 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re
for (size_t RequestIndex = 0; const CacheChunkRequest& ChunkRequest : ChunkRequests)
{
- const bool QueryLocal = (ChunkRequest.Policy & CachePolicy::QueryLocal) == CachePolicy::QueryLocal;
- const bool QueryRemote = (ChunkRequest.Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote;
+ const bool QueryLocal = EnumHasAllFlags(ChunkRequest.Policy, CachePolicy::QueryLocal);
+ const bool QueryRemote = EnumHasAllFlags(ChunkRequest.Policy, CachePolicy::QueryRemote);
if (QueryLocal)
{
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index 1bf3940e7..a7ecba845 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -41,7 +41,7 @@ enum class CachePolicy : uint32_t;
*
* Additionally, attachments may be addressed as:
*
- * {BucketId}/{KeyHash}/{PayloadHash}
+ * {BucketId}/{KeyHash}/{ValueHash}
*
* Where the two initial components are the same as for the main endpoint
*
@@ -84,12 +84,12 @@ private:
};
[[nodiscard]] bool ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef);
- void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
- void HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
- void HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
- void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
- void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
- void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
+ void HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL);
+ void HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl);
+ void HandlePutCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL);
+ void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL);
+ void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL);
+ void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromURL);
void HandleRpcRequest(zen::HttpServerRequest& Request);
void HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest);
void HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest);
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index 4afe012dd..3c4f6f3d8 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -91,6 +91,16 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
const char* DefaultHttp = "asio";
#endif
+ // Note to those adding future options; std::filesystem::path-type options
+ // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >>
+ // stream operator to convert argv value into the options type. std::fs::path
+ // expects paths in streams to be quoted but argv paths are unquoted. By
+ // going into a std::string first, paths with whitespace parse correctly.
+ std::string DataDir;
+ std::string ContentDir;
+ std::string AbsLogFile;
+ std::string ConfigFile;
+
cxxopts::Options options("zenserver", "Zen Server");
options.add_options()("dedicated",
"Enable dedicated server mode",
@@ -99,10 +109,10 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
options.add_options()("help", "Show command line help");
options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId));
- options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::filesystem::path>(ServerOptions.DataDir));
- options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::filesystem::path>(ServerOptions.ContentDir));
- options.add_options()("abslog", "Path to log file", cxxopts::value<std::filesystem::path>(ServerOptions.AbsLogFile));
- options.add_options()("config", "Path to Lua config file", cxxopts::value<std::filesystem::path>(ServerOptions.ConfigFile));
+ options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
+ options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::string>(ContentDir));
+ options.add_options()("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile));
+ options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
options.add_options()("no-sentry",
"Disable Sentry crash handler",
cxxopts::value<bool>(ServerOptions.NoSentry)->default_value("false"));
@@ -327,6 +337,10 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
exit(0);
}
+ ServerOptions.DataDir = DataDir;
+ ServerOptions.ContentDir = ContentDir;
+ ServerOptions.AbsLogFile = AbsLogFile;
+ ServerOptions.ConfigFile = ConfigFile;
ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
if (!ServerOptions.ConfigFile.empty())
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index 3d6641a4f..b0343a61b 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -378,7 +378,7 @@ namespace detail {
if (It == std::end(CacheRecord.PayloadIds))
{
- OutReason = fmt::format("payload '{}' MISSING from local cache", PayloadId);
+ OutReason = fmt::format("value '{}' MISSING from local cache", PayloadId);
return false;
}
@@ -394,7 +394,7 @@ namespace detail {
if (!BlobResult.Success)
{
- OutReason = fmt::format("upload payload '{}' FAILED, reason '{}'", PayloadId, BlobResult.Reason);
+ OutReason = fmt::format("upload value '{}' FAILED, reason '{}'", PayloadId, BlobResult.Reason);
return false;
}
@@ -473,7 +473,7 @@ namespace detail {
Sb << MissingHash.ToHexString() << ",";
}
- return {.Reason = fmt::format("finalize '{}/{}' FAILED, still needs payload(s) '{}'",
+ return {.Reason = fmt::format("finalize '{}/{}' FAILED, still needs value(s) '{}'",
CacheRecord.Key.Bucket,
CacheRecord.Key.Hash,
Sb.ToString()),
@@ -646,9 +646,8 @@ namespace detail {
}
BatchRequest.EndArray();
- BatchRequest.BeginObject("Policy"sv);
- CacheRecordPolicy::Save(Policy, BatchRequest);
- BatchRequest.EndObject();
+ BatchRequest.SetName("Policy"sv);
+ Policy.Save(BatchRequest);
}
BatchRequest.EndObject();
@@ -726,7 +725,7 @@ namespace detail {
CbObjectWriter BatchRequest;
BatchRequest << "Method"sv
- << "GetCachePayloads";
+ << "GetCacheValues";
BatchRequest.BeginObject("Params"sv);
{
@@ -743,12 +742,11 @@ namespace detail {
BatchRequest << "Bucket"sv << Request.Key.Bucket;
BatchRequest << "Hash"sv << Request.Key.Hash;
BatchRequest.EndObject();
-
- BatchRequest.AddObjectId("PayloadId"sv, Request.PayloadId);
+ BatchRequest.AddObjectId("ValueId"sv, Request.PayloadId);
BatchRequest << "ChunkId"sv << Request.ChunkId;
BatchRequest << "RawOffset"sv << Request.RawOffset;
BatchRequest << "RawSize"sv << Request.RawSize;
- BatchRequest << "Policy"sv << static_cast<uint32_t>(Request.Policy);
+ BatchRequest << "Policy"sv << WriteToString<128>(Request.Policy).ToView();
}
BatchRequest.EndObject();
}
@@ -828,7 +826,7 @@ namespace detail {
}
else
{
- return {.Reason = std::string("invalid payload buffer"), .Success = false};
+ return {.Reason = std::string("invalid value buffer"), .Success = false};
}
}
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index 0e93f1c3d..7be93b4af 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -404,7 +404,10 @@ GcScheduler::Shutdown()
m_Status = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
m_GcSignal.notify_one();
- m_GcThread.join();
+ if (m_GcThread.joinable())
+ {
+ m_GcThread.join();
+ }
}
}
diff --git a/zenutil/cache/cachepolicy.cpp b/zenutil/cache/cachepolicy.cpp
index e1c31d885..142ba682a 100644
--- a/zenutil/cache/cachepolicy.cpp
+++ b/zenutil/cache/cachepolicy.cpp
@@ -6,62 +6,113 @@
#include <zencore/compactbinarybuilder.h>
#include <zencore/string.h>
+#include <algorithm>
+#include <unordered_map>
+
namespace zen {
using namespace std::literals;
-namespace detail { namespace cacheopt {
- constexpr std::string_view Local = "local"sv;
- constexpr std::string_view Remote = "remote"sv;
- constexpr std::string_view Data = "data"sv;
- constexpr std::string_view Meta = "meta"sv;
- constexpr std::string_view Value = "value"sv;
- constexpr std::string_view Attachments = "attachments"sv;
-}} // namespace detail::cacheopt
-
-CachePolicy
-ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default)
+namespace detail::CachePolicyImpl {
+ constexpr char DelimiterChar = ',';
+ constexpr std::string_view Delimiter = ","sv;
+ constexpr std::string_view None = "None"sv;
+ constexpr std::string_view QueryLocal = "QueryLocal"sv;
+ constexpr std::string_view QueryRemote = "QueryRemote"sv;
+ constexpr std::string_view Query = "Query"sv;
+ constexpr std::string_view StoreLocal = "StoreLocal"sv;
+ constexpr std::string_view StoreRemote = "StoreRemote"sv;
+ constexpr std::string_view Store = "Store"sv;
+ constexpr std::string_view SkipMeta = "SkipMeta"sv;
+ constexpr std::string_view SkipData = "SkipData"sv;
+ constexpr std::string_view PartialRecord = "PartialRecord"sv;
+ constexpr std::string_view KeepAlive = "KeepAlive"sv;
+ constexpr std::string_view Local = "Local"sv;
+ constexpr std::string_view Remote = "Remote"sv;
+ constexpr std::string_view Default = "Default"sv;
+ constexpr std::string_view Disable = "Disable"sv;
+
+ using TextToPolicyMap = std::unordered_map<std::string_view, CachePolicy>;
+ const TextToPolicyMap TextToPolicy = {{None, CachePolicy::None},
+ {QueryLocal, CachePolicy::QueryLocal},
+ {QueryRemote, CachePolicy::QueryRemote},
+ {Query, CachePolicy::Query},
+ {StoreLocal, CachePolicy::StoreLocal},
+ {StoreRemote, CachePolicy::StoreRemote},
+ {Store, CachePolicy::Store},
+ {SkipMeta, CachePolicy::SkipMeta},
+ {SkipData, CachePolicy::SkipData},
+ {PartialRecord, CachePolicy::PartialRecord},
+ {KeepAlive, CachePolicy::KeepAlive},
+ {Local, CachePolicy::Local},
+ {Remote, CachePolicy::Remote},
+ {Default, CachePolicy::Default},
+ {Disable, CachePolicy::Disable}};
+
+ using PolicyTextPair = std::pair<CachePolicy, std::string_view>;
+ const PolicyTextPair FlagsToString[]{
+ // Order of these Flags is important: we want the aliases before the atomic values,
+ // and the bigger aliases first, to reduce the number of tokens we add
+ {CachePolicy::Default, Default},
+ {CachePolicy::Remote, Remote},
+ {CachePolicy::Local, Local},
+ {CachePolicy::Store, Store},
+ {CachePolicy::Query, Query},
+
+ // Order of Atomics doesn't matter, so arbitrarily we list them in enum order
+ {CachePolicy::QueryLocal, QueryLocal},
+ {CachePolicy::QueryRemote, QueryRemote},
+ {CachePolicy::StoreLocal, StoreLocal},
+ {CachePolicy::StoreRemote, StoreRemote},
+ {CachePolicy::SkipMeta, SkipMeta},
+ {CachePolicy::SkipData, SkipData},
+ {CachePolicy::PartialRecord, PartialRecord},
+ {CachePolicy::KeepAlive, KeepAlive},
+
+ // None must come at the end of the array, to write out only if no others exist
+ {CachePolicy::None, None},
+ };
+ constexpr CachePolicy KnownFlags =
+ CachePolicy::Default | CachePolicy::SkipMeta | CachePolicy::SkipData | CachePolicy::KeepAlive | CachePolicy::PartialRecord;
+} // namespace detail::CachePolicyImpl
+
+StringBuilderBase&
+AppendToBuilderImpl(StringBuilderBase& Builder, CachePolicy Policy)
{
- if (QueryPolicy.empty())
+ // Remove any bits we don't recognize; write None if there are not any bits we recognize
+ Policy = Policy & detail::CachePolicyImpl::KnownFlags;
+ for (const detail::CachePolicyImpl::PolicyTextPair& Pair : detail::CachePolicyImpl::FlagsToString)
{
- return Default;
- }
-
- CachePolicy Result = CachePolicy::None;
-
- ForEachStrTok(QueryPolicy, ',', [&Result](const std::string_view& Token) {
- if (Token == detail::cacheopt::Local)
+ if (EnumHasAllFlags(Policy, Pair.first))
{
- Result |= CachePolicy::QueryLocal;
+ EnumRemoveFlags(Policy, Pair.first);
+ Builder << Pair.second << detail::CachePolicyImpl::DelimiterChar;
+ if (Policy == CachePolicy::None)
+ {
+ break;
+ }
}
- if (Token == detail::cacheopt::Remote)
- {
- Result |= CachePolicy::QueryRemote;
- }
- return true;
- });
-
- return Result;
+ }
+ Builder.RemoveSuffix(1); // Text will have been added by CachePolicy::None if not by anything else
+ return Builder;
+}
+StringBuilderBase&
+operator<<(StringBuilderBase& Builder, CachePolicy Policy)
+{
+ return AppendToBuilderImpl(Builder, Policy);
}
CachePolicy
-ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default)
+ParseCachePolicy(std::string_view Text)
{
- if (StorePolicy.empty())
- {
- return Default;
- }
+ ZEN_ASSERT(!Text.empty()); // Empty string is not valid input to ParseCachePolicy
CachePolicy Result = CachePolicy::None;
-
- ForEachStrTok(StorePolicy, ',', [&Result](const std::string_view& Token) {
- if (Token == detail::cacheopt::Local)
- {
- Result |= CachePolicy::StoreLocal;
- }
- if (Token == detail::cacheopt::Remote)
+ ForEachStrTok(Text, detail::CachePolicyImpl::DelimiterChar, [&Result](const std::string_view& Token) {
+ auto it = detail::CachePolicyImpl::TextToPolicy.find(Token);
+ if (it != detail::CachePolicyImpl::TextToPolicy.end())
{
- Result |= CachePolicy::StoreRemote;
+ Result |= it->second;
}
return true;
});
@@ -69,101 +120,120 @@ ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default)
return Result;
}
-CachePolicy
-ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default)
-{
- if (SkipPolicy.empty())
+namespace Private {
+
+ class CacheRecordPolicyShared final : public ICacheRecordPolicyShared
{
- return Default;
- }
+ public:
+ inline std::span<const CacheValuePolicy> GetValuePolicies() const final { return Values; }
- CachePolicy Result = CachePolicy::None;
+ inline void AddValuePolicy(const CacheValuePolicy& Policy) final { Values.push_back(Policy); }
- ForEachStrTok(SkipPolicy, ',', [&Result](const std::string_view& Token) {
- if (Token == detail::cacheopt::Meta)
- {
- Result |= CachePolicy::SkipMeta;
- }
- if (Token == detail::cacheopt::Value)
+ inline void Build() final
{
- Result |= CachePolicy::SkipValue;
+ std::sort(Values.begin(), Values.end(), [](const CacheValuePolicy& A, const CacheValuePolicy& B) { return A.Id < B.Id; });
}
- if (Token == detail::cacheopt::Attachments)
- {
- Result |= CachePolicy::SkipAttachments;
- }
- if (Token == detail::cacheopt::Data)
- {
- Result |= CachePolicy::SkipData;
- }
- return true;
- });
- return Result;
-}
+ private:
+ std::vector<CacheValuePolicy> Values;
+ };
-CacheRecordPolicy::CacheRecordPolicy(const CachePolicy RecordPolicy, const CachePolicy PayloadPolicy)
-: m_RecordPolicy(RecordPolicy)
-, m_DefaultPayloadPolicy(PayloadPolicy)
-{
-}
+} // namespace Private
CachePolicy
-CacheRecordPolicy::GetPayloadPolicy(const Oid& PayloadId) const
+CacheRecordPolicy::GetValuePolicy(const Oid& Id) const
{
- if (const auto It = m_PayloadPolicies.find(PayloadId); It != m_PayloadPolicies.end())
+ if (Shared)
{
- return It->second;
+ if (std::span<const CacheValuePolicy> Values = Shared->GetValuePolicies(); !Values.empty())
+ {
+ auto Iter =
+ std::lower_bound(Values.begin(), Values.end(), Id, [](const CacheValuePolicy& A, const Oid& B) { return A.Id < B; });
+ if (Iter != Values.end() && Iter->Id == Id)
+ {
+ return Iter->Policy;
+ }
+ }
}
-
- return m_DefaultPayloadPolicy;
+ return DefaultValuePolicy;
}
-bool
-CacheRecordPolicy::Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy)
+void
+CacheRecordPolicy::Save(CbWriter& Writer) const
{
- using namespace std::literals;
-
- const uint32_t RecordPolicy = RecordPolicyObject["RecordPolicy"sv].AsUInt32(static_cast<uint32_t>(CachePolicy::Default));
- const uint32_t DefaultPayloadPolicy =
- RecordPolicyObject["DefaultPayloadPolicy"sv].AsUInt32(static_cast<uint32_t>(CachePolicy::Default));
-
- OutRecordPolicy.m_RecordPolicy = static_cast<CachePolicy>(RecordPolicy);
- OutRecordPolicy.m_DefaultPayloadPolicy = static_cast<CachePolicy>(DefaultPayloadPolicy);
-
- for (CbFieldView PayloadPolicyView : RecordPolicyObject["PayloadPolicies"sv])
+ Writer.BeginObject();
{
- CbObjectView PayloadPolicyObject = PayloadPolicyView.AsObjectView();
- const Oid PayloadId = PayloadPolicyObject["Id"sv].AsObjectId();
- const uint32_t PayloadPolicy = PayloadPolicyObject["Policy"sv].AsUInt32();
-
- if (PayloadId != Oid::Zero && PayloadPolicy != 0)
+ // The RecordPolicy is calculated from the ValuePolicies and does not need to be saved separately.
+ Writer << "DefaultValuePolicy"sv << WriteToString<128>(GetDefaultValuePolicy());
+ if (!IsUniform())
{
- OutRecordPolicy.m_PayloadPolicies.emplace(PayloadId, static_cast<CachePolicy>(PayloadPolicy));
+ // FCacheRecordPolicyBuilder guarantees IsUniform -> non-empty GetValuePolicies. Small size penalty here if not.
+ Writer.BeginArray("ValuePolicies"sv);
+ {
+ for (const CacheValuePolicy& ValuePolicy : GetValuePolicies())
+ {
+ // FCacheRecordPolicyBuilder is responsible for ensuring that each ValuePolicy != DefaultValuePolicy
+ // If it lets any duplicates through we will incur a small serialization size penalty here
+ Writer.BeginObject();
+ Writer << "Id"sv << ValuePolicy.Id;
+ Writer << "Policy"sv << WriteToString<128>(ValuePolicy.Policy);
+ Writer.EndObject();
+ }
+ }
+ Writer.EndArray();
}
}
+ Writer.EndObject();
+}
+
+CacheRecordPolicy
+CacheRecordPolicy::Load(CbObjectView Object, CachePolicy DefaultPolicy)
+{
+ std::string_view PolicyText = Object["DefaultValuePolicy"sv].AsString();
+ CachePolicy DefaultValuePolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy;
+
+ CacheRecordPolicyBuilder Builder(DefaultValuePolicy);
+ for (CbFieldView ValueObjectField : Object["ValuePolicies"sv])
+ {
+ CbObjectView ValueObject = ValueObjectField.AsObjectView();
+ const Oid ValueId = ValueObject["Id"sv].AsObjectId();
+ PolicyText = ValueObject["Policy"sv].AsString();
+ CachePolicy ValuePolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultValuePolicy;
+ // FCacheRecordPolicyBuilder should guarantee that FValueId(ValueId).IsValid and ValuePolicy != DefaultValuePolicy
+ // If it lets any through we will have unused data in the record we create.
+ Builder.AddValuePolicy(ValueId, ValuePolicy);
+ }
- return true;
+ return Builder.Build();
}
void
-CacheRecordPolicy::Save(const CacheRecordPolicy& Policy, CbWriter& Writer)
+CacheRecordPolicyBuilder::AddValuePolicy(const CacheValuePolicy& Policy)
{
- Writer << "RecordPolicy"sv << static_cast<uint32_t>(Policy.GetRecordPolicy());
- Writer << "DefaultPayloadPolicy"sv << static_cast<uint32_t>(Policy.GetDefaultPayloadPolicy());
+ if (!Shared)
+ {
+ Shared = new Private::CacheRecordPolicyShared;
+ }
+ Shared->AddValuePolicy(Policy);
+}
- if (!Policy.m_PayloadPolicies.empty())
+CacheRecordPolicy
+CacheRecordPolicyBuilder::Build()
+{
+ CacheRecordPolicy Policy(BasePolicy);
+ if (Shared)
{
- Writer.BeginArray("PayloadPolicies"sv);
- for (const auto& Kv : Policy.m_PayloadPolicies)
+ Shared->Build();
+ const auto PolicyOr = [](CachePolicy A, CachePolicy B) { return A | (B & ~CachePolicy::SkipData); };
+ const std::span<const CacheValuePolicy> Values = Shared->GetValuePolicies();
+ Policy.RecordPolicy = BasePolicy;
+ for (const CacheValuePolicy& ValuePolicy : Values)
{
- Writer.BeginObject();
- Writer.AddObjectId("Id"sv, Kv.first);
- Writer << "Policy"sv << static_cast<uint32_t>(Kv.second);
- Writer.EndObject();
+ Policy.RecordPolicy = PolicyOr(Policy.RecordPolicy, ValuePolicy.Policy);
}
- Writer.EndArray();
+ Policy.Shared = std::move(Shared);
}
+ return Policy;
}
} // namespace zen
diff --git a/zenutil/include/zenutil/cache/cachepolicy.h b/zenutil/include/zenutil/cache/cachepolicy.h
index 5675ccf4d..f967f707b 100644
--- a/zenutil/include/zenutil/cache/cachepolicy.h
+++ b/zenutil/include/zenutil/cache/cachepolicy.h
@@ -2,10 +2,14 @@
#pragma once
+#include <zencore/compactbinary.h>
+#include <zencore/enumflags.h>
+#include <zencore/refcount.h>
#include <zencore/string.h>
#include <zencore/uid.h>
#include <gsl/gsl-lite.hpp>
+#include <span>
#include <unordered_map>
namespace zen {
@@ -34,16 +38,21 @@ enum class CachePolicy : uint32_t
/** Skip fetching the metadata for record requests. */
SkipMeta = 1 << 4,
- /** Skip fetching the value for record, chunk, or value requests. */
- SkipValue = 1 << 5,
- /** Skip fetching the attachments for record requests. */
- SkipAttachments = 1 << 6,
+ /** Skip fetching the data for values. */
+ SkipData = 1 << 5,
+
/**
- * Skip fetching the data for any requests.
+ * Partial output will be provided with the error status when a required value is missing.
+ *
+ * This is meant for cases when the missing values can be individually recovered, or rebuilt,
+ * without rebuilding the whole record. The cache automatically adds this flag when there are
+ * other cache stores that it may be able to recover missing values from.
+ *
+ * Missing values will be returned in the records or chunks, but with only the hash and size.
*
- * Put requests with skip flags may assume that record existence implies payload existence.
+ * Applying this flag for a put of a record allows a partial record to be stored.
*/
- SkipData = SkipMeta | SkipValue | SkipAttachments,
+ PartialRecord = 1 << 6,
/**
* Keep records in the cache for at least the duration of the session.
@@ -53,18 +62,6 @@ enum class CachePolicy : uint32_t
*/
KeepAlive = 1 << 7,
- /**
- * Partial output will be provided with the error status when a required payload is missing.
- *
- * This is meant for cases when the missing payloads can be individually recovered or rebuilt
- * without rebuilding the whole record. The cache automatically adds this flag when there are
- * other cache stores that it may be able to recover missing payloads from.
- *
- * Requests for records would return records where the missing payloads have a hash and size,
- * but no data. Requests for chunks or values would return the hash and size, but no data.
- */
- PartialOnError = 1 << 8,
-
/** Allow cache requests to query and store records and values in local caches. */
Local = QueryLocal | StoreLocal,
/** Allow cache requests to query and store records and values in remote caches. */
@@ -78,35 +75,103 @@ enum class CachePolicy : uint32_t
};
gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy);
+/** Serialize Policy to text and append to Builder. Appended text will not be empty. */
+StringBuilderBase& operator<<(StringBuilderBase& Builder, CachePolicy Policy);
+/** Parse text written by operator<< back into an ECachePolicy. Text must not be empty. */
+CachePolicy ParseCachePolicy(std::string_view Text);
-CachePolicy ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default = CachePolicy::Query);
-
-CachePolicy ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default = CachePolicy::Store);
-
-CachePolicy ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default = CachePolicy::None);
+/** A value ID and the cache policy to use for that value. */
+struct CacheValuePolicy
+{
+ Oid Id;
+ CachePolicy Policy = CachePolicy::Default;
+};
+namespace Private {
+ /** Interface for the private implementation of the cache record policy. */
+ class ICacheRecordPolicyShared : public RefCounted
+ {
+ public:
+ virtual ~ICacheRecordPolicyShared() = default;
+ virtual std::span<const CacheValuePolicy> GetValuePolicies() const = 0;
+ virtual void AddValuePolicy(const CacheValuePolicy& Policy) = 0;
+ virtual void Build() = 0;
+ };
+} // namespace Private
+
+/**
+ * Flags to control the behavior of cache record requests, with optional overrides by value.
+ *
+ * Examples:
+ * - A base policy of Disable, with value policy overrides of Default, will fetch those values if
+ * they exist in the record, and skip data for any other values.
+ * - A base policy of Default, with value policy overrides of (Query | SkipData), will skip those
+ * values, but still check if they exist, and will load any other values.
+ */
class CacheRecordPolicy
{
public:
+ /** Construct a cache record policy that uses the default policy. */
CacheRecordPolicy() = default;
- CacheRecordPolicy(const CachePolicy RecordPolicy, const CachePolicy DefaultPayloadPolicy = CachePolicy::Default);
- CachePolicy GetRecordPolicy() const { return m_RecordPolicy; }
- CachePolicy GetPayloadPolicy(const Oid& PayloadId) const;
- CachePolicy GetDefaultPayloadPolicy() const { return m_DefaultPayloadPolicy; }
+ /** Construct a cache record policy with a uniform policy for the record and every value. */
+ inline CacheRecordPolicy(CachePolicy Policy) : RecordPolicy(Policy), DefaultValuePolicy(Policy) {}
+
+ /** Returns true if the record and every value use the same cache policy. */
+ inline bool IsUniform() const { return !Shared && RecordPolicy == DefaultValuePolicy; }
- bool HasRecordPolicy(const CachePolicy Policy) const { return (m_RecordPolicy & Policy) == Policy; }
- bool HasPayloadPolicy(const Oid& PayloadId, const CachePolicy Policy) const { return (GetPayloadPolicy(PayloadId) & Policy) == Policy; }
+ /** Returns the cache policy to use for the record. */
+ inline CachePolicy GetRecordPolicy() const { return RecordPolicy; }
- static bool Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy);
- static void Save(const CacheRecordPolicy& Policy, CbWriter& Writer);
+ /** Returns the cache policy to use for the value. */
+ CachePolicy GetValuePolicy(const Oid& Id) const;
+
+ /** Returns the cache policy to use for values with no override. */
+ inline CachePolicy GetDefaultValuePolicy() const { return DefaultValuePolicy; }
+
+ /** Returns the array of cache policy overrides for values, sorted by ID. */
+ inline std::span<const CacheValuePolicy> GetValuePolicies() const
+ {
+ return Shared ? Shared->GetValuePolicies() : std::span<const CacheValuePolicy>();
+ }
+
+ /** Save the values from *this into the given writer. */
+ void Save(CbWriter& Writer) const;
+
+ /**
+ * Returns a policy loaded from values on Object.
+ * Invalid data will result in a uniform CacheRecordPolicy with defaultValuePolicy == DefaultPolicy.
+ */
+ static CacheRecordPolicy Load(CbObjectView Object, CachePolicy DefaultPolicy = CachePolicy::Default);
private:
- using PayloadPolicyMap = std::unordered_map<Oid, CachePolicy, Oid::Hasher>;
+ friend class CacheRecordPolicyBuilder;
+
+ CachePolicy RecordPolicy = CachePolicy::Default;
+ CachePolicy DefaultValuePolicy = CachePolicy::Default;
+ RefPtr<const Private::ICacheRecordPolicyShared> Shared;
+};
- CachePolicy m_RecordPolicy = CachePolicy::Default;
- CachePolicy m_DefaultPayloadPolicy = CachePolicy::Default;
- PayloadPolicyMap m_PayloadPolicies;
+/** A cache record policy builder is used to construct a cache record policy. */
+class CacheRecordPolicyBuilder
+{
+public:
+ /** Construct a policy builder that uses the default policy as its base policy. */
+ CacheRecordPolicyBuilder() = default;
+
+ /** Construct a policy builder that uses the provided policy for the record and values with no override. */
+ inline explicit CacheRecordPolicyBuilder(CachePolicy Policy) : BasePolicy(Policy) {}
+
+ /** Adds a cache policy override for a value. */
+ void AddValuePolicy(const CacheValuePolicy& Policy);
+ inline void AddValuePolicy(const Oid& Id, CachePolicy Policy) { AddValuePolicy({Id, Policy}); }
+
+ /** Build a cache record policy, which makes this builder subsequently unusable. */
+ CacheRecordPolicy Build();
+
+private:
+ CachePolicy BasePolicy = CachePolicy::Default;
+ RefPtr<Private::ICacheRecordPolicyShared> Shared;
};
} // namespace zen