aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/cache/structuredcachestore.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/cache/structuredcachestore.cpp')
-rw-r--r--src/zenserver/cache/structuredcachestore.cpp72
1 files changed, 51 insertions, 21 deletions
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 9e14892e3..0a2947b16 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -33,6 +33,7 @@ ZEN_THIRD_PARTY_INCLUDES_START
ZEN_THIRD_PARTY_INCLUDES_END
#if ZEN_WITH_TESTS
+# include <zencore/jobqueue.h>
# include <zencore/testing.h>
# include <zencore/testutils.h>
# include <zencore/workthreadpool.h>
@@ -42,10 +43,15 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
-ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, const std::filesystem::path& RootDir)
+ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc,
+ JobQueue& JobQueue,
+ const std::filesystem::path& RootDir,
+ const ZenCacheMemoryLayer::Configuration MemLayerConfig)
: GcStorage(Gc)
, GcContributor(Gc)
, m_RootDir(RootDir)
+, m_JobQueue(JobQueue)
+, m_MemLayer(m_JobQueue, MemLayerConfig)
, m_DiskLayer(RootDir)
{
ZEN_INFO("initializing structured cache at '{}'", RootDir);
@@ -187,7 +193,7 @@ ZenCacheNamespace::CollectGarbage(GcContext& GcCtx)
{
ZEN_TRACE_CPU("Z$::Namespace::CollectGarbage");
- m_MemLayer.Reset();
+ (void)m_MemLayer.CollectGarbage(GcCtx.CacheExpireTime());
m_DiskLayer.CollectGarbage(GcCtx);
}
@@ -252,10 +258,14 @@ ZEN_DEFINE_LOG_CATEGORY_STATIC(LogCacheActivity, "z$");
static constinit std::string_view UE4DDCNamespaceName = "ue4.ddc";
-ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration, const DiskWriteBlocker* InDiskWriteBlocker)
+ZenCacheStore::ZenCacheStore(GcManager& Gc,
+ JobQueue& JobQueue,
+ const Configuration& Configuration,
+ const DiskWriteBlocker* InDiskWriteBlocker)
: m_Log(logging::Get("z$"))
, m_DiskWriteBlocker(InDiskWriteBlocker)
, m_Gc(Gc)
+, m_JobQueue(JobQueue)
, m_Configuration(Configuration)
, m_ExitLogging(false)
{
@@ -293,7 +303,10 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration,
for (const std::string& NamespaceName : Namespaces)
{
m_Namespaces[NamespaceName] =
- std::make_unique<ZenCacheNamespace>(Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName));
+ std::make_unique<ZenCacheNamespace>(Gc,
+ m_JobQueue,
+ m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName),
+ m_Configuration.MemLayerConfig);
}
}
@@ -577,7 +590,10 @@ ZenCacheStore::GetNamespace(std::string_view Namespace)
auto NewNamespace = m_Namespaces.insert_or_assign(
std::string(Namespace),
- std::make_unique<ZenCacheNamespace>(m_Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace)));
+ std::make_unique<ZenCacheNamespace>(m_Gc,
+ m_JobQueue,
+ m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace),
+ m_Configuration.MemLayerConfig));
return NewNamespace.first->second.get();
}
@@ -752,7 +768,8 @@ TEST_CASE("z$.store")
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
const int kIterationCount = 100;
@@ -788,6 +805,8 @@ TEST_CASE("z$.store")
TEST_CASE("z$.size")
{
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+
const auto CreateCacheValue = [](size_t Size) -> CbObject {
std::vector<uint8_t> Buf;
Buf.resize(Size);
@@ -806,7 +825,7 @@ TEST_CASE("z$.size")
{
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256);
@@ -826,7 +845,7 @@ TEST_CASE("z$.size")
{
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
const GcStorageSize SerializedSize = Zcs.StorageSize();
CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -849,7 +868,7 @@ TEST_CASE("z$.size")
{
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64);
@@ -869,7 +888,7 @@ TEST_CASE("z$.size")
{
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
const GcStorageSize SerializedSize = Zcs.StorageSize();
CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -888,6 +907,8 @@ TEST_CASE("z$.gc")
{
using namespace testutils;
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+
SUBCASE("gather references does NOT add references for expired cache entries")
{
ScopedTemporaryDirectory TempDir;
@@ -906,7 +927,7 @@ TEST_CASE("z$.gc")
{
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
const auto Bucket = "teardrinker"sv;
// Create a cache record
@@ -943,7 +964,7 @@ TEST_CASE("z$.gc")
// Expect timestamps to be serialized
{
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
std::vector<IoHash> Keep;
// Collect garbage with 1 hour max cache duration
@@ -964,7 +985,7 @@ TEST_CASE("z$.gc")
{
ScopedTemporaryDirectory TempDir;
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
const auto Bucket = "fortysixandtwo"sv;
const GcClock::TimePoint CurrentTime = GcClock::Now();
@@ -1009,7 +1030,7 @@ TEST_CASE("z$.gc")
{
ScopedTemporaryDirectory TempDir;
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
const auto Bucket = "rightintwo"sv;
std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
@@ -1104,7 +1125,8 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true))
WorkerThreadPool ThreadPool(4);
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path());
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path());
{
std::atomic<size_t> WorkCompleted = 0;
@@ -1313,6 +1335,8 @@ TEST_CASE("z$.namespaces")
return Writer.Save();
};
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+
ScopedTemporaryDirectory TempDir;
CreateDirectories(TempDir.Path());
@@ -1321,7 +1345,7 @@ TEST_CASE("z$.namespaces")
IoHash Key2;
{
GcManager Gc;
- ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = false}, nullptr);
+ ZenCacheStore Zcs(Gc, *JobQueue, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = false}, nullptr);
const auto Bucket = "teardrinker"sv;
const auto CustomNamespace = "mynamespace"sv;
@@ -1346,7 +1370,7 @@ TEST_CASE("z$.namespaces")
{
GcManager Gc;
- ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}, nullptr);
+ ZenCacheStore Zcs(Gc, *JobQueue, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}, nullptr);
const auto Bucket = "teardrinker"sv;
const auto CustomNamespace = "mynamespace"sv;
@@ -1379,6 +1403,8 @@ TEST_CASE("z$.drop.bucket")
return Writer.Save();
};
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+
ScopedTemporaryDirectory TempDir;
CreateDirectories(TempDir.Path());
@@ -1407,7 +1433,7 @@ TEST_CASE("z$.drop.bucket")
WorkerThreadPool Workers(1);
{
GcManager Gc;
- ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}, nullptr);
+ ZenCacheStore Zcs(Gc, *JobQueue, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}, nullptr);
const auto Bucket = "teardrinker"sv;
const auto Namespace = "mynamespace"sv;
@@ -1454,6 +1480,8 @@ TEST_CASE("z$.drop.namespace")
return Writer.Save();
};
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+
ScopedTemporaryDirectory TempDir;
CreateDirectories(TempDir.Path());
@@ -1478,7 +1506,7 @@ TEST_CASE("z$.drop.namespace")
WorkerThreadPool Workers(1);
{
GcManager Gc;
- ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}, nullptr);
+ ZenCacheStore Zcs(Gc, *JobQueue, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}, nullptr);
const auto Bucket1 = "teardrinker1"sv;
const auto Bucket2 = "teardrinker2"sv;
const auto Namespace1 = "mynamespace1"sv;
@@ -1543,7 +1571,8 @@ TEST_CASE("z$.blocked.disklayer.put")
};
GcManager Gc;
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
CbObject CacheValue = CreateCacheValue(64 * 1024 + 64);
@@ -1637,7 +1666,8 @@ TEST_CASE("z$.scrub")
GcManager Gc;
CidStore CidStore(Gc);
- ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ auto JobQueue = MakeJobQueue(1, "testqueue");
+ ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
CidStore.Initialize(CidConfig);
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-12 07:24:59 +0000