aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-05-04 19:45:57 +0200
committerDan Engelbrecht <[email protected]>2022-05-04 19:55:07 +0200
commitef12415d287c9307c0c4774aeacff6c91966f693 (patch)
tree34f436c3108249b28b120abd6b2346ac2e6251f5
parentdefault namespace fix (diff)
downloadzen-ef12415d287c9307c0c4774aeacff6c91966f693.tar.xz
zen-ef12415d287c9307c0c4774aeacff6c91966f693.zip
cleanup
-rw-r--r--zenserver/cache/namespacecachestore.cpp208
-rw-r--r--zenserver/cache/namespacecachestore.h49
-rw-r--r--zenserver/cache/structuredcache.cpp13
-rw-r--r--zenserver/cache/structuredcache.h14
-rw-r--r--zenserver/cache/structuredcachestore.cpp256
-rw-r--r--zenserver/cache/structuredcachestore.h33
-rw-r--r--zenserver/upstream/upstreamapply.h2
-rw-r--r--zenserver/upstream/upstreamcache.cpp7
-rw-r--r--zenserver/upstream/upstreamcache.h4
-rw-r--r--zenserver/zenserver.cpp5
10 files changed, 274 insertions, 317 deletions
diff --git a/zenserver/cache/namespacecachestore.cpp b/zenserver/cache/namespacecachestore.cpp
deleted file mode 100644
index 1cf76b0ae..000000000
--- a/zenserver/cache/namespacecachestore.cpp
+++ /dev/null
@@ -1,208 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include "namespacecachestore.h"
-#include "structuredcachestore.h"
-
-namespace zen {
-
-const char* NamespaceDirPrefix = "ns_";
-
-NamespaceCacheStore::NamespaceCacheStore(std::filesystem::path BasePath, CasGc& Gc)
-: GcStorage(Gc)
-, GcContributor(Gc)
-, m_Log(logging::Get("namespacecachestore"))
-, m_BasePath(BasePath)
-{
- CreateDirectories(m_BasePath);
- std::vector<std::string> ExistingFolders = FindExistingFolders();
-
- std::vector<std::string> LegacyBuckets;
- std::vector<std::string> Namespaces;
- for (const std::string& DirName : ExistingFolders)
- {
- if (DirName.starts_with(NamespaceDirPrefix))
- {
- Namespaces.push_back(DirName.substr(3));
- continue;
- }
- LegacyBuckets.push_back(DirName);
- }
-
- if (std::find(Namespaces.begin(), Namespaces.end(), "") == Namespaces.end())
- {
- std::filesystem::path DefaultNamespaceFolder = m_BasePath / NamespaceDirPrefix;
- CreateDirectories(DefaultNamespaceFolder);
-
- // Move any non-namespace folders into the default namespace folder
- for (const std::string& DirName : LegacyBuckets)
- {
- std::filesystem::path LegacyFolder = m_BasePath / DirName;
- std::filesystem::path NewPath = DefaultNamespaceFolder / DirName;
- std::filesystem::rename(LegacyFolder, NewPath);
- }
- Namespaces.push_back("");
- }
-
- for (const std::string& NamespaceName : Namespaces)
- {
- ZenCacheStore* Store = new ZenCacheStore(Gc, m_BasePath / (NamespaceDirPrefix + NamespaceName));
- m_Namespaces[NamespaceName] = Store;
- }
-}
-
-NamespaceCacheStore::~NamespaceCacheStore()
-{
- for (const auto& Entry : m_Namespaces)
- {
- delete Entry.second;
- }
- m_Namespaces.clear();
-}
-
-std::vector<std::string>
-NamespaceCacheStore::FindExistingFolders() const
-{
- FileSystemTraversal Traversal;
- struct Visitor : public FileSystemTraversal::TreeVisitor
- {
- virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
- [[maybe_unused]] const path_view& File,
- [[maybe_unused]] uint64_t FileSize) override
- {
- }
-
- virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
- {
- std::string DirName8 = WideToUtf8(DirectoryName);
- Dirs.push_back(DirName8);
- return false;
- }
-
- std::vector<std::string> Dirs;
- } Visit;
-
- Traversal.TraverseFileSystem(m_BasePath, Visit);
- return Visit.Dirs;
-}
-
-bool
-NamespaceCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue)
-{
- ZenCacheStore* Store = GetStore(Namespace);
- if (!Store)
- {
- return false;
- }
- return Store->Get(Bucket, HashKey, OutValue);
-}
-
-void
-NamespaceCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value)
-{
- ZenCacheStore* Store = GetStore(Namespace);
- if (!Store)
- {
- return;
- }
- Store->Put(Bucket, HashKey, Value);
-}
-
-bool
-NamespaceCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket)
-{
- ZenCacheStore* Store = GetStore(Namespace);
- if (!Store)
- {
- return false;
- }
- return Store->DropBucket(Bucket);
-}
-
-void
-NamespaceCacheStore::Flush()
-{
- std::vector<ZenCacheStore*> Stores;
- RwLock::SharedLockScope _(m_NamespacesLock);
- Stores.reserve(m_Namespaces.size());
- for (const auto& Entry : m_Namespaces)
- {
- Stores.push_back(Entry.second);
- }
- _.ReleaseNow();
- for (ZenCacheStore* Store : Stores)
- {
- Store->Flush();
- }
-}
-
-void
-NamespaceCacheStore::Scrub(ScrubContext& Ctx)
-{
- std::vector<ZenCacheStore*> Stores = GetAllStores();
- for (ZenCacheStore* Store : Stores)
- {
- Store->Scrub(Ctx);
- }
-}
-
-ZenCacheStore*
-NamespaceCacheStore::GetStore(const std::string& Namespace)
-{
- RwLock::SharedLockScope _(m_NamespacesLock);
- if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end())
- {
- return It->second;
- }
- return nullptr;
-}
-
-std::vector<ZenCacheStore*>
-NamespaceCacheStore::GetAllStores() const
-{
- std::vector<ZenCacheStore*> Stores;
- RwLock::SharedLockScope _(m_NamespacesLock);
- Stores.reserve(m_Namespaces.size());
- for (const auto& Entry : m_Namespaces)
- {
- Stores.push_back(Entry.second);
- }
- return Stores;
-}
-
-void
-NamespaceCacheStore::GatherReferences(GcContext& GcCtx)
-{
- std::vector<ZenCacheStore*> Stores = GetAllStores();
- for (ZenCacheStore* Store : Stores)
- {
- Store->GatherReferences(GcCtx);
- }
-}
-
-void
-NamespaceCacheStore::CollectGarbage(GcContext& GcCtx)
-{
- std::vector<ZenCacheStore*> Stores = GetAllStores();
- for (ZenCacheStore* Store : Stores)
- {
- Store->CollectGarbage(GcCtx);
- }
-}
-
-GcStorageSize
-NamespaceCacheStore::StorageSize() const
-{
- std::vector<ZenCacheStore*> Stores = GetAllStores();
- GcStorageSize Size;
- for (ZenCacheStore* Store : Stores)
- {
- GcStorageSize StoreSize = Store->StorageSize();
- Size.MemorySize += StoreSize.MemorySize;
- Size.DiskSize += StoreSize.DiskSize;
- }
- return Size;
-}
-
-} // namespace zen
diff --git a/zenserver/cache/namespacecachestore.h b/zenserver/cache/namespacecachestore.h
deleted file mode 100644
index 85f4150bf..000000000
--- a/zenserver/cache/namespacecachestore.h
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/logging.h>
-#include <zencore/refcount.h>
-
-#include <zenstore/gc.h>
-
-#include <string>
-
-namespace zen {
-
-class ScrubContext;
-class ZenCacheStore;
-struct ZenCacheValue;
-
-/** NamespaceCache Store
- */
-class NamespaceCacheStore : public RefCounted, public GcStorage, public GcContributor
-{
-public:
- NamespaceCacheStore(std::filesystem::path BasePath, CasGc& Gc);
- ~NamespaceCacheStore();
-
- bool Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
- bool DropBucket(const std::string& Namespace, std::string_view Bucket);
- void Flush();
- void Scrub(ScrubContext& Ctx);
-
- spdlog::logger& Log() { return m_Log; }
- const std::filesystem::path& BasePath() const { return m_BasePath; }
-
- virtual void GatherReferences(GcContext& GcCtx) override;
- virtual void CollectGarbage(GcContext& GcCtx) override;
- virtual GcStorageSize StorageSize() const override;
-
-private:
- std::vector<std::string> FindExistingFolders() const;
- ZenCacheStore* GetStore(const std::string& Namespace);
- std::vector<ZenCacheStore*> GetAllStores() const;
- spdlog::logger& m_Log;
- std::filesystem::path m_BasePath;
- mutable RwLock m_NamespacesLock;
- std::unordered_map<std::string, ZenCacheStore*> m_Namespaces;
-};
-
-} // namespace zen
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 276c99081..8deb958be 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -21,7 +21,6 @@
//#include "cachekey.h"
#include "monitoring/httpstats.h"
-#include "namespacecachestore.h"
#include "structuredcachestore.h"
#include "upstream/jupiter.h"
#include "upstream/upstreamcache.h"
@@ -73,13 +72,13 @@ struct PutRequestData
//////////////////////////////////////////////////////////////////////////
-HttpStructuredCacheService::HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore,
- CidStore& InCidStore,
- HttpStatsService& StatsService,
- HttpStatusService& StatusService,
- UpstreamCache& UpstreamCache)
+HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& CacheStore,
+ CidStore& InCidStore,
+ HttpStatsService& StatsService,
+ HttpStatusService& StatusService,
+ UpstreamCache& UpstreamCache)
: m_Log(logging::Get("cache"))
-, m_CacheStore(InNamespaceCacheStore)
+, m_CacheStore(CacheStore)
, m_StatsService(StatsService)
, m_StatusService(StatusService)
, m_CidStore(InCidStore)
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index 7e9847838..c41afef12 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -25,7 +25,7 @@ class CbObjectView;
struct PutRequestData;
class ScrubContext;
class UpstreamCache;
-class NamespaceCacheStore;
+class ZenCacheStore;
enum class CachePolicy : uint32_t;
namespace cache::detail {
@@ -64,11 +64,11 @@ namespace cache::detail {
class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider
{
public:
- HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore,
- CidStore& InCidStore,
- HttpStatsService& StatsService,
- HttpStatusService& StatusService,
- UpstreamCache& UpstreamCache);
+ HttpStructuredCacheService(ZenCacheStore& CacheStore,
+ CidStore& InCidStore,
+ HttpStatsService& StatsService,
+ HttpStatusService& StatusService,
+ UpstreamCache& UpstreamCache);
~HttpStructuredCacheService();
virtual const char* BaseUri() const override;
@@ -140,7 +140,7 @@ private:
spdlog::logger& Log() { return m_Log; }
spdlog::logger& m_Log;
- NamespaceCacheStore& m_CacheStore;
+ ZenCacheStore& m_CacheStore;
HttpStatsService& m_StatsService;
HttpStatusService& m_StatusService;
CidStore& m_CidStore;
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 2869191fd..075b7d408 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -232,7 +232,7 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
WriteFile(Path, Object.GetBuffer().AsIoBuffer());
}
-ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
+ZenCacheNamespace::ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir)
: GcStorage(Gc)
, GcContributor(Gc)
, m_RootDir(RootDir)
@@ -248,12 +248,12 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
#endif
}
-ZenCacheStore::~ZenCacheStore()
+ZenCacheNamespace::~ZenCacheNamespace()
{
}
bool
-ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
{
ZEN_TRACE_CPU("Z$::Get");
@@ -291,7 +291,7 @@ ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheVal
}
void
-ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
{
ZEN_TRACE_CPU("Z$::Put");
@@ -327,7 +327,7 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
}
bool
-ZenCacheStore::DropBucket(std::string_view Bucket)
+ZenCacheNamespace::DropBucket(std::string_view Bucket)
{
ZEN_INFO("dropping bucket '{}'", Bucket);
@@ -343,13 +343,13 @@ ZenCacheStore::DropBucket(std::string_view Bucket)
}
void
-ZenCacheStore::Flush()
+ZenCacheNamespace::Flush()
{
m_DiskLayer.Flush();
}
void
-ZenCacheStore::Scrub(ScrubContext& Ctx)
+ZenCacheNamespace::Scrub(ScrubContext& Ctx)
{
if (m_LastScrubTime == Ctx.ScrubTimestamp())
{
@@ -363,7 +363,7 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheStore::GatherReferences(GcContext& GcCtx)
+ZenCacheNamespace::GatherReferences(GcContext& GcCtx)
{
Stopwatch Timer;
const auto Guard =
@@ -377,14 +377,14 @@ ZenCacheStore::GatherReferences(GcContext& GcCtx)
}
void
-ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+ZenCacheNamespace::CollectGarbage(GcContext& GcCtx)
{
m_MemLayer.Reset();
m_DiskLayer.CollectGarbage(GcCtx);
}
GcStorageSize
-ZenCacheStore::StorageSize() const
+ZenCacheNamespace::StorageSize() const
{
return {.DiskSize = m_DiskLayer.TotalSize(), .MemorySize = m_MemLayer.TotalSize()};
}
@@ -2098,6 +2098,200 @@ ZenCacheDiskLayer::TotalSize() const
return TotalSize;
}
+//////////////////////////// ZenCacheStore
+
+const char* ZenCacheNamespaceDirPrefix = "ns_";
+
+namespace {
+
+ std::vector<std::string> FindExistingFolders(const std::filesystem::path& RootPath)
+ {
+ FileSystemTraversal Traversal;
+ struct Visitor : public FileSystemTraversal::TreeVisitor
+ {
+ virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override {}
+
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view& DirectoryName) override
+ {
+ std::string DirName8 = WideToUtf8(DirectoryName);
+ Dirs.push_back(DirName8);
+ return false;
+ }
+
+ std::vector<std::string> Dirs;
+ } Visit;
+
+ Traversal.TraverseFileSystem(RootPath, Visit);
+ return Visit.Dirs;
+ }
+
+} // namespace
+
+ZenCacheStore::ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc) : GcStorage(Gc), GcContributor(Gc)
+{
+ CreateDirectories(BasePath);
+ std::vector<std::string> ExistingFolders = FindExistingFolders(BasePath);
+
+ std::vector<std::string> LegacyBuckets;
+ std::vector<std::string> Namespaces;
+ for (const std::string& DirName : ExistingFolders)
+ {
+ if (DirName.starts_with(ZenCacheNamespaceDirPrefix))
+ {
+ Namespaces.push_back(DirName.substr(3));
+ continue;
+ }
+ LegacyBuckets.push_back(DirName);
+ }
+
+ ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size());
+
+ if (std::find(Namespaces.begin(), Namespaces.end(), "") == Namespaces.end())
+ {
+ ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size());
+ std::filesystem::path DefaultNamespaceFolder = BasePath / ZenCacheNamespaceDirPrefix;
+ CreateDirectories(DefaultNamespaceFolder);
+
+ // Move any non-namespace folders into the default namespace folder
+ for (const std::string& DirName : LegacyBuckets)
+ {
+ std::filesystem::path LegacyFolder = BasePath / DirName;
+ std::filesystem::path NewPath = DefaultNamespaceFolder / DirName;
+ std::filesystem::rename(LegacyFolder, NewPath);
+ }
+ Namespaces.push_back("");
+ }
+
+ for (const std::string& NamespaceName : Namespaces)
+ {
+ Ref<ZenCacheNamespace> Store = new ZenCacheNamespace(Gc, BasePath / (ZenCacheNamespaceDirPrefix + NamespaceName));
+ m_Namespaces[NamespaceName] = Store;
+ }
+}
+
+ZenCacheStore::~ZenCacheStore()
+{
+ m_Namespaces.clear();
+}
+
+bool
+ZenCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+ Ref<ZenCacheNamespace> Store = GetStore(Namespace);
+ if (!Store)
+ {
+ return false;
+ }
+ return Store->Get(Bucket, HashKey, OutValue);
+}
+
+void
+ZenCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value)
+{
+ Ref<ZenCacheNamespace> Store = GetStore(Namespace);
+ if (!Store)
+ {
+ return;
+ }
+ Store->Put(Bucket, HashKey, Value);
+}
+
+bool
+ZenCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket)
+{
+ Ref<ZenCacheNamespace> Store = GetStore(Namespace);
+ if (!Store)
+ {
+ return false;
+ }
+ return Store->DropBucket(Bucket);
+}
+
+void
+ZenCacheStore::Flush()
+{
+ std::vector<Ref<ZenCacheNamespace>> Stores;
+ RwLock::SharedLockScope _(m_NamespacesLock);
+ Stores.reserve(m_Namespaces.size());
+ for (const auto& Entry : m_Namespaces)
+ {
+ Stores.push_back(Entry.second);
+ }
+ _.ReleaseNow();
+ for (const Ref<ZenCacheNamespace>& Store : Stores)
+ {
+ Store->Flush();
+ }
+}
+
+void
+ZenCacheStore::Scrub(ScrubContext& Ctx)
+{
+ std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+ for (const Ref<ZenCacheNamespace>& Store : Stores)
+ {
+ Store->Scrub(Ctx);
+ }
+}
+
+Ref<ZenCacheNamespace>
+ZenCacheStore::GetStore(const std::string& Namespace)
+{
+ RwLock::SharedLockScope _(m_NamespacesLock);
+ if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end())
+ {
+ return It->second;
+ }
+ return nullptr;
+}
+
+std::vector<Ref<ZenCacheNamespace>>
+ZenCacheStore::GetAllStores() const
+{
+ std::vector<Ref<ZenCacheNamespace>> Stores;
+ RwLock::SharedLockScope _(m_NamespacesLock);
+ Stores.reserve(m_Namespaces.size());
+ for (const auto& Entry : m_Namespaces)
+ {
+ Stores.push_back(Entry.second);
+ }
+ return Stores;
+}
+
+void
+ZenCacheStore::GatherReferences(GcContext& GcCtx)
+{
+ std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+ for (const Ref<ZenCacheNamespace>& Store : Stores)
+ {
+ Store->GatherReferences(GcCtx);
+ }
+}
+
+void
+ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+{
+ std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+ for (const Ref<ZenCacheNamespace>& Store : Stores)
+ {
+ Store->CollectGarbage(GcCtx);
+ }
+}
+
+GcStorageSize
+ZenCacheStore::StorageSize() const
+{
+ std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+ GcStorageSize Size;
+ for (const Ref<ZenCacheNamespace>& Store : Stores)
+ {
+ GcStorageSize StoreSize = Store->StorageSize();
+ Size.MemorySize += StoreSize.MemorySize;
+ Size.DiskSize += StoreSize.DiskSize;
+ }
+ return Size;
+}
+
//////////////////////////////////////////////////////////////////////////
#if ZEN_WITH_TESTS
@@ -2136,7 +2330,7 @@ TEST_CASE("z$.store")
CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
const int kIterationCount = 100;
@@ -2189,8 +2383,8 @@ TEST_CASE("z$.size")
GcStorageSize CacheSize;
{
- CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ CasGc Gc;
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256);
@@ -2209,8 +2403,8 @@ TEST_CASE("z$.size")
}
{
- CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ CasGc Gc;
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
const GcStorageSize SerializedSize = Zcs.StorageSize();
CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2232,8 +2426,8 @@ TEST_CASE("z$.size")
GcStorageSize CacheSize;
{
- CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ CasGc Gc;
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64);
@@ -2252,8 +2446,8 @@ TEST_CASE("z$.size")
}
{
- CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ CasGc Gc;
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
const GcStorageSize SerializedSize = Zcs.StorageSize();
CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2290,9 +2484,9 @@ TEST_CASE("z$.gc")
};
{
- CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
- const auto Bucket = "teardrinker"sv;
+ CasGc Gc;
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+ const auto Bucket = "teardrinker"sv;
// Create a cache record
const IoHash Key = CreateKey(42);
@@ -2328,7 +2522,7 @@ TEST_CASE("z$.gc")
// Expect timestamps to be serialized
{
CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
std::vector<IoHash> Keep;
// Collect garbage with 1 hour max cache duration
@@ -2349,7 +2543,7 @@ TEST_CASE("z$.gc")
{
ScopedTemporaryDirectory TempDir;
CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
const auto Bucket = "fortysixandtwo"sv;
const GcClock::TimePoint CurrentTime = GcClock::Now();
@@ -2397,7 +2591,7 @@ TEST_CASE("z$.gc")
{
ScopedTemporaryDirectory TempDir;
CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
const auto Bucket = "rightintwo"sv;
const GcClock::TimePoint CurrentTime = GcClock::Now();
@@ -2490,7 +2684,7 @@ TEST_CASE("z$.legacyconversion")
const std::string Bucket = "rightintwo";
{
CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path());
+ ZenCacheNamespace Zcs(Gc, TempDir.Path());
const GcClock::TimePoint CurrentTime = GcClock::Now();
for (size_t i = 0; i < ChunkCount; i++)
@@ -2578,8 +2772,8 @@ TEST_CASE("z$.legacyconversion")
std::filesystem::remove(IndexPath);
{
- CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path());
+ CasGc Gc;
+ ZenCacheNamespace Zcs(Gc, TempDir.Path());
for (size_t i = 0; i < ChunkCount; i += 2)
{
@@ -2639,9 +2833,9 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true))
CreateDirectories(TempDir.Path());
- WorkerThreadPool ThreadPool(4);
- CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path());
+ WorkerThreadPool ThreadPool(4);
+ CasGc Gc;
+ ZenCacheNamespace Zcs(Gc, TempDir.Path());
{
std::atomic<size_t> WorkCompleted = 0;
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 0c2a7c0b2..a803b0603 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -322,11 +322,11 @@ private:
ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
};
-class ZenCacheStore final : public GcStorage, public GcContributor
+class ZenCacheNamespace final : public RefCounted, public GcStorage, public GcContributor
{
public:
- ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir);
- ~ZenCacheStore();
+ ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir);
+ ~ZenCacheNamespace();
bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
@@ -349,8 +349,31 @@ private:
std::unique_ptr<ZenCacheTracker> m_AccessTracker;
#endif
- ZenCacheStore(const ZenCacheStore&) = delete;
- ZenCacheStore& operator=(const ZenCacheStore&) = delete;
+ ZenCacheNamespace(const ZenCacheNamespace&) = delete;
+ ZenCacheNamespace& operator=(const ZenCacheNamespace&) = delete;
+};
+
+class ZenCacheStore final : public GcStorage, public GcContributor
+{
+public:
+ ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc);
+ ~ZenCacheStore();
+
+ bool Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool DropBucket(const std::string& Namespace, std::string_view Bucket);
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+
+ virtual void GatherReferences(GcContext& GcCtx) override;
+ virtual void CollectGarbage(GcContext& GcCtx) override;
+ virtual GcStorageSize StorageSize() const override;
+
+private:
+ Ref<ZenCacheNamespace> GetStore(const std::string& Namespace);
+ std::vector<Ref<ZenCacheNamespace>> GetAllStores() const;
+ mutable RwLock m_NamespacesLock;
+ std::unordered_map<std::string, Ref<ZenCacheNamespace>> m_Namespaces;
};
void z$_forcelink();
diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h
index 2edc6dc49..1deaf00a5 100644
--- a/zenserver/upstream/upstreamapply.h
+++ b/zenserver/upstream/upstreamapply.h
@@ -23,7 +23,7 @@ class CbObjectWriter;
class CidStore;
class CloudCacheTokenProvider;
class WorkerThreadPool;
-class ZenCacheStore;
+class ZenCacheNamespace;
struct CloudCacheClientOptions;
struct UpstreamAuthConfig;
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index 49f384774..c89227106 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -19,7 +19,6 @@
#include <zenstore/cidstore.h>
#include <auth/authmgr.h>
-#include "cache/namespacecachestore.h"
#include "cache/structuredcache.h"
#include "cache/structuredcachestore.h"
#include "diag/logging.h"
@@ -1174,7 +1173,7 @@ namespace detail {
class UpstreamCacheImpl final : public UpstreamCache
{
public:
- UpstreamCacheImpl(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore)
+ UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
: m_Log(logging::Get("upstream"))
, m_Options(Options)
, m_CacheStore(CacheStore)
@@ -1688,7 +1687,7 @@ private:
spdlog::logger& m_Log;
UpstreamCacheOptions m_Options;
- NamespaceCacheStore& m_CacheStore;
+ ZenCacheStore& m_CacheStore;
CidStore& m_CidStore;
UpstreamQueue m_UpstreamQueue;
std::shared_mutex m_EndpointsMutex;
@@ -1713,7 +1712,7 @@ UpstreamEndpoint::CreateJupiterEndpoint(const CloudCacheClientOptions& Options,
}
std::unique_ptr<UpstreamCache>
-UpstreamCache::Create(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore)
+UpstreamCache::Create(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
{
return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore);
}
diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h
index 54386e80d..6f18b3119 100644
--- a/zenserver/upstream/upstreamcache.h
+++ b/zenserver/upstream/upstreamcache.h
@@ -24,7 +24,7 @@ class CbObjectView;
class CbPackage;
class CbObjectWriter;
class CidStore;
-class NamespaceCacheStore;
+class ZenCacheStore;
struct CloudCacheClientOptions;
class CloudCacheTokenProvider;
struct ZenStructuredCacheClientOptions;
@@ -206,7 +206,7 @@ public:
virtual void GetStatus(CbObjectWriter& CbO) = 0;
- static std::unique_ptr<UpstreamCache> Create(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore);
+ static std::unique_ptr<UpstreamCache> Create(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore);
};
} // namespace zen
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 9b7083312..e572e6a49 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -102,7 +102,6 @@ ZEN_THIRD_PARTY_INCLUDES_END
#include "admin/admin.h"
#include "auth/authmgr.h"
#include "auth/authservice.h"
-#include "cache/namespacecachestore.h"
#include "cache/structuredcache.h"
#include "cache/structuredcachestore.h"
#include "compute/function.h"
@@ -612,7 +611,7 @@ private:
zen::GcScheduler m_GcScheduler{m_CasGc};
std::unique_ptr<zen::CasStore> m_CasStore{zen::CreateCasStore(m_CasGc)};
std::unique_ptr<zen::CidStore> m_CidStore;
- std::unique_ptr<zen::NamespaceCacheStore> m_CacheStore;
+ std::unique_ptr<zen::ZenCacheStore> m_CacheStore;
zen::CasScrubber m_Scrubber{*m_CasStore};
zen::HttpTestService m_TestService;
zen::HttpTestingService m_TestingService;
@@ -756,7 +755,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
using namespace std::literals;
ZEN_INFO("instantiating structured cache service");
- m_CacheStore = std::make_unique<NamespaceCacheStore>(m_DataRoot / "cache", m_CasGc);
+ m_CacheStore = std::make_unique<ZenCacheStore>(m_DataRoot / "cache", m_CasGc);
const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig;