diff options
| author | Dan Engelbrecht <[email protected]> | 2022-05-04 19:45:57 +0200 |
|---|---|---|
| committer | Dan Engelbrecht <[email protected]> | 2022-05-04 19:55:07 +0200 |
| commit | ef12415d287c9307c0c4774aeacff6c91966f693 (patch) | |
| tree | 34f436c3108249b28b120abd6b2346ac2e6251f5 | |
| parent | default namespace fix (diff) | |
| download | zen-ef12415d287c9307c0c4774aeacff6c91966f693.tar.xz zen-ef12415d287c9307c0c4774aeacff6c91966f693.zip | |
cleanup
| -rw-r--r-- | zenserver/cache/namespacecachestore.cpp | 208 | ||||
| -rw-r--r-- | zenserver/cache/namespacecachestore.h | 49 | ||||
| -rw-r--r-- | zenserver/cache/structuredcache.cpp | 13 | ||||
| -rw-r--r-- | zenserver/cache/structuredcache.h | 14 | ||||
| -rw-r--r-- | zenserver/cache/structuredcachestore.cpp | 256 | ||||
| -rw-r--r-- | zenserver/cache/structuredcachestore.h | 33 | ||||
| -rw-r--r-- | zenserver/upstream/upstreamapply.h | 2 | ||||
| -rw-r--r-- | zenserver/upstream/upstreamcache.cpp | 7 | ||||
| -rw-r--r-- | zenserver/upstream/upstreamcache.h | 4 | ||||
| -rw-r--r-- | zenserver/zenserver.cpp | 5 |
10 files changed, 274 insertions, 317 deletions
diff --git a/zenserver/cache/namespacecachestore.cpp b/zenserver/cache/namespacecachestore.cpp deleted file mode 100644 index 1cf76b0ae..000000000 --- a/zenserver/cache/namespacecachestore.cpp +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "namespacecachestore.h" -#include "structuredcachestore.h" - -namespace zen { - -const char* NamespaceDirPrefix = "ns_"; - -NamespaceCacheStore::NamespaceCacheStore(std::filesystem::path BasePath, CasGc& Gc) -: GcStorage(Gc) -, GcContributor(Gc) -, m_Log(logging::Get("namespacecachestore")) -, m_BasePath(BasePath) -{ - CreateDirectories(m_BasePath); - std::vector<std::string> ExistingFolders = FindExistingFolders(); - - std::vector<std::string> LegacyBuckets; - std::vector<std::string> Namespaces; - for (const std::string& DirName : ExistingFolders) - { - if (DirName.starts_with(NamespaceDirPrefix)) - { - Namespaces.push_back(DirName.substr(3)); - continue; - } - LegacyBuckets.push_back(DirName); - } - - if (std::find(Namespaces.begin(), Namespaces.end(), "") == Namespaces.end()) - { - std::filesystem::path DefaultNamespaceFolder = m_BasePath / NamespaceDirPrefix; - CreateDirectories(DefaultNamespaceFolder); - - // Move any non-namespace folders into the default namespace folder - for (const std::string& DirName : LegacyBuckets) - { - std::filesystem::path LegacyFolder = m_BasePath / DirName; - std::filesystem::path NewPath = DefaultNamespaceFolder / DirName; - std::filesystem::rename(LegacyFolder, NewPath); - } - Namespaces.push_back(""); - } - - for (const std::string& NamespaceName : Namespaces) - { - ZenCacheStore* Store = new ZenCacheStore(Gc, m_BasePath / (NamespaceDirPrefix + NamespaceName)); - m_Namespaces[NamespaceName] = Store; - } -} - -NamespaceCacheStore::~NamespaceCacheStore() -{ - for (const auto& Entry : m_Namespaces) - { - delete Entry.second; - } - m_Namespaces.clear(); -} - -std::vector<std::string> -NamespaceCacheStore::FindExistingFolders() const -{ - FileSystemTraversal Traversal; - struct Visitor : public FileSystemTraversal::TreeVisitor - { - virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent, - [[maybe_unused]] const path_view& File, - [[maybe_unused]] uint64_t FileSize) override - { - } - - virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override - { - std::string DirName8 = WideToUtf8(DirectoryName); - Dirs.push_back(DirName8); - return false; - } - - std::vector<std::string> Dirs; - } Visit; - - Traversal.TraverseFileSystem(m_BasePath, Visit); - return Visit.Dirs; -} - -bool -NamespaceCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue) -{ - ZenCacheStore* Store = GetStore(Namespace); - if (!Store) - { - return false; - } - return Store->Get(Bucket, HashKey, OutValue); -} - -void -NamespaceCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value) -{ - ZenCacheStore* Store = GetStore(Namespace); - if (!Store) - { - return; - } - Store->Put(Bucket, HashKey, Value); -} - -bool -NamespaceCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket) -{ - ZenCacheStore* Store = GetStore(Namespace); - if (!Store) - { - return false; - } - return Store->DropBucket(Bucket); -} - -void -NamespaceCacheStore::Flush() -{ - std::vector<ZenCacheStore*> Stores; - RwLock::SharedLockScope _(m_NamespacesLock); - Stores.reserve(m_Namespaces.size()); - for (const auto& Entry : m_Namespaces) - { - Stores.push_back(Entry.second); - } - _.ReleaseNow(); - for (ZenCacheStore* Store : Stores) - { - Store->Flush(); - } -} - -void -NamespaceCacheStore::Scrub(ScrubContext& Ctx) -{ - std::vector<ZenCacheStore*> Stores = GetAllStores(); - for (ZenCacheStore* Store : Stores) - { - Store->Scrub(Ctx); - } -} - -ZenCacheStore* -NamespaceCacheStore::GetStore(const std::string& Namespace) -{ - RwLock::SharedLockScope _(m_NamespacesLock); - if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end()) - { - return It->second; - } - return nullptr; -} - -std::vector<ZenCacheStore*> -NamespaceCacheStore::GetAllStores() const -{ - std::vector<ZenCacheStore*> Stores; - RwLock::SharedLockScope _(m_NamespacesLock); - Stores.reserve(m_Namespaces.size()); - for (const auto& Entry : m_Namespaces) - { - Stores.push_back(Entry.second); - } - return Stores; -} - -void -NamespaceCacheStore::GatherReferences(GcContext& GcCtx) -{ - std::vector<ZenCacheStore*> Stores = GetAllStores(); - for (ZenCacheStore* Store : Stores) - { - Store->GatherReferences(GcCtx); - } -} - -void -NamespaceCacheStore::CollectGarbage(GcContext& GcCtx) -{ - std::vector<ZenCacheStore*> Stores = GetAllStores(); - for (ZenCacheStore* Store : Stores) - { - Store->CollectGarbage(GcCtx); - } -} - -GcStorageSize -NamespaceCacheStore::StorageSize() const -{ - std::vector<ZenCacheStore*> Stores = GetAllStores(); - GcStorageSize Size; - for (ZenCacheStore* Store : Stores) - { - GcStorageSize StoreSize = Store->StorageSize(); - Size.MemorySize += StoreSize.MemorySize; - Size.DiskSize += StoreSize.DiskSize; - } - return Size; -} - -} // namespace zen diff --git a/zenserver/cache/namespacecachestore.h b/zenserver/cache/namespacecachestore.h deleted file mode 100644 index 85f4150bf..000000000 --- a/zenserver/cache/namespacecachestore.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/logging.h> -#include <zencore/refcount.h> - -#include <zenstore/gc.h> - -#include <string> - -namespace zen { - -class ScrubContext; -class ZenCacheStore; -struct ZenCacheValue; - -/** NamespaceCache Store - */ -class NamespaceCacheStore : public RefCounted, public GcStorage, public GcContributor -{ -public: - NamespaceCacheStore(std::filesystem::path BasePath, CasGc& Gc); - ~NamespaceCacheStore(); - - bool Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); - void Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); - bool DropBucket(const std::string& Namespace, std::string_view Bucket); - void Flush(); - void Scrub(ScrubContext& Ctx); - - spdlog::logger& Log() { return m_Log; } - const std::filesystem::path& BasePath() const { return m_BasePath; } - - virtual void GatherReferences(GcContext& GcCtx) override; - virtual void CollectGarbage(GcContext& GcCtx) override; - virtual GcStorageSize StorageSize() const override; - -private: - std::vector<std::string> FindExistingFolders() const; - ZenCacheStore* GetStore(const std::string& Namespace); - std::vector<ZenCacheStore*> GetAllStores() const; - spdlog::logger& m_Log; - std::filesystem::path m_BasePath; - mutable RwLock m_NamespacesLock; - std::unordered_map<std::string, ZenCacheStore*> m_Namespaces; -}; - -} // namespace zen diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 276c99081..8deb958be 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -21,7 +21,6 @@ //#include "cachekey.h" #include "monitoring/httpstats.h" -#include "namespacecachestore.h" #include "structuredcachestore.h" #include "upstream/jupiter.h" #include "upstream/upstreamcache.h" @@ -73,13 +72,13 @@ struct PutRequestData ////////////////////////////////////////////////////////////////////////// -HttpStructuredCacheService::HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore, - CidStore& InCidStore, - HttpStatsService& StatsService, - HttpStatusService& StatusService, - UpstreamCache& UpstreamCache) +HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& CacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache) : m_Log(logging::Get("cache")) -, m_CacheStore(InNamespaceCacheStore) +, m_CacheStore(CacheStore) , m_StatsService(StatsService) , m_StatusService(StatusService) , m_CidStore(InCidStore) diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 7e9847838..c41afef12 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -25,7 +25,7 @@ class CbObjectView; struct PutRequestData; class ScrubContext; class UpstreamCache; -class NamespaceCacheStore; +class ZenCacheStore; enum class CachePolicy : uint32_t; namespace cache::detail { @@ -64,11 +64,11 @@ namespace cache::detail { class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider { public: - HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore, - CidStore& InCidStore, - HttpStatsService& StatsService, - HttpStatusService& StatusService, - UpstreamCache& UpstreamCache); + HttpStructuredCacheService(ZenCacheStore& CacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache); ~HttpStructuredCacheService(); virtual const char* BaseUri() const override; @@ -140,7 +140,7 @@ private: spdlog::logger& Log() { return m_Log; } spdlog::logger& m_Log; - NamespaceCacheStore& m_CacheStore; + ZenCacheStore& m_CacheStore; HttpStatsService& m_StatsService; HttpStatusService& m_StatusService; CidStore& m_CidStore; diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 2869191fd..075b7d408 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -232,7 +232,7 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object) WriteFile(Path, Object.GetBuffer().AsIoBuffer()); } -ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) +ZenCacheNamespace::ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir) : GcStorage(Gc) , GcContributor(Gc) , m_RootDir(RootDir) @@ -248,12 +248,12 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) #endif } -ZenCacheStore::~ZenCacheStore() +ZenCacheNamespace::~ZenCacheNamespace() { } bool -ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) { ZEN_TRACE_CPU("Z$::Get"); @@ -291,7 +291,7 @@ ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheVal } void -ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) { ZEN_TRACE_CPU("Z$::Put"); @@ -327,7 +327,7 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa } bool -ZenCacheStore::DropBucket(std::string_view Bucket) +ZenCacheNamespace::DropBucket(std::string_view Bucket) { ZEN_INFO("dropping bucket '{}'", Bucket); @@ -343,13 +343,13 @@ ZenCacheStore::DropBucket(std::string_view Bucket) } void -ZenCacheStore::Flush() +ZenCacheNamespace::Flush() { m_DiskLayer.Flush(); } void -ZenCacheStore::Scrub(ScrubContext& Ctx) +ZenCacheNamespace::Scrub(ScrubContext& Ctx) { if (m_LastScrubTime == Ctx.ScrubTimestamp()) { @@ -363,7 +363,7 @@ ZenCacheStore::Scrub(ScrubContext& Ctx) } void -ZenCacheStore::GatherReferences(GcContext& GcCtx) +ZenCacheNamespace::GatherReferences(GcContext& GcCtx) { Stopwatch Timer; const auto Guard = @@ -377,14 +377,14 @@ ZenCacheStore::GatherReferences(GcContext& GcCtx) } void -ZenCacheStore::CollectGarbage(GcContext& GcCtx) +ZenCacheNamespace::CollectGarbage(GcContext& GcCtx) { m_MemLayer.Reset(); m_DiskLayer.CollectGarbage(GcCtx); } GcStorageSize -ZenCacheStore::StorageSize() const +ZenCacheNamespace::StorageSize() const { return {.DiskSize = m_DiskLayer.TotalSize(), .MemorySize = m_MemLayer.TotalSize()}; } @@ -2098,6 +2098,200 @@ ZenCacheDiskLayer::TotalSize() const return TotalSize; } +//////////////////////////// ZenCacheStore + +const char* ZenCacheNamespaceDirPrefix = "ns_"; + +namespace { + + std::vector<std::string> FindExistingFolders(const std::filesystem::path& RootPath) + { + FileSystemTraversal Traversal; + struct Visitor : public FileSystemTraversal::TreeVisitor + { + virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override {} + + virtual bool VisitDirectory(const std::filesystem::path&, const path_view& DirectoryName) override + { + std::string DirName8 = WideToUtf8(DirectoryName); + Dirs.push_back(DirName8); + return false; + } + + std::vector<std::string> Dirs; + } Visit; + + Traversal.TraverseFileSystem(RootPath, Visit); + return Visit.Dirs; + } + +} // namespace + +ZenCacheStore::ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc) : GcStorage(Gc), GcContributor(Gc) +{ + CreateDirectories(BasePath); + std::vector<std::string> ExistingFolders = FindExistingFolders(BasePath); + + std::vector<std::string> LegacyBuckets; + std::vector<std::string> Namespaces; + for (const std::string& DirName : ExistingFolders) + { + if (DirName.starts_with(ZenCacheNamespaceDirPrefix)) + { + Namespaces.push_back(DirName.substr(3)); + continue; + } + LegacyBuckets.push_back(DirName); + } + + ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size()); + + if (std::find(Namespaces.begin(), Namespaces.end(), "") == Namespaces.end()) + { + ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size()); + std::filesystem::path DefaultNamespaceFolder = BasePath / ZenCacheNamespaceDirPrefix; + CreateDirectories(DefaultNamespaceFolder); + + // Move any non-namespace folders into the default namespace folder + for (const std::string& DirName : LegacyBuckets) + { + std::filesystem::path LegacyFolder = BasePath / DirName; + std::filesystem::path NewPath = DefaultNamespaceFolder / DirName; + std::filesystem::rename(LegacyFolder, NewPath); + } + Namespaces.push_back(""); + } + + for (const std::string& NamespaceName : Namespaces) + { + Ref<ZenCacheNamespace> Store = new ZenCacheNamespace(Gc, BasePath / (ZenCacheNamespaceDirPrefix + NamespaceName)); + m_Namespaces[NamespaceName] = Store; + } +} + +ZenCacheStore::~ZenCacheStore() +{ + m_Namespaces.clear(); +} + +bool +ZenCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue) +{ + Ref<ZenCacheNamespace> Store = GetStore(Namespace); + if (!Store) + { + return false; + } + return Store->Get(Bucket, HashKey, OutValue); +} + +void +ZenCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value) +{ + Ref<ZenCacheNamespace> Store = GetStore(Namespace); + if (!Store) + { + return; + } + Store->Put(Bucket, HashKey, Value); +} + +bool +ZenCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket) +{ + Ref<ZenCacheNamespace> Store = GetStore(Namespace); + if (!Store) + { + return false; + } + return Store->DropBucket(Bucket); +} + +void +ZenCacheStore::Flush() +{ + std::vector<Ref<ZenCacheNamespace>> Stores; + RwLock::SharedLockScope _(m_NamespacesLock); + Stores.reserve(m_Namespaces.size()); + for (const auto& Entry : m_Namespaces) + { + Stores.push_back(Entry.second); + } + _.ReleaseNow(); + for (const Ref<ZenCacheNamespace>& Store : Stores) + { + Store->Flush(); + } +} + +void +ZenCacheStore::Scrub(ScrubContext& Ctx) +{ + std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores(); + for (const Ref<ZenCacheNamespace>& Store : Stores) + { + Store->Scrub(Ctx); + } +} + +Ref<ZenCacheNamespace> +ZenCacheStore::GetStore(const std::string& Namespace) +{ + RwLock::SharedLockScope _(m_NamespacesLock); + if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end()) + { + return It->second; + } + return nullptr; +} + +std::vector<Ref<ZenCacheNamespace>> +ZenCacheStore::GetAllStores() const +{ + std::vector<Ref<ZenCacheNamespace>> Stores; + RwLock::SharedLockScope _(m_NamespacesLock); + Stores.reserve(m_Namespaces.size()); + for (const auto& Entry : m_Namespaces) + { + Stores.push_back(Entry.second); + } + return Stores; +} + +void +ZenCacheStore::GatherReferences(GcContext& GcCtx) +{ + std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores(); + for (const Ref<ZenCacheNamespace>& Store : Stores) + { + Store->GatherReferences(GcCtx); + } +} + +void +ZenCacheStore::CollectGarbage(GcContext& GcCtx) +{ + std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores(); + for (const Ref<ZenCacheNamespace>& Store : Stores) + { + Store->CollectGarbage(GcCtx); + } +} + +GcStorageSize +ZenCacheStore::StorageSize() const +{ + std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores(); + GcStorageSize Size; + for (const Ref<ZenCacheNamespace>& Store : Stores) + { + GcStorageSize StoreSize = Store->StorageSize(); + Size.MemorySize += StoreSize.MemorySize; + Size.DiskSize += StoreSize.DiskSize; + } + return Size; +} + ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS @@ -2136,7 +2330,7 @@ TEST_CASE("z$.store") CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const int kIterationCount = 100; @@ -2189,8 +2383,8 @@ TEST_CASE("z$.size") GcStorageSize CacheSize; { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256); @@ -2209,8 +2403,8 @@ TEST_CASE("z$.size") } { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const GcStorageSize SerializedSize = Zcs.StorageSize(); CHECK_EQ(SerializedSize.MemorySize, 0); @@ -2232,8 +2426,8 @@ TEST_CASE("z$.size") GcStorageSize CacheSize; { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64); @@ -2252,8 +2446,8 @@ TEST_CASE("z$.size") } { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const GcStorageSize SerializedSize = Zcs.StorageSize(); CHECK_EQ(SerializedSize.MemorySize, 0); @@ -2290,9 +2484,9 @@ TEST_CASE("z$.gc") }; { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); - const auto Bucket = "teardrinker"sv; + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); + const auto Bucket = "teardrinker"sv; // Create a cache record const IoHash Key = CreateKey(42); @@ -2328,7 +2522,7 @@ TEST_CASE("z$.gc") // Expect timestamps to be serialized { CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); std::vector<IoHash> Keep; // Collect garbage with 1 hour max cache duration @@ -2349,7 +2543,7 @@ TEST_CASE("z$.gc") { ScopedTemporaryDirectory TempDir; CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const auto Bucket = "fortysixandtwo"sv; const GcClock::TimePoint CurrentTime = GcClock::Now(); @@ -2397,7 +2591,7 @@ TEST_CASE("z$.gc") { ScopedTemporaryDirectory TempDir; CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const auto Bucket = "rightintwo"sv; const GcClock::TimePoint CurrentTime = GcClock::Now(); @@ -2490,7 +2684,7 @@ TEST_CASE("z$.legacyconversion") const std::string Bucket = "rightintwo"; { CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path()); + ZenCacheNamespace Zcs(Gc, TempDir.Path()); const GcClock::TimePoint CurrentTime = GcClock::Now(); for (size_t i = 0; i < ChunkCount; i++) @@ -2578,8 +2772,8 @@ TEST_CASE("z$.legacyconversion") std::filesystem::remove(IndexPath); { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path()); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path()); for (size_t i = 0; i < ChunkCount; i += 2) { @@ -2639,9 +2833,9 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true)) CreateDirectories(TempDir.Path()); - WorkerThreadPool ThreadPool(4); - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path()); + WorkerThreadPool ThreadPool(4); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path()); { std::atomic<size_t> WorkCompleted = 0; diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 0c2a7c0b2..a803b0603 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -322,11 +322,11 @@ private: ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete; }; -class ZenCacheStore final : public GcStorage, public GcContributor +class ZenCacheNamespace final : public RefCounted, public GcStorage, public GcContributor { public: - ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir); - ~ZenCacheStore(); + ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir); + ~ZenCacheNamespace(); bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); @@ -349,8 +349,31 @@ private: std::unique_ptr<ZenCacheTracker> m_AccessTracker; #endif - ZenCacheStore(const ZenCacheStore&) = delete; - ZenCacheStore& operator=(const ZenCacheStore&) = delete; + ZenCacheNamespace(const ZenCacheNamespace&) = delete; + ZenCacheNamespace& operator=(const ZenCacheNamespace&) = delete; +}; + +class ZenCacheStore final : public GcStorage, public GcContributor +{ +public: + ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc); + ~ZenCacheStore(); + + bool Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); + bool DropBucket(const std::string& Namespace, std::string_view Bucket); + void Flush(); + void Scrub(ScrubContext& Ctx); + + virtual void GatherReferences(GcContext& GcCtx) override; + virtual void CollectGarbage(GcContext& GcCtx) override; + virtual GcStorageSize StorageSize() const override; + +private: + Ref<ZenCacheNamespace> GetStore(const std::string& Namespace); + std::vector<Ref<ZenCacheNamespace>> GetAllStores() const; + mutable RwLock m_NamespacesLock; + std::unordered_map<std::string, Ref<ZenCacheNamespace>> m_Namespaces; }; void z$_forcelink(); diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index 2edc6dc49..1deaf00a5 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -23,7 +23,7 @@ class CbObjectWriter; class CidStore; class CloudCacheTokenProvider; class WorkerThreadPool; -class ZenCacheStore; +class ZenCacheNamespace; struct CloudCacheClientOptions; struct UpstreamAuthConfig; diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 49f384774..c89227106 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -19,7 +19,6 @@ #include <zenstore/cidstore.h> #include <auth/authmgr.h> -#include "cache/namespacecachestore.h" #include "cache/structuredcache.h" #include "cache/structuredcachestore.h" #include "diag/logging.h" @@ -1174,7 +1173,7 @@ namespace detail { class UpstreamCacheImpl final : public UpstreamCache { public: - UpstreamCacheImpl(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore) + UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) : m_Log(logging::Get("upstream")) , m_Options(Options) , m_CacheStore(CacheStore) @@ -1688,7 +1687,7 @@ private: spdlog::logger& m_Log; UpstreamCacheOptions m_Options; - NamespaceCacheStore& m_CacheStore; + ZenCacheStore& m_CacheStore; CidStore& m_CidStore; UpstreamQueue m_UpstreamQueue; std::shared_mutex m_EndpointsMutex; @@ -1713,7 +1712,7 @@ UpstreamEndpoint::CreateJupiterEndpoint(const CloudCacheClientOptions& Options, } std::unique_ptr<UpstreamCache> -UpstreamCache::Create(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore) +UpstreamCache::Create(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) { return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore); } diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 54386e80d..6f18b3119 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -24,7 +24,7 @@ class CbObjectView; class CbPackage; class CbObjectWriter; class CidStore; -class NamespaceCacheStore; +class ZenCacheStore; struct CloudCacheClientOptions; class CloudCacheTokenProvider; struct ZenStructuredCacheClientOptions; @@ -206,7 +206,7 @@ public: virtual void GetStatus(CbObjectWriter& CbO) = 0; - static std::unique_ptr<UpstreamCache> Create(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore); + static std::unique_ptr<UpstreamCache> Create(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore); }; } // namespace zen diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 9b7083312..e572e6a49 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -102,7 +102,6 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "admin/admin.h" #include "auth/authmgr.h" #include "auth/authservice.h" -#include "cache/namespacecachestore.h" #include "cache/structuredcache.h" #include "cache/structuredcachestore.h" #include "compute/function.h" @@ -612,7 +611,7 @@ private: zen::GcScheduler m_GcScheduler{m_CasGc}; std::unique_ptr<zen::CasStore> m_CasStore{zen::CreateCasStore(m_CasGc)}; std::unique_ptr<zen::CidStore> m_CidStore; - std::unique_ptr<zen::NamespaceCacheStore> m_CacheStore; + std::unique_ptr<zen::ZenCacheStore> m_CacheStore; zen::CasScrubber m_Scrubber{*m_CasStore}; zen::HttpTestService m_TestService; zen::HttpTestingService m_TestingService; @@ -756,7 +755,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) using namespace std::literals; ZEN_INFO("instantiating structured cache service"); - m_CacheStore = std::make_unique<NamespaceCacheStore>(m_DataRoot / "cache", m_CasGc); + m_CacheStore = std::make_unique<ZenCacheStore>(m_DataRoot / "cache", m_CasGc); const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; |