From ef12415d287c9307c0c4774aeacff6c91966f693 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 May 2022 19:45:57 +0200 Subject: cleanup --- zenserver/cache/structuredcachestore.cpp | 256 +++++++++++++++++++++++++++---- 1 file changed, 225 insertions(+), 31 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 2869191fd..075b7d408 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -232,7 +232,7 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object) WriteFile(Path, Object.GetBuffer().AsIoBuffer()); } -ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) +ZenCacheNamespace::ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir) : GcStorage(Gc) , GcContributor(Gc) , m_RootDir(RootDir) @@ -248,12 +248,12 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) #endif } -ZenCacheStore::~ZenCacheStore() +ZenCacheNamespace::~ZenCacheNamespace() { } bool -ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) { ZEN_TRACE_CPU("Z$::Get"); @@ -291,7 +291,7 @@ ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheVal } void -ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) { ZEN_TRACE_CPU("Z$::Put"); @@ -327,7 +327,7 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa } bool -ZenCacheStore::DropBucket(std::string_view Bucket) +ZenCacheNamespace::DropBucket(std::string_view Bucket) { ZEN_INFO("dropping bucket '{}'", Bucket); @@ -343,13 +343,13 @@ ZenCacheStore::DropBucket(std::string_view Bucket) } void -ZenCacheStore::Flush() +ZenCacheNamespace::Flush() { m_DiskLayer.Flush(); } void -ZenCacheStore::Scrub(ScrubContext& Ctx) +ZenCacheNamespace::Scrub(ScrubContext& Ctx) { if (m_LastScrubTime == Ctx.ScrubTimestamp()) { @@ -363,7 +363,7 @@ ZenCacheStore::Scrub(ScrubContext& Ctx) } void -ZenCacheStore::GatherReferences(GcContext& GcCtx) +ZenCacheNamespace::GatherReferences(GcContext& GcCtx) { Stopwatch Timer; const auto Guard = @@ -377,14 +377,14 @@ ZenCacheStore::GatherReferences(GcContext& GcCtx) } void -ZenCacheStore::CollectGarbage(GcContext& GcCtx) +ZenCacheNamespace::CollectGarbage(GcContext& GcCtx) { m_MemLayer.Reset(); m_DiskLayer.CollectGarbage(GcCtx); } GcStorageSize -ZenCacheStore::StorageSize() const +ZenCacheNamespace::StorageSize() const { return {.DiskSize = m_DiskLayer.TotalSize(), .MemorySize = m_MemLayer.TotalSize()}; } @@ -2098,6 +2098,200 @@ ZenCacheDiskLayer::TotalSize() const return TotalSize; } +//////////////////////////// ZenCacheStore + +const char* ZenCacheNamespaceDirPrefix = "ns_"; + +namespace { + + std::vector FindExistingFolders(const std::filesystem::path& RootPath) + { + FileSystemTraversal Traversal; + struct Visitor : public FileSystemTraversal::TreeVisitor + { + virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override {} + + virtual bool VisitDirectory(const std::filesystem::path&, const path_view& DirectoryName) override + { + std::string DirName8 = WideToUtf8(DirectoryName); + Dirs.push_back(DirName8); + return false; + } + + std::vector Dirs; + } Visit; + + Traversal.TraverseFileSystem(RootPath, Visit); + return Visit.Dirs; + } + +} // namespace + +ZenCacheStore::ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc) : GcStorage(Gc), GcContributor(Gc) +{ + CreateDirectories(BasePath); + std::vector ExistingFolders = FindExistingFolders(BasePath); + + std::vector LegacyBuckets; + std::vector Namespaces; + for (const std::string& DirName : ExistingFolders) + { + if (DirName.starts_with(ZenCacheNamespaceDirPrefix)) + { + Namespaces.push_back(DirName.substr(3)); + continue; + } + LegacyBuckets.push_back(DirName); + } + + ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size()); + + if (std::find(Namespaces.begin(), Namespaces.end(), "") == Namespaces.end()) + { + ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size()); + std::filesystem::path DefaultNamespaceFolder = BasePath / ZenCacheNamespaceDirPrefix; + CreateDirectories(DefaultNamespaceFolder); + + // Move any non-namespace folders into the default namespace folder + for (const std::string& DirName : LegacyBuckets) + { + std::filesystem::path LegacyFolder = BasePath / DirName; + std::filesystem::path NewPath = DefaultNamespaceFolder / DirName; + std::filesystem::rename(LegacyFolder, NewPath); + } + Namespaces.push_back(""); + } + + for (const std::string& NamespaceName : Namespaces) + { + Ref Store = new ZenCacheNamespace(Gc, BasePath / (ZenCacheNamespaceDirPrefix + NamespaceName)); + m_Namespaces[NamespaceName] = Store; + } +} + +ZenCacheStore::~ZenCacheStore() +{ + m_Namespaces.clear(); +} + +bool +ZenCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue) +{ + Ref Store = GetStore(Namespace); + if (!Store) + { + return false; + } + return Store->Get(Bucket, HashKey, OutValue); +} + +void +ZenCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value) +{ + Ref Store = GetStore(Namespace); + if (!Store) + { + return; + } + Store->Put(Bucket, HashKey, Value); +} + +bool +ZenCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket) +{ + Ref Store = GetStore(Namespace); + if (!Store) + { + return false; + } + return Store->DropBucket(Bucket); +} + +void +ZenCacheStore::Flush() +{ + std::vector> Stores; + RwLock::SharedLockScope _(m_NamespacesLock); + Stores.reserve(m_Namespaces.size()); + for (const auto& Entry : m_Namespaces) + { + Stores.push_back(Entry.second); + } + _.ReleaseNow(); + for (const Ref& Store : Stores) + { + Store->Flush(); + } +} + +void +ZenCacheStore::Scrub(ScrubContext& Ctx) +{ + std::vector> Stores = GetAllStores(); + for (const Ref& Store : Stores) + { + Store->Scrub(Ctx); + } +} + +Ref +ZenCacheStore::GetStore(const std::string& Namespace) +{ + RwLock::SharedLockScope _(m_NamespacesLock); + if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end()) + { + return It->second; + } + return nullptr; +} + +std::vector> +ZenCacheStore::GetAllStores() const +{ + std::vector> Stores; + RwLock::SharedLockScope _(m_NamespacesLock); + Stores.reserve(m_Namespaces.size()); + for (const auto& Entry : m_Namespaces) + { + Stores.push_back(Entry.second); + } + return Stores; +} + +void +ZenCacheStore::GatherReferences(GcContext& GcCtx) +{ + std::vector> Stores = GetAllStores(); + for (const Ref& Store : Stores) + { + Store->GatherReferences(GcCtx); + } +} + +void +ZenCacheStore::CollectGarbage(GcContext& GcCtx) +{ + std::vector> Stores = GetAllStores(); + for (const Ref& Store : Stores) + { + Store->CollectGarbage(GcCtx); + } +} + +GcStorageSize +ZenCacheStore::StorageSize() const +{ + std::vector> Stores = GetAllStores(); + GcStorageSize Size; + for (const Ref& Store : Stores) + { + GcStorageSize StoreSize = Store->StorageSize(); + Size.MemorySize += StoreSize.MemorySize; + Size.DiskSize += StoreSize.DiskSize; + } + return Size; +} + ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS @@ -2136,7 +2330,7 @@ TEST_CASE("z$.store") CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const int kIterationCount = 100; @@ -2189,8 +2383,8 @@ TEST_CASE("z$.size") GcStorageSize CacheSize; { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256); @@ -2209,8 +2403,8 @@ TEST_CASE("z$.size") } { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const GcStorageSize SerializedSize = Zcs.StorageSize(); CHECK_EQ(SerializedSize.MemorySize, 0); @@ -2232,8 +2426,8 @@ TEST_CASE("z$.size") GcStorageSize CacheSize; { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64); @@ -2252,8 +2446,8 @@ TEST_CASE("z$.size") } { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const GcStorageSize SerializedSize = Zcs.StorageSize(); CHECK_EQ(SerializedSize.MemorySize, 0); @@ -2290,9 +2484,9 @@ TEST_CASE("z$.gc") }; { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); - const auto Bucket = "teardrinker"sv; + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); + const auto Bucket = "teardrinker"sv; // Create a cache record const IoHash Key = CreateKey(42); @@ -2328,7 +2522,7 @@ TEST_CASE("z$.gc") // Expect timestamps to be serialized { CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); std::vector Keep; // Collect garbage with 1 hour max cache duration @@ -2349,7 +2543,7 @@ TEST_CASE("z$.gc") { ScopedTemporaryDirectory TempDir; CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const auto Bucket = "fortysixandtwo"sv; const GcClock::TimePoint CurrentTime = GcClock::Now(); @@ -2397,7 +2591,7 @@ TEST_CASE("z$.gc") { ScopedTemporaryDirectory TempDir; CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); const auto Bucket = "rightintwo"sv; const GcClock::TimePoint CurrentTime = GcClock::Now(); @@ -2490,7 +2684,7 @@ TEST_CASE("z$.legacyconversion") const std::string Bucket = "rightintwo"; { CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path()); + ZenCacheNamespace Zcs(Gc, TempDir.Path()); const GcClock::TimePoint CurrentTime = GcClock::Now(); for (size_t i = 0; i < ChunkCount; i++) @@ -2578,8 +2772,8 @@ TEST_CASE("z$.legacyconversion") std::filesystem::remove(IndexPath); { - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path()); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path()); for (size_t i = 0; i < ChunkCount; i += 2) { @@ -2639,9 +2833,9 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true)) CreateDirectories(TempDir.Path()); - WorkerThreadPool ThreadPool(4); - CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path()); + WorkerThreadPool ThreadPool(4); + CasGc Gc; + ZenCacheNamespace Zcs(Gc, TempDir.Path()); { std::atomic WorkCompleted = 0; -- cgit v1.2.3 From 861a92d1ee6c54eeb9035190501baf8ea888591f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 May 2022 09:55:09 +0200 Subject: cleanup and review feedback --- zenserver/cache/structuredcachestore.cpp | 33 +++++++++++++------------------- 1 file changed, 13 insertions(+), 20 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 075b7d408..7db18a7bb 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2100,7 +2100,7 @@ ZenCacheDiskLayer::TotalSize() const //////////////////////////// ZenCacheStore -const char* ZenCacheNamespaceDirPrefix = "ns_"; +static constexpr std::string_view ZenCacheNamespaceDirPrefix = "ns_"; namespace { @@ -2146,10 +2146,10 @@ ZenCacheStore::ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc) : GcStor ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size()); - if (std::find(Namespaces.begin(), Namespaces.end(), "") == Namespaces.end()) + if (std::find(Namespaces.begin(), Namespaces.end(), DefaultNamespace) == Namespaces.end()) { ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size()); - std::filesystem::path DefaultNamespaceFolder = BasePath / ZenCacheNamespaceDirPrefix; + std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, DefaultNamespace); CreateDirectories(DefaultNamespaceFolder); // Move any non-namespace folders into the default namespace folder @@ -2159,12 +2159,12 @@ ZenCacheStore::ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc) : GcStor std::filesystem::path NewPath = DefaultNamespaceFolder / DirName; std::filesystem::rename(LegacyFolder, NewPath); } - Namespaces.push_back(""); + Namespaces.push_back(std::string(DefaultNamespace)); } for (const std::string& NamespaceName : Namespaces) { - Ref Store = new ZenCacheNamespace(Gc, BasePath / (ZenCacheNamespaceDirPrefix + NamespaceName)); + Ref Store = new ZenCacheNamespace(Gc, BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, NamespaceName)); m_Namespaces[NamespaceName] = Store; } } @@ -2175,7 +2175,7 @@ ZenCacheStore::~ZenCacheStore() } bool -ZenCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue) { Ref Store = GetStore(Namespace); if (!Store) @@ -2186,9 +2186,9 @@ ZenCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const } void -ZenCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheStore::Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value) { - Ref Store = GetStore(Namespace); + Ref Store = GetStore(std::string(Namespace)); if (!Store) { return; @@ -2197,9 +2197,9 @@ ZenCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const } bool -ZenCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket) +ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket) { - Ref Store = GetStore(Namespace); + Ref Store = GetStore(std::string(Namespace)); if (!Store) { return false; @@ -2210,14 +2210,7 @@ ZenCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket) void ZenCacheStore::Flush() { - std::vector> Stores; - RwLock::SharedLockScope _(m_NamespacesLock); - Stores.reserve(m_Namespaces.size()); - for (const auto& Entry : m_Namespaces) - { - Stores.push_back(Entry.second); - } - _.ReleaseNow(); + std::vector> Stores = GetAllStores(); for (const Ref& Store : Stores) { Store->Flush(); @@ -2235,10 +2228,10 @@ ZenCacheStore::Scrub(ScrubContext& Ctx) } Ref -ZenCacheStore::GetStore(const std::string& Namespace) +ZenCacheStore::GetStore(std::string_view Namespace) { RwLock::SharedLockScope _(m_NamespacesLock); - if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end()) + if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end()) { return It->second; } -- cgit v1.2.3 From 7b842505d25fcd8f0c52656c608c1a66f45ccf96 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 May 2022 10:16:39 +0200 Subject: mac/linux build fix --- zenserver/cache/structuredcachestore.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 7db18a7bb..23ad550c9 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2113,8 +2113,12 @@ namespace { virtual bool VisitDirectory(const std::filesystem::path&, const path_view& DirectoryName) override { - std::string DirName8 = WideToUtf8(DirectoryName); - Dirs.push_back(DirName8); +#if ZEN_PLATFORM_WINDOWS + std::string DirectoryName8 = WideToUtf8(DirectoryName); +#else + std::string DirectoryName8 = std::string(DirectoryName); +#endif + Dirs.push_back(DirectoryName8); return false; } -- cgit v1.2.3 From d484acb3d32662c9e1faf9a99efad543f607732a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 May 2022 10:49:35 +0200 Subject: revert back constructor order for ZenCacheStore --- zenserver/cache/structuredcachestore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 23ad550c9..a734e9eb1 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2131,7 +2131,7 @@ namespace { } // namespace -ZenCacheStore::ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc) : GcStorage(Gc), GcContributor(Gc) +ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc) { CreateDirectories(BasePath); std::vector ExistingFolders = FindExistingFolders(BasePath); -- cgit v1.2.3 From e4b96fade542151fca17b5ac61e3eaad263ce92c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 6 May 2022 11:53:11 +0200 Subject: Added GetDirectoryContent utility --- zenserver/cache/structuredcachestore.cpp | 78 ++++++-------------------------- 1 file changed, 13 insertions(+), 65 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index a734e9eb1..da48253e2 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1956,49 +1956,23 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z void ZenCacheDiskLayer::DiscoverBuckets() { - FileSystemTraversal Traversal; - struct Visitor : public FileSystemTraversal::TreeVisitor - { - virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent, - [[maybe_unused]] const path_view& File, - [[maybe_unused]] uint64_t FileSize) override - { - } - - virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override - { - Dirs.push_back((decltype(Dirs)::value_type)(DirectoryName)); - return false; - } - - std::vector Dirs; - } Visit; - - Traversal.TraverseFileSystem(m_RootDir, Visit); + DirectoryContent DirContent; + GetDirectoryContent(m_RootDir, DirectoryContent::IncludeDirsFlag, DirContent); // Initialize buckets RwLock::ExclusiveLockScope _(m_Lock); - for (const auto& BucketName : Visit.Dirs) + for (const std::filesystem::path& BucketPath : DirContent.Directories) { + std::string BucketName = PathToUtf8(BucketPath.stem()); // New bucket needs to be created - -#if ZEN_PLATFORM_WINDOWS - std::string BucketName8 = WideToUtf8(BucketName); -#else - const auto& BucketName8 = BucketName; -#endif - - if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end()) + if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) { } else { - auto InsertResult = m_Buckets.try_emplace(BucketName8, BucketName8); - - std::filesystem::path BucketPath = m_RootDir; - BucketPath /= BucketName8; + auto InsertResult = m_Buckets.try_emplace(BucketName, BucketName); CacheBucket& Bucket = InsertResult.first->second; @@ -2006,11 +1980,11 @@ ZenCacheDiskLayer::DiscoverBuckets() if (Bucket.IsOk()) { - ZEN_INFO("Discovered bucket '{}'", BucketName8); + ZEN_INFO("Discovered bucket '{}'", BucketName); } else { - ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName8, m_RootDir); + ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); m_Buckets.erase(InsertResult.first); } @@ -2102,44 +2076,18 @@ ZenCacheDiskLayer::TotalSize() const static constexpr std::string_view ZenCacheNamespaceDirPrefix = "ns_"; -namespace { - - std::vector FindExistingFolders(const std::filesystem::path& RootPath) - { - FileSystemTraversal Traversal; - struct Visitor : public FileSystemTraversal::TreeVisitor - { - virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override {} - - virtual bool VisitDirectory(const std::filesystem::path&, const path_view& DirectoryName) override - { -#if ZEN_PLATFORM_WINDOWS - std::string DirectoryName8 = WideToUtf8(DirectoryName); -#else - std::string DirectoryName8 = std::string(DirectoryName); -#endif - Dirs.push_back(DirectoryName8); - return false; - } - - std::vector Dirs; - } Visit; - - Traversal.TraverseFileSystem(RootPath, Visit); - return Visit.Dirs; - } - -} // namespace - ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc) { CreateDirectories(BasePath); - std::vector ExistingFolders = FindExistingFolders(BasePath); + + DirectoryContent DirContent; + GetDirectoryContent(BasePath, DirectoryContent::IncludeDirsFlag, DirContent); std::vector LegacyBuckets; std::vector Namespaces; - for (const std::string& DirName : ExistingFolders) + for (const std::filesystem::path& DirPath : DirContent.Directories) { + std::string DirName = PathToUtf8(DirPath.stem()); if (DirName.starts_with(ZenCacheNamespaceDirPrefix)) { Namespaces.push_back(DirName.substr(3)); -- cgit v1.2.3 From 6db10b5a491297d45c14efae453c420f0d7fa58c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 6 May 2022 12:12:09 +0200 Subject: review feedback and cleanup --- zenserver/cache/structuredcachestore.cpp | 81 ++++++++++++++------------------ 1 file changed, 36 insertions(+), 45 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index da48253e2..3ac319961 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2109,7 +2109,12 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor { std::filesystem::path LegacyFolder = BasePath / DirName; std::filesystem::path NewPath = DefaultNamespaceFolder / DirName; - std::filesystem::rename(LegacyFolder, NewPath); + std::error_code Ec; + std::filesystem::rename(LegacyFolder, NewPath, Ec); + if (Ec) + { + ZEN_ERROR("Unable to move '{}' to '{}', reason '{}'", LegacyFolder, NewPath, Ec.message()); + } } Namespaces.push_back(std::string(DefaultNamespace)); } @@ -2129,54 +2134,45 @@ ZenCacheStore::~ZenCacheStore() bool ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue) { - Ref Store = GetStore(Namespace); - if (!Store) + if (Ref Store = GetStore(Namespace); Store) { - return false; + return Store->Get(Bucket, HashKey, OutValue); } - return Store->Get(Bucket, HashKey, OutValue); + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString()); + return false; } void ZenCacheStore::Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value) { - Ref Store = GetStore(std::string(Namespace)); - if (!Store) + if (Ref Store = GetStore(Namespace); Store) { - return; + return Store->Put(Bucket, HashKey, Value); } - Store->Put(Bucket, HashKey, Value); + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString()); } bool ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket) { - Ref Store = GetStore(std::string(Namespace)); - if (!Store) + if (Ref Store = GetStore(Namespace); Store) { - return false; + return Store->DropBucket(Bucket); } - return Store->DropBucket(Bucket); + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}'", Namespace, Bucket); + return false; } void ZenCacheStore::Flush() { - std::vector> Stores = GetAllStores(); - for (const Ref& Store : Stores) - { - Store->Flush(); - } + IterateStores([&](const Ref& Store) { Store->Flush(); }); } void ZenCacheStore::Scrub(ScrubContext& Ctx) { - std::vector> Stores = GetAllStores(); - for (const Ref& Store : Stores) - { - Store->Scrub(Ctx); - } + IterateStores([&](const Ref& Store) { Store->Scrub(Ctx); }); } Ref @@ -2190,50 +2186,45 @@ ZenCacheStore::GetStore(std::string_view Namespace) return nullptr; } -std::vector> -ZenCacheStore::GetAllStores() const +void +ZenCacheStore::IterateStores(const std::function& Store)>& Callback) const { std::vector> Stores; - RwLock::SharedLockScope _(m_NamespacesLock); - Stores.reserve(m_Namespaces.size()); - for (const auto& Entry : m_Namespaces) { - Stores.push_back(Entry.second); + RwLock::SharedLockScope _(m_NamespacesLock); + Stores.reserve(m_Namespaces.size()); + for (const auto& Entry : m_Namespaces) + { + Stores.push_back(Entry.second); + } + } + for (const Ref& Store : Stores) + { + Callback(Store); } - return Stores; } void ZenCacheStore::GatherReferences(GcContext& GcCtx) { - std::vector> Stores = GetAllStores(); - for (const Ref& Store : Stores) - { - Store->GatherReferences(GcCtx); - } + IterateStores([&](const Ref& Store) { Store->GatherReferences(GcCtx); }); } void ZenCacheStore::CollectGarbage(GcContext& GcCtx) { - std::vector> Stores = GetAllStores(); - for (const Ref& Store : Stores) - { - Store->CollectGarbage(GcCtx); - } + IterateStores([&](const Ref& Store) { Store->CollectGarbage(GcCtx); }); } GcStorageSize ZenCacheStore::StorageSize() const { - std::vector> Stores = GetAllStores(); - GcStorageSize Size; - for (const Ref& Store : Stores) - { + GcStorageSize Size; + IterateStores([&](const Ref& Store) { GcStorageSize StoreSize = Store->StorageSize(); Size.MemorySize += StoreSize.MemorySize; Size.DiskSize += StoreSize.DiskSize; - } + }); return Size; } -- cgit v1.2.3 From 2f6461e3ed7851bc5592b6bc9efdfb0d973fe284 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 6 May 2022 23:32:35 +0200 Subject: remove use of Ref<> in ZenCacheStore naming cleanup --- zenserver/cache/structuredcachestore.cpp | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 3ac319961..1d43e9591 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2121,8 +2121,8 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor for (const std::string& NamespaceName : Namespaces) { - Ref Store = new ZenCacheNamespace(Gc, BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, NamespaceName)); - m_Namespaces[NamespaceName] = Store; + m_Namespaces[NamespaceName] = + std::make_unique(Gc, BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, NamespaceName)); } } @@ -2134,7 +2134,7 @@ ZenCacheStore::~ZenCacheStore() bool ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue) { - if (Ref Store = GetStore(Namespace); Store) + if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store) { return Store->Get(Bucket, HashKey, OutValue); } @@ -2145,7 +2145,7 @@ ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const Io void ZenCacheStore::Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value) { - if (Ref Store = GetStore(Namespace); Store) + if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store) { return Store->Put(Bucket, HashKey, Value); } @@ -2155,7 +2155,7 @@ ZenCacheStore::Put(std::string_view Namespace, std::string_view Bucket, const Io bool ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket) { - if (Ref Store = GetStore(Namespace); Store) + if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store) { return Store->DropBucket(Bucket); } @@ -2166,62 +2166,62 @@ ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket) void ZenCacheStore::Flush() { - IterateStores([&](const Ref& Store) { Store->Flush(); }); + IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Flush(); }); } void ZenCacheStore::Scrub(ScrubContext& Ctx) { - IterateStores([&](const Ref& Store) { Store->Scrub(Ctx); }); + IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); }); } -Ref -ZenCacheStore::GetStore(std::string_view Namespace) +ZenCacheNamespace* +ZenCacheStore::GetNamespace(std::string_view Namespace) { RwLock::SharedLockScope _(m_NamespacesLock); if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end()) { - return It->second; + return It->second.get(); } return nullptr; } void -ZenCacheStore::IterateStores(const std::function& Store)>& Callback) const +ZenCacheStore::IterateNamespaces(const std::function& Callback) const { - std::vector> Stores; + std::vector > Namespaces; { RwLock::SharedLockScope _(m_NamespacesLock); - Stores.reserve(m_Namespaces.size()); + Namespaces.reserve(m_Namespaces.size()); for (const auto& Entry : m_Namespaces) { - Stores.push_back(Entry.second); + Namespaces.push_back({Entry.first, *Entry.second}); } } - for (const Ref& Store : Stores) + for (auto& Entry : Namespaces) { - Callback(Store); + Callback(Entry.first, Entry.second); } } void ZenCacheStore::GatherReferences(GcContext& GcCtx) { - IterateStores([&](const Ref& Store) { Store->GatherReferences(GcCtx); }); + IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.GatherReferences(GcCtx); }); } void ZenCacheStore::CollectGarbage(GcContext& GcCtx) { - IterateStores([&](const Ref& Store) { Store->CollectGarbage(GcCtx); }); + IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.CollectGarbage(GcCtx); }); } GcStorageSize ZenCacheStore::StorageSize() const { GcStorageSize Size; - IterateStores([&](const Ref& Store) { - GcStorageSize StoreSize = Store->StorageSize(); + IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { + GcStorageSize StoreSize = Store.StorageSize(); Size.MemorySize += StoreSize.MemorySize; Size.DiskSize += StoreSize.DiskSize; }); -- cgit v1.2.3