From 37edbbd4437bd756236d99684ac4d7b0b6725aa8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 20 May 2022 13:13:09 +0200 Subject: Automatically create namespaces on requests (if enabled via configuration) --- zenserver/cache/structuredcachestore.cpp | 71 +++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 19 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index da948fd72..f582b61d8 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2130,12 +2130,16 @@ ZenCacheDiskLayer::TotalSize() const static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc"; -ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc) +ZenCacheStore::ZenCacheStore(CasGc& Gc, const Configuration& Configuration) +: GcStorage(Gc) +, GcContributor(Gc) +, m_Gc(Gc) +, m_Configuration(Configuration) { - CreateDirectories(BasePath); + CreateDirectories(m_Configuration.BasePath); DirectoryContent DirContent; - GetDirectoryContent(BasePath, DirectoryContent::IncludeDirsFlag, DirContent); + GetDirectoryContent(m_Configuration.BasePath, DirectoryContent::IncludeDirsFlag, DirContent); std::vector LegacyBuckets; std::vector Namespaces; @@ -2150,7 +2154,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor LegacyBuckets.push_back(DirName); } - ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size()); + ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), m_Configuration.BasePath, LegacyBuckets.size()); if (std::find(Namespaces.begin(), Namespaces.end(), UE4DDCNamespaceName) == Namespaces.end()) { @@ -2158,13 +2162,14 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor ZEN_INFO("Moving #{} legacy buckets to '{}' namespace", LegacyBuckets.size(), UE4DDCNamespaceName); - std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", NamespaceDiskPrefix, UE4DDCNamespaceName); + std::filesystem::path DefaultNamespaceFolder = + m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, UE4DDCNamespaceName); CreateDirectories(DefaultNamespaceFolder); // Move any non-namespace folders into the default namespace folder for (const std::string& DirName : LegacyBuckets) { - std::filesystem::path LegacyFolder = BasePath / DirName; + std::filesystem::path LegacyFolder = m_Configuration.BasePath / DirName; std::filesystem::path NewPath = DefaultNamespaceFolder / DirName; std::error_code Ec; std::filesystem::rename(LegacyFolder, NewPath, Ec); @@ -2179,7 +2184,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor for (const std::string& NamespaceName : Namespaces) { m_Namespaces[NamespaceName] = - std::make_unique(Gc, BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName)); + std::make_unique(Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName)); } } @@ -2247,7 +2252,23 @@ ZenCacheStore::GetNamespace(std::string_view Namespace) return It->second.get(); } } - return nullptr; + _.ReleaseNow(); + + if (!m_Configuration.AllowAutomaticCreationOfNamespaces) + { + return nullptr; + } + + RwLock::ExclusiveLockScope __(m_NamespacesLock); + if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end()) + { + return It->second.get(); + } + + auto NewNamespace = m_Namespaces.insert_or_assign( + std::string(Namespace), + std::make_unique(m_Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace))); + return NewNamespace.first->second.get(); } void @@ -3048,40 +3069,52 @@ TEST_CASE("z$.namespaces") ScopedTemporaryDirectory TempDir; CreateDirectories(TempDir.Path()); + IoHash Key1; + IoHash Key2; { CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = false}); const auto Bucket = "teardrinker"sv; const auto CustomNamespace = "mynamespace"sv; // Create a cache record - const IoHash Key = CreateKey(42); - CbObject CacheValue = CreateCacheValue(4096); + Key1 = CreateKey(42); + CbObject CacheValue = CreateCacheValue(4096); IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); Buffer.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue = {.Value = Buffer}; - Zcs.Put(ZenCacheStore::DefaultNamespace, Bucket, Key, PutValue); + Zcs.Put(ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue); ZenCacheValue GetValue; - CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key, GetValue)); + CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue)); + CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue)); - CHECK(!Zcs.Get(CustomNamespace, Bucket, Key, GetValue)); + // This should just be dropped as we don't allow creating of namespaces on the fly + Zcs.Put(CustomNamespace, Bucket, Key1, PutValue); + CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue)); + } - // This should just be dropped for now until we decide how we add namespaces - Zcs.Put(CustomNamespace, Bucket, Key, PutValue); - CHECK(!Zcs.Get(CustomNamespace, Bucket, Key, GetValue)); + { + CasGc Gc; + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}); + const auto Bucket = "teardrinker"sv; + const auto CustomNamespace = "mynamespace"sv; - const IoHash Key2 = CreateKey(43); - CbObject CacheValue2 = CreateCacheValue(4096); + Key2 = CreateKey(43); + CbObject CacheValue2 = CreateCacheValue(4096); IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer(); Buffer2.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue2 = {.Value = Buffer2}; Zcs.Put(CustomNamespace, Bucket, Key2, PutValue2); + ZenCacheValue GetValue; CHECK(!Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key2, GetValue)); + CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue)); + CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue)); + CHECK(Zcs.Get(CustomNamespace, Bucket, Key2, GetValue)); } } -- cgit v1.2.3 From dba8b362221188c4e3125e39eb0654613a7d3dd2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 23 May 2022 14:44:48 +0200 Subject: De/fix namespace folder scanning (#103) --- zenserver/cache/structuredcachestore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index da948fd72..7509c5a71 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2141,7 +2141,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor std::vector Namespaces; for (const std::filesystem::path& DirPath : DirContent.Directories) { - std::string DirName = PathToUtf8(DirPath.stem()); + std::string DirName = PathToUtf8(DirPath.filename()); if (DirName.starts_with(NamespaceDiskPrefix)) { Namespaces.push_back(DirName.substr(NamespaceDiskPrefix.length())); -- cgit v1.2.3 From 898a48b5121ed09879e34dc3315387d927a91ebf Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 13:24:51 +0200 Subject: Make sure to hold exclusive lock over index and all shard locks. Clear index on drop. --- zenserver/cache/structuredcachestore.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 3189a14cc..1db99280e 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1199,8 +1199,16 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& void ZenCacheDiskLayer::CacheBucket::Drop() { + RwLock::ExclusiveLockScope _(m_IndexLock); + std::vector> ShardLocks; + ShardLocks.reserve(256); + for (RwLock& Lock : m_ShardedLocks) + { + ShardLocks.push_back(std::make_unique(Lock)); + } m_BlockStore.Close(); m_SlogFile.Close(); + m_Index.clear(); DeleteDirectories(m_BucketDir); } @@ -2274,7 +2282,7 @@ ZenCacheStore::GetNamespace(std::string_view Namespace) void ZenCacheStore::IterateNamespaces(const std::function& Callback) const { - std::vector > Namespaces; + std::vector> Namespaces; { RwLock::SharedLockScope _(m_NamespacesLock); Namespaces.reserve(m_Namespaces.size()); -- cgit v1.2.3 From ba135d4a90746ed905a825f48fc3ccfe0641301e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 13:52:45 +0200 Subject: Use rename/delete and keep pointer for dropped buckets --- zenserver/cache/structuredcachestore.cpp | 157 ++++++++++++++++++++----------- 1 file changed, 104 insertions(+), 53 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 1db99280e..be9e408b4 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -580,19 +580,6 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket() { } -bool -ZenCacheDiskLayer::CacheBucket::Delete(std::filesystem::path BucketDir) -{ - if (std::filesystem::exists(BucketDir)) - { - DeleteDirectories(BucketDir); - - return true; - } - - return false; -} - void ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate) { @@ -1158,11 +1145,6 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, bool ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue) { - if (!m_IsOk) - { - return false; - } - RwLock::SharedLockScope _(m_IndexLock); auto It = m_Index.find(HashKey); if (It == m_Index.end()) @@ -1184,11 +1166,6 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal void ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value) { - if (!m_IsOk) - { - return; - } - if (Value.Value.Size() >= m_LargeObjectThreshold) { return PutStandaloneCacheValue(HashKey, Value); @@ -1196,10 +1173,43 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& PutInlineCacheValue(HashKey, Value); } +static bool +DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view BucketName) +{ + int dropIndex = 0; + do + { + if (!std::filesystem::exists(BucketDir)) + { + return false; + } + + std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, dropIndex); + std::filesystem::path DroppedBucketPath = BucketDir.parent_path() / DroppedBucketName; + if (std::filesystem::exists(DroppedBucketPath)) + { + dropIndex++; + continue; + } + + std::error_code Ec; + std::filesystem::rename(BucketDir, DroppedBucketPath, Ec); + if (!Ec) + { + DeleteDirectories(DroppedBucketPath); + return true; + } + // TODO: Do we need to bail at some point? + zen::Sleep(100); + } while (true); +} + void ZenCacheDiskLayer::CacheBucket::Drop() { - RwLock::ExclusiveLockScope _(m_IndexLock); + RwLock::ExclusiveLockScope _(m_IndexLock); + m_IsOk = false; + std::vector> ShardLocks; ShardLocks.reserve(256); for (RwLock& Lock : m_ShardedLocks) @@ -1208,8 +1218,10 @@ ZenCacheDiskLayer::CacheBucket::Drop() } m_BlockStore.Close(); m_SlogFile.Close(); + + DeleteBucketFromDisk(m_BucketDir, m_BucketName); + m_Index.clear(); - DeleteDirectories(m_BucketDir); } void @@ -1711,7 +1723,12 @@ ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx) for (auto& Kv : m_Buckets) { - Kv.second.CollectGarbage(GcCtx); + CacheBucket& Bucket = *Kv.second; + if (!Bucket.IsOk()) + { + continue; + } + Bucket.CollectGarbage(GcCtx); } } @@ -1724,7 +1741,11 @@ ZenCacheDiskLayer::UpdateAccessTimes(const zen::access_tracking::AccessTimes& Ac { if (auto It = m_Buckets.find(Kv.first); It != m_Buckets.end()) { - CacheBucket& Bucket = It->second; + CacheBucket& Bucket = *It->second; + if (!Bucket.IsOk()) + { + continue; + } Bucket.UpdateAccessTimes(Kv.second); } } @@ -1937,7 +1958,11 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach if (it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); + if (!Bucket->IsOk()) + { + return false; + } } } @@ -1949,22 +1974,25 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); } else { - auto It = m_Buckets.try_emplace(BucketName, BucketName); - Bucket = &It.first->second; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + Bucket = InsertResult.first->second.get(); std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; Bucket->OpenOrCreate(BucketPath); + if (!Bucket->IsOk()) + { + return false; + } } } ZEN_ASSERT(Bucket != nullptr); - return Bucket->Get(HashKey, OutValue); } @@ -1981,7 +2009,11 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z if (it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); + if (!Bucket->IsOk()) + { + return; + } } } @@ -1993,26 +2025,27 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); } else { - auto It = m_Buckets.try_emplace(BucketName, BucketName); - Bucket = &It.first->second; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + Bucket = InsertResult.first->second.get(); std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; Bucket->OpenOrCreate(BucketPath); + if (!Bucket->IsOk()) + { + return; + } } } ZEN_ASSERT(Bucket != nullptr); - if (Bucket->IsOk()) - { - Bucket->Put(HashKey, Value); - } + Bucket->Put(HashKey, Value); } void @@ -2034,9 +2067,8 @@ ZenCacheDiskLayer::DiscoverBuckets() } else { - auto InsertResult = m_Buckets.try_emplace(BucketName, BucketName); - - CacheBucket& Bucket = InsertResult.first->second; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + CacheBucket& Bucket = *InsertResult.first->second; Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false); @@ -2063,19 +2095,23 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) if (it != m_Buckets.end()) { - CacheBucket* Bucket = &it->second; - - Bucket->Drop(); - + CacheBucket& Bucket = *it->second; + m_DroppedBuckets.push_back(std::move(it->second)); m_Buckets.erase(it); + if (!Bucket.IsOk()) + { + return false; + } + + Bucket.Drop(); return true; } + // Make sure we remove the folder even if we don't know about the bucket std::filesystem::path BucketPath = m_RootDir; BucketPath /= std::string(InBucket); - - return CacheBucket::Delete(BucketPath); + return DeleteBucketFromDisk(BucketPath, InBucket); } void @@ -2088,7 +2124,12 @@ ZenCacheDiskLayer::Flush() Buckets.reserve(m_Buckets.size()); for (auto& Kv : m_Buckets) { - Buckets.push_back(&Kv.second); + CacheBucket* Bucket = Kv.second.get(); + if (!Bucket->IsOk()) + { + continue; + } + Buckets.push_back(Bucket); } } @@ -2105,7 +2146,12 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) for (auto& Kv : m_Buckets) { - Kv.second.Scrub(Ctx); + CacheBucket& Bucket = *Kv.second; + if (!Bucket.IsOk()) + { + continue; + } + Bucket.Scrub(Ctx); } } @@ -2116,7 +2162,12 @@ ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx) for (auto& Kv : m_Buckets) { - Kv.second.GatherReferences(GcCtx); + CacheBucket& Bucket = *Kv.second; + if (!Bucket.IsOk()) + { + continue; + } + Bucket.GatherReferences(GcCtx); } } @@ -2128,7 +2179,7 @@ ZenCacheDiskLayer::TotalSize() const for (auto& Kv : m_Buckets) { - TotalSize += Kv.second.TotalSize(); + TotalSize += Kv.second->TotalSize(); } return TotalSize; -- cgit v1.2.3 From 4d1faaa2076bff77734d152b1403c3c90884bc83 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 23:22:52 +0200 Subject: drop bucket test --- zenserver/cache/structuredcachestore.cpp | 73 ++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index be9e408b4..f3f6503f3 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -3177,6 +3177,79 @@ TEST_CASE("z$.namespaces") } } +TEST_CASE("z$.drop.bucket") +{ + using namespace testutils; + + const auto CreateCacheValue = [](size_t Size) -> CbObject { + std::vector Buf; + Buf.resize(Size); + + CbObjectWriter Writer; + Writer.AddBinary("Binary"sv, Buf.data(), Buf.size()); + return Writer.Save(); + }; + + ScopedTemporaryDirectory TempDir; + CreateDirectories(TempDir.Path()); + + IoHash Key1; + IoHash Key2; + + auto PutValue = + [&CreateCacheValue](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, size_t KeyIndex, size_t Size) { + // Create a cache record + IoHash Key = CreateKey(KeyIndex); + CbObject CacheValue = CreateCacheValue(Size); + + IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); + Buffer.SetContentType(ZenContentType::kCbObject); + + ZenCacheValue PutValue = {.Value = Buffer}; + Zcs.Put(Namespace, Bucket, Key, PutValue); + return Key; + }; + auto GetValue = [](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) { + ZenCacheValue GetValue; + Zcs.Get(Namespace, Bucket, Key, GetValue); + return GetValue; + }; + WorkerThreadPool Workers(1); + { + CasGc Gc; + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}); + const auto Bucket = "teardrinker"sv; + const auto Namespace = "mynamespace"sv; + + Key1 = PutValue(Zcs, Namespace, Bucket, 42, 4096); + Key2 = PutValue(Zcs, Namespace, Bucket, 43, 2048); + + ZenCacheValue Value1 = GetValue(Zcs, Namespace, Bucket, Key1); + CHECK(Value1.Value); + + std::atomic_bool WorkComplete = false; + Workers.ScheduleWork([&]() { + zen::Sleep(100); + Value1.Value = IoBuffer{}; + Value1 = GetValue(Zcs, Namespace, Bucket, Key1); + CHECK(!Value1.Value); + WorkComplete = true; + }); + // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket + // Our DropBucket execution blocks any incoming request from completing until we are done with the drop + Zcs.DropBucket(Namespace, Bucket); + while (!WorkComplete) + { + zen::Sleep(1); + } + CHECK(!Value1.Value); + + // Entire bucket should be dropped, but doing a request should will re-create the namespace but it must still be empty + ZenCacheValue Value2 = GetValue(Zcs, Namespace, Bucket, Key2); + CHECK(!Value2.Value); + } +} + TEST_CASE("z$.blocked.disklayer.put") { ScopedTemporaryDirectory TempDir; -- cgit v1.2.3 From 49ad6da1f77daa8c91a087046d82ab78d2e41314 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 12:34:51 +0200 Subject: If a bucket is in m_BucketMap it is OK, no need for separate flag --- zenserver/cache/structuredcachestore.cpp | 77 +++++++++----------------------- 1 file changed, 21 insertions(+), 56 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index f3f6503f3..bb85f9824 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -572,7 +572,7 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue ////////////////////////////////////////////////////////////////////////// -ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)) +ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)), m_BucketId(Oid::Zero) { } @@ -580,7 +580,7 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket() { } -void +bool ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate) { using namespace std::literals; @@ -598,7 +598,10 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo if (Manifest) { m_BucketId = Manifest["BucketId"].AsObjectId(); - m_IsOk = m_BucketId != Oid::Zero; + if (m_BucketId == Oid::Zero) + { + return false; + } } else if (AllowCreate) { @@ -612,7 +615,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo } else { - return; + return false; } OpenLog(BucketDir, IsNew); @@ -628,7 +631,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo } } - m_IsOk = true; + return true; } void @@ -1208,7 +1211,6 @@ void ZenCacheDiskLayer::CacheBucket::Drop() { RwLock::ExclusiveLockScope _(m_IndexLock); - m_IsOk = false; std::vector> ShardLocks; ShardLocks.reserve(256); @@ -1724,10 +1726,6 @@ ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx) for (auto& Kv : m_Buckets) { CacheBucket& Bucket = *Kv.second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.CollectGarbage(GcCtx); } } @@ -1742,10 +1740,6 @@ ZenCacheDiskLayer::UpdateAccessTimes(const zen::access_tracking::AccessTimes& Ac if (auto It = m_Buckets.find(Kv.first); It != m_Buckets.end()) { CacheBucket& Bucket = *It->second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.UpdateAccessTimes(Kv.second); } } @@ -1959,10 +1953,6 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach if (it != m_Buckets.end()) { Bucket = it->second.get(); - if (!Bucket->IsOk()) - { - return false; - } } } @@ -1984,9 +1974,9 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; - Bucket->OpenOrCreate(BucketPath); - if (!Bucket->IsOk()) + if (!Bucket->OpenOrCreate(BucketPath)) { + m_Buckets.erase(BucketName); return false; } } @@ -2010,10 +2000,6 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z if (it != m_Buckets.end()) { Bucket = it->second.get(); - if (!Bucket->IsOk()) - { - return; - } } } @@ -2035,9 +2021,9 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; - Bucket->OpenOrCreate(BucketPath); - if (!Bucket->IsOk()) + if (!Bucket->OpenOrCreate(BucketPath)) { + m_Buckets.erase(BucketName); return; } } @@ -2064,25 +2050,20 @@ ZenCacheDiskLayer::DiscoverBuckets() // New bucket needs to be created if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) { + continue; } - else - { - auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); - CacheBucket& Bucket = *InsertResult.first->second; - Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false); + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + CacheBucket& Bucket = *InsertResult.first->second; - if (Bucket.IsOk()) - { - ZEN_INFO("Discovered bucket '{}'", BucketName); - } - else - { - ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); + if (!Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false)) + { + ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); - m_Buckets.erase(InsertResult.first); - } + m_Buckets.erase(InsertResult.first); + continue; } + ZEN_INFO("Discovered bucket '{}'", BucketName); } } @@ -2098,10 +2079,6 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) CacheBucket& Bucket = *it->second; m_DroppedBuckets.push_back(std::move(it->second)); m_Buckets.erase(it); - if (!Bucket.IsOk()) - { - return false; - } Bucket.Drop(); @@ -2125,10 +2102,6 @@ ZenCacheDiskLayer::Flush() for (auto& Kv : m_Buckets) { CacheBucket* Bucket = Kv.second.get(); - if (!Bucket->IsOk()) - { - continue; - } Buckets.push_back(Bucket); } } @@ -2147,10 +2120,6 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) for (auto& Kv : m_Buckets) { CacheBucket& Bucket = *Kv.second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.Scrub(Ctx); } } @@ -2163,10 +2132,6 @@ ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx) for (auto& Kv : m_Buckets) { CacheBucket& Bucket = *Kv.second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.GatherReferences(GcCtx); } } -- cgit v1.2.3 From 0e325ea21a56a154bdf4b92745095c588f69c817 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 12:39:59 +0200 Subject: dropIndex -> DropIndex --- zenserver/cache/structuredcachestore.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index bb85f9824..9ebec1e76 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1179,7 +1179,7 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& static bool DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view BucketName) { - int dropIndex = 0; + int DropIndex = 0; do { if (!std::filesystem::exists(BucketDir)) @@ -1187,11 +1187,11 @@ DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view Bu return false; } - std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, dropIndex); + std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, DropIndex); std::filesystem::path DroppedBucketPath = BucketDir.parent_path() / DroppedBucketName; if (std::filesystem::exists(DroppedBucketPath)) { - dropIndex++; + DropIndex++; continue; } -- cgit v1.2.3 From d0c46ee88f3b08c0abc4f1cfba40b966f3c6a59e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 13:49:58 +0200 Subject: Make sure ZenCacheMemoryLayer handles dropped buckets correctly (just like ZenCacheDiskLayer) --- zenserver/cache/structuredcachestore.cpp | 119 ++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 32 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 9ebec1e76..d4b8bff39 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -404,14 +404,14 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa { RwLock::SharedLockScope _(m_Lock); - auto it = m_Buckets.find(std::string(InBucket)); + auto It = m_Buckets.find(std::string(InBucket)); - if (it == m_Buckets.end()) + if (It == m_Buckets.end()) { return false; } - CacheBucket* Bucket = &it->second; + CacheBucket* Bucket = It->second.get(); _.ReleaseNow(); @@ -425,14 +425,15 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa void ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) { - CacheBucket* Bucket = nullptr; + const auto BucketName = std::string(InBucket); + CacheBucket* Bucket = nullptr; { RwLock::SharedLockScope _(m_Lock); if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end()) { - Bucket = &It->second; + Bucket = It->second.get(); } } @@ -444,11 +445,12 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end()) { - Bucket = &It->second; + Bucket = It->second.get(); } else { - Bucket = &m_Buckets[std::string(InBucket)]; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique()); + Bucket = InsertResult.first->second.get(); } } @@ -458,11 +460,37 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const } bool -ZenCacheMemoryLayer::DropBucket(std::string_view Bucket) +ZenCacheMemoryLayer::DropBucket(std::string_view InBucket) { RwLock::ExclusiveLockScope _(m_Lock); - return !!m_Buckets.erase(std::string(Bucket)); + auto It = m_Buckets.find(std::string(InBucket)); + + if (It != m_Buckets.end()) + { + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It); + Bucket.Drop(); + return true; + } + return false; +} + +void +ZenCacheMemoryLayer::Drop() +{ + RwLock::ExclusiveLockScope _(m_Lock); + std::vector> Buckets; + Buckets.reserve(m_Buckets.size()); + while (!m_Buckets.empty()) + { + const auto& It = m_Buckets.begin(); + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It->first); + Bucket.Drop(); + } } void @@ -472,7 +500,7 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx) for (auto& Kv : m_Buckets) { - Kv.second.Scrub(Ctx); + Kv.second->Scrub(Ctx); } } @@ -486,7 +514,7 @@ ZenCacheMemoryLayer::GatherAccessTimes(zen::access_tracking::AccessTimes& Access for (auto& Kv : m_Buckets) { std::vector& Bucket = AccessTimes.Buckets[Kv.first]; - Kv.second.GatherAccessTimes(Bucket); + Kv.second->GatherAccessTimes(Bucket); } } @@ -505,7 +533,7 @@ ZenCacheMemoryLayer::TotalSize() const for (auto& Kv : m_Buckets) { - TotalSize += Kv.second.TotalSize(); + TotalSize += Kv.second->TotalSize(); } return TotalSize; @@ -570,6 +598,13 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue m_TotalSize.fetch_add(Value.Value.GetSize(), std::memory_order::relaxed); } +void +ZenCacheMemoryLayer::CacheBucket::Drop() +{ + RwLock::ExclusiveLockScope _(m_BucketLock); + m_CacheMap.clear(); +} + ////////////////////////////////////////////////////////////////////////// ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)), m_BucketId(Oid::Zero) @@ -1207,7 +1242,7 @@ DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view Bu } while (true); } -void +bool ZenCacheDiskLayer::CacheBucket::Drop() { RwLock::ExclusiveLockScope _(m_IndexLock); @@ -1221,9 +1256,10 @@ ZenCacheDiskLayer::CacheBucket::Drop() m_BlockStore.Close(); m_SlogFile.Close(); - DeleteBucketFromDisk(m_BucketDir, m_BucketName); + bool Deleted = DeleteBucketFromDisk(m_BucketDir, m_BucketName); m_Index.clear(); + return Deleted; } void @@ -1948,11 +1984,11 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach { RwLock::SharedLockScope _(m_Lock); - auto it = m_Buckets.find(BucketName); + auto It = m_Buckets.find(BucketName); - if (it != m_Buckets.end()) + if (It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } } @@ -1962,9 +1998,9 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach RwLock::ExclusiveLockScope _(m_Lock); - if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) + if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } else { @@ -1995,11 +2031,11 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z { RwLock::SharedLockScope _(m_Lock); - auto it = m_Buckets.find(BucketName); + auto It = m_Buckets.find(BucketName); - if (it != m_Buckets.end()) + if (It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } } @@ -2009,9 +2045,9 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z RwLock::ExclusiveLockScope _(m_Lock); - if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) + if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } else { @@ -2072,17 +2108,15 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) { RwLock::ExclusiveLockScope _(m_Lock); - auto it = m_Buckets.find(std::string(InBucket)); + auto It = m_Buckets.find(std::string(InBucket)); - if (it != m_Buckets.end()) + if (It != m_Buckets.end()) { - CacheBucket& Bucket = *it->second; - m_DroppedBuckets.push_back(std::move(it->second)); - m_Buckets.erase(it); + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It); - Bucket.Drop(); - - return true; + return Bucket.Drop(); } // Make sure we remove the folder even if we don't know about the bucket @@ -2091,6 +2125,27 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) return DeleteBucketFromDisk(BucketPath, InBucket); } +bool +ZenCacheDiskLayer::Drop() +{ + RwLock::ExclusiveLockScope _(m_Lock); + + std::vector> Buckets; + Buckets.reserve(m_Buckets.size()); + while (!m_Buckets.empty()) + { + const auto& It = m_Buckets.begin(); + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It->first); + if (!Bucket.Drop()) + { + return false; + } + } + return true; +} + void ZenCacheDiskLayer::Flush() { -- cgit v1.2.3 From 5feb2725a3c6461e913a95ea271046855c773ac5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 14:08:11 +0200 Subject: namespace drop --- zenserver/cache/structuredcachestore.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index d4b8bff39..6af673ebc 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -342,6 +342,14 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket) return AnyDropped; } +bool +ZenCacheNamespace::Drop() +{ + m_MemLayer.Drop(); + const bool DiskDropped = m_DiskLayer.Drop(); + return DiskDropped; +} + void ZenCacheNamespace::Flush() { @@ -2300,7 +2308,22 @@ ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket) { return Store->DropBucket(Bucket); } - ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}'", Namespace, Bucket); + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropBucket, bucket '{}'", Namespace, Bucket); + return false; +} + +bool +ZenCacheStore::DropNamespace(std::string_view InNamespace) +{ + RwLock::SharedLockScope _(m_NamespacesLock); + if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end()) + { + ZenCacheNamespace& Namespace = *It->second; + m_DroppedNamespaces.push_back(std::move(It->second)); + m_Namespaces.erase(It); + return Namespace.Drop(); + } + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); return false; } -- cgit v1.2.3 From 1a3709fd35f592b146d794835ab0e1a95ee078e0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 14:17:59 +0200 Subject: clean up namespace folders --- zenserver/cache/structuredcachestore.cpp | 78 +++++++++++++++++--------------- 1 file changed, 42 insertions(+), 36 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 6af673ebc..aa3be3974 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -205,6 +205,36 @@ namespace { return true; } + bool MoveAndDeleteDirectory(const std::filesystem::path& Dir) + { + int DropIndex = 0; + do + { + if (!std::filesystem::exists(Dir)) + { + return false; + } + + std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); + std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName; + if (std::filesystem::exists(DroppedBucketPath)) + { + DropIndex++; + continue; + } + + std::error_code Ec; + std::filesystem::rename(Dir, DroppedBucketPath, Ec); + if (!Ec) + { + DeleteDirectories(DroppedBucketPath); + return true; + } + // TODO: Do we need to bail at some point? + zen::Sleep(100); + } while (true); + } + } // namespace namespace fs = std::filesystem; @@ -346,8 +376,11 @@ bool ZenCacheNamespace::Drop() { m_MemLayer.Drop(); - const bool DiskDropped = m_DiskLayer.Drop(); - return DiskDropped; + if (!m_DiskLayer.Drop()) + { + return false; + } + return MoveAndDeleteDirectory(m_RootDir); } void @@ -1219,37 +1252,6 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& PutInlineCacheValue(HashKey, Value); } -static bool -DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view BucketName) -{ - int DropIndex = 0; - do - { - if (!std::filesystem::exists(BucketDir)) - { - return false; - } - - std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, DropIndex); - std::filesystem::path DroppedBucketPath = BucketDir.parent_path() / DroppedBucketName; - if (std::filesystem::exists(DroppedBucketPath)) - { - DropIndex++; - continue; - } - - std::error_code Ec; - std::filesystem::rename(BucketDir, DroppedBucketPath, Ec); - if (!Ec) - { - DeleteDirectories(DroppedBucketPath); - return true; - } - // TODO: Do we need to bail at some point? - zen::Sleep(100); - } while (true); -} - bool ZenCacheDiskLayer::CacheBucket::Drop() { @@ -1264,7 +1266,7 @@ ZenCacheDiskLayer::CacheBucket::Drop() m_BlockStore.Close(); m_SlogFile.Close(); - bool Deleted = DeleteBucketFromDisk(m_BucketDir, m_BucketName); + bool Deleted = MoveAndDeleteDirectory(m_BucketDir); m_Index.clear(); return Deleted; @@ -2130,7 +2132,7 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) // Make sure we remove the folder even if we don't know about the bucket std::filesystem::path BucketPath = m_RootDir; BucketPath /= std::string(InBucket); - return DeleteBucketFromDisk(BucketPath, InBucket); + return MoveAndDeleteDirectory(BucketPath); } bool @@ -2150,6 +2152,7 @@ ZenCacheDiskLayer::Drop() { return false; } + return MoveAndDeleteDirectory(m_RootDir); } return true; } @@ -2321,7 +2324,10 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace) ZenCacheNamespace& Namespace = *It->second; m_DroppedNamespaces.push_back(std::move(It->second)); m_Namespaces.erase(It); - return Namespace.Drop(); + if (!Namespace.Drop()) + { + return false; + } } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); return false; -- cgit v1.2.3 From 91283d188c6b954874d888714d54071414364b53 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 14:39:40 +0200 Subject: bugfixes and test for namespace drop --- zenserver/cache/structuredcachestore.cpp | 107 ++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 15 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index aa3be3974..e4dbf390a 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -376,11 +376,7 @@ bool ZenCacheNamespace::Drop() { m_MemLayer.Drop(); - if (!m_DiskLayer.Drop()) - { - return false; - } - return MoveAndDeleteDirectory(m_RootDir); + return m_DiskLayer.Drop(); } void @@ -2152,9 +2148,8 @@ ZenCacheDiskLayer::Drop() { return false; } - return MoveAndDeleteDirectory(m_RootDir); } - return true; + return MoveAndDeleteDirectory(m_RootDir); } void @@ -2324,10 +2319,7 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace) ZenCacheNamespace& Namespace = *It->second; m_DroppedNamespaces.push_back(std::move(It->second)); m_Namespaces.erase(It); - if (!Namespace.Drop()) - { - return false; - } + return Namespace.Drop(); } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); return false; @@ -3280,25 +3272,110 @@ TEST_CASE("z$.drop.bucket") Workers.ScheduleWork([&]() { zen::Sleep(100); Value1.Value = IoBuffer{}; - Value1 = GetValue(Zcs, Namespace, Bucket, Key1); - CHECK(!Value1.Value); WorkComplete = true; }); // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket // Our DropBucket execution blocks any incoming request from completing until we are done with the drop - Zcs.DropBucket(Namespace, Bucket); + CHECK(Zcs.DropBucket(Namespace, Bucket)); while (!WorkComplete) { zen::Sleep(1); } - CHECK(!Value1.Value); // Entire bucket should be dropped, but doing a request should will re-create the namespace but it must still be empty + Value1 = GetValue(Zcs, Namespace, Bucket, Key1); + CHECK(!Value1.Value); ZenCacheValue Value2 = GetValue(Zcs, Namespace, Bucket, Key2); CHECK(!Value2.Value); } } +TEST_CASE("z$.drop.namespace") +{ + using namespace testutils; + + const auto CreateCacheValue = [](size_t Size) -> CbObject { + std::vector Buf; + Buf.resize(Size); + + CbObjectWriter Writer; + Writer.AddBinary("Binary"sv, Buf.data(), Buf.size()); + return Writer.Save(); + }; + + ScopedTemporaryDirectory TempDir; + CreateDirectories(TempDir.Path()); + + auto PutValue = + [&CreateCacheValue](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, size_t KeyIndex, size_t Size) { + // Create a cache record + IoHash Key = CreateKey(KeyIndex); + CbObject CacheValue = CreateCacheValue(Size); + + IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); + Buffer.SetContentType(ZenContentType::kCbObject); + + ZenCacheValue PutValue = {.Value = Buffer}; + Zcs.Put(Namespace, Bucket, Key, PutValue); + return Key; + }; + auto GetValue = [](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) { + ZenCacheValue GetValue; + Zcs.Get(Namespace, Bucket, Key, GetValue); + return GetValue; + }; + WorkerThreadPool Workers(1); + { + CasGc Gc; + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}); + const auto Bucket1 = "teardrinker1"sv; + const auto Bucket2 = "teardrinker2"sv; + const auto Namespace1 = "mynamespace1"sv; + const auto Namespace2 = "mynamespace2"sv; + + IoHash Key1 = PutValue(Zcs, Namespace1, Bucket1, 42, 4096); + IoHash Key2 = PutValue(Zcs, Namespace1, Bucket2, 43, 2048); + IoHash Key3 = PutValue(Zcs, Namespace2, Bucket1, 44, 4096); + IoHash Key4 = PutValue(Zcs, Namespace2, Bucket2, 45, 2048); + + ZenCacheValue Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1); + CHECK(Value1.Value); + ZenCacheValue Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2); + CHECK(Value2.Value); + ZenCacheValue Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3); + CHECK(Value3.Value); + ZenCacheValue Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4); + CHECK(Value4.Value); + + std::atomic_bool WorkComplete = false; + Workers.ScheduleWork([&]() { + zen::Sleep(100); + Value1.Value = IoBuffer{}; + Value2.Value = IoBuffer{}; + Value3.Value = IoBuffer{}; + Value4.Value = IoBuffer{}; + WorkComplete = true; + }); + // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket + // Our DropBucket execution blocks any incoming request from completing until we are done with the drop + CHECK(Zcs.DropNamespace(Namespace1)); + while (!WorkComplete) + { + zen::Sleep(1); + } + + // Entire namespace should be dropped, but doing a request should will re-create the namespace but it must still be empty + Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1); + CHECK(!Value1.Value); + Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2); + CHECK(!Value2.Value); + Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3); + CHECK(Value3.Value); + Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4); + CHECK(Value4.Value); + } +} + TEST_CASE("z$.blocked.disklayer.put") { ScopedTemporaryDirectory TempDir; -- cgit v1.2.3 From da0ca23fa55f7e900853bdfd06523b78fce32259 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 28 May 2022 00:29:17 +0200 Subject: Enable FILE_SHARE_DELETE on standalone files in disk buckets This allows us to delete the file even if it is open for read. We do a delete, the rename since we are not allowed to do a rename-overwrite, only delete. As we have the shard lock for the file we want to replace we can safely do a delete+rename. In the rare case that we fail to rename the file into place the old data is lost. As this is a *cache* and it should be very rare this is OK. --- zenserver/cache/structuredcachestore.cpp | 184 +++++++++++++------------------ 1 file changed, 77 insertions(+), 107 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index e4dbf390a..d1b242c5c 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1206,7 +1206,7 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); - if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath())) + if (IoBuffer Data = IoBufferBuilder::MakeFromFileWithSharedDelete(DataFilePath.ToPath())) { OutValue.Value = Data; OutValue.Value.SetContentType(Loc.GetContentType()); @@ -1831,110 +1831,79 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c BuildPath(DataFilePath, HashKey); std::filesystem::path FsPath{DataFilePath.ToPath()}; - // We retry to move the file since it can be held open for read. - // This happens if the server processes a Get request for the file or - // if we are busy sending the file upstream - int RetryCount = 4; - do - { - Ec.clear(); - { - RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); - - DataFile.MoveTemporaryIntoPlace(FsPath, Ec); - - // Once we have called MoveTemporaryIntoPlace automatic clean up the temp file - // will be disabled as the file handle has already been closed - CleanUpTempFile = Ec ? true : false; + RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); - if (Ec) - { - std::error_code ExistingEc; - uint64_t OldFileSize = std::filesystem::file_size(FsPath, ExistingEc); - if (!ExistingEc && (OldFileSize == NewFileSize)) - { - ZEN_INFO( - "Failed to move temporary file '{}' to '{}' for '{}'. Target file has same size, assuming concurrent write of same " - "value, " - "move " - "failed with reason '{}'", - DataFile.GetPath(), - FsPath.string(), - m_BucketDir, - Ec.message()); - return; - } - } - } + std::filesystem::remove(FsPath, Ec); + if (Ec && Ec.value() != ENOENT) + { + throw std::system_error(Ec, fmt::format("Failed to replace file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); + } - if (!Ec) + DataFile.MoveTemporaryIntoPlace(FsPath, Ec); + if (Ec) + { + std::filesystem::path ParentPath = FsPath.parent_path(); + if (std::filesystem::is_directory(ParentPath)) { - uint8_t EntryFlags = DiskLocation::kStandaloneFile; - - if (Value.Value.GetContentType() == ZenContentType::kCbObject) - { - EntryFlags |= DiskLocation::kStructured; - } - else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary) - { - EntryFlags |= DiskLocation::kCompressed; - } - - DiskLocation Loc(NewFileSize, EntryFlags); - IndexEntry Entry = IndexEntry(Loc, GcClock::TickCount()); - - uint64_t OldFileSize = 0; - RwLock::ExclusiveLockScope _(m_IndexLock); - if (auto It = m_Index.find(HashKey); It == m_Index.end()) - { - // Previously unknown object - m_Index.insert({HashKey, Entry}); - } - else - { - // TODO: should check if write is idempotent and bail out if it is? - OldFileSize = It.value().Location.Size(); - It.value() = Entry; - } - - m_SlogFile.Append({.Key = HashKey, .Location = Loc}); - if (OldFileSize <= NewFileSize) - { - m_TotalSize.fetch_add(NewFileSize - OldFileSize, std::memory_order::relaxed); - } - else - { - m_TotalSize.fetch_sub(OldFileSize - NewFileSize, std::memory_order::relaxed); - } - return; + throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); } - - std::filesystem::path ParentPath = FsPath.parent_path(); - if (!std::filesystem::is_directory(ParentPath)) + Ec.clear(); + std::filesystem::create_directories(ParentPath, Ec); + if (Ec) { - Ec.clear(); - std::filesystem::create_directories(ParentPath, Ec); - if (!Ec) - { - // Retry without sleep - continue; - } throw std::system_error( Ec, fmt::format("Failed to create parent directory '{}' for file '{}' for put in '{}'", ParentPath, FsPath, m_BucketDir)); } - ZEN_INFO("Failed renaming temporary file '{}' to '{}' for put in '{}', pausing and retrying, reason '{}'", - DataFile.GetPath().string(), - FsPath.string(), - m_BucketDir, - Ec.message()); + DataFile.MoveTemporaryIntoPlace(FsPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); + } + } + + // Once we have called MoveTemporaryIntoPlace automatic clean up the temp file + // will be disabled as the file handle has already been closed + CleanUpTempFile = false; - // Semi arbitrary back-off - zen::Sleep(200 * (5 - RetryCount)); // Sleep at most for a total of 3 seconds - } while (RetryCount-- > 0); + uint8_t EntryFlags = DiskLocation::kStandaloneFile; - throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); + if (Value.Value.GetContentType() == ZenContentType::kCbObject) + { + EntryFlags |= DiskLocation::kStructured; + } + else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary) + { + EntryFlags |= DiskLocation::kCompressed; + } + + DiskLocation Loc(NewFileSize, EntryFlags); + IndexEntry Entry = IndexEntry(Loc, GcClock::TickCount()); + + uint64_t OldFileSize = 0; + RwLock::ExclusiveLockScope _(m_IndexLock); + if (auto It = m_Index.find(HashKey); It == m_Index.end()) + { + // Previously unknown object + m_Index.insert({HashKey, Entry}); + } + else + { + // TODO: should check if write is idempotent and bail out if it is? + OldFileSize = It.value().Location.Size(); + It.value() = Entry; + } + + m_SlogFile.Append({.Key = HashKey, .Location = Loc}); + if (OldFileSize <= NewFileSize) + { + m_TotalSize.fetch_add(NewFileSize - OldFileSize, std::memory_order::relaxed); + } + else + { + m_TotalSize.fetch_sub(OldFileSize - NewFileSize, std::memory_order::relaxed); + } } void @@ -3384,7 +3353,7 @@ TEST_CASE("z$.blocked.disklayer.put") const auto CreateCacheValue = [](size_t Size) -> CbObject { std::vector Buf; - Buf.resize(Size); + Buf.resize(Size, Size & 0xff); CbObjectWriter Writer; Writer.AddBinary("Binary"sv, Buf.data(), Buf.size()); @@ -3406,25 +3375,26 @@ TEST_CASE("z$.blocked.disklayer.put") ZenCacheValue BufferGet; CHECK(Zcs.Get("test_bucket", HashKey, BufferGet)); - // Overwriting with a value of same size should go fine - Zcs.Put("test_bucket", HashKey, {.Value = Buffer}); - CbObject CacheValue2 = CreateCacheValue(64 * 1024 + 64 + 1); IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer(); Buffer2.SetContentType(ZenContentType::kCbObject); -# if ZEN_PLATFORM_WINDOWS - // On Windows platform, overwriting with different size while we have - // it open for read should throw exception if file is held open - CHECK_THROWS(Zcs.Put("test_bucket", HashKey, {.Value = Buffer2})); -# else - // Other platforms should handle overwrite just fine + + // We should be able to overwrite even if the file is open for read Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}); -# endif - BufferGet = ZenCacheValue{}; + MemoryView OldView = BufferGet.Value.GetView(); - // Read access has been removed, we should now be able to overwrite it - Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}); + ZenCacheValue BufferGet2; + CHECK(Zcs.Get("test_bucket", HashKey, BufferGet2)); + MemoryView NewView = BufferGet2.Value.GetView(); + + // Make sure file openend for read before we wrote it still have old data + CHECK(OldView.GetSize() == Buffer.GetSize()); + CHECK(memcmp(OldView.GetData(), Buffer.GetData(), OldView.GetSize()) == 0); + + // Make sure we get the new data when reading after we write new data + CHECK(NewView.GetSize() == Buffer2.GetSize()); + CHECK(memcmp(NewView.GetData(), Buffer2.GetData(), NewView.GetSize()) == 0); } #endif -- cgit v1.2.3 From 13385d08ce740a57de3662beffaf5cca4586c7e5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 30 May 2022 13:59:48 +0200 Subject: add comment about removing files --- zenserver/cache/structuredcachestore.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index d1b242c5c..ee0835fd3 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1833,6 +1833,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); + // We do a speculative remove of the file instead of probing with a exists call and check the error code instead std::filesystem::remove(FsPath, Ec); if (Ec && Ec.value() != ENOENT) { -- cgit v1.2.3