unblock cache bucket drop (#406)

* don't hold exclusive locks while deleting files from a dropped bucket/namespace
* cleaner detection of missing namespace when issuing a drop

1 files changed, 82 insertions, 26 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index d80da6ea6..d26c6dfd2 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -196,14 +196,14 @@ namespace cache::impl {
 		return true;
 	}
 
-	bool MoveAndDeleteDirectory(const std::filesystem::path& Dir)
+	std::filesystem::path MoveDroppedDirectory(const std::filesystem::path& Dir)
 	{
 		int DropIndex = 0;
 		do
 		{
 			if (!IsDir(Dir))
 			{
-				return false;
+				return {};
 			}
 
 			std::string			  DroppedName		= fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex);
@@ -218,12 +218,11 @@ namespace cache::impl {
 			RenameDirectory(Dir, DroppedBucketPath, Ec);
 			if (!Ec)
 			{
-				DeleteDirectories(DroppedBucketPath);
-				return true;
+				return DroppedBucketPath;
 			}
-			// TODO: Do we need to bail at some point?
 			zen::Sleep(100);
-		} while (true);
+		} while (DropIndex < 10);
+		return {};
 	}
 }  // namespace cache::impl
 
@@ -1944,7 +1943,7 @@ ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock
 	}
 }
 
-bool
+std::function<void()>
 ZenCacheDiskLayer::CacheBucket::Drop()
 {
 	ZEN_TRACE_CPU("Z$::Bucket::Drop");
@@ -1960,7 +1959,7 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_BlockStore.Close();
 	m_SlogFile.Close();
 
-	const bool Deleted = cache::impl::MoveAndDeleteDirectory(m_BucketDir);
+	std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(m_BucketDir);
 
 	m_Index.clear();
 	m_Payloads.clear();
@@ -1973,7 +1972,21 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_OuterCacheMemoryUsage.fetch_sub(m_MemCachedSize.load());
 	m_MemCachedSize.store(0);
 
-	return Deleted;
+	if (DroppedPath.empty())
+	{
+		return {};
+	}
+	else
+	{
+		return [DroppedPath = std::move(DroppedPath)]() {
+			std::error_code Ec;
+			(void)DeleteDirectories(DroppedPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message());
+			}
+		};
+	}
 }
 
 void
@@ -3904,7 +3917,11 @@ ZenCacheDiskLayer::DiscoverBuckets()
 		if (IsKnownBadBucketName(BucketName))
 		{
 			BadBucketDirectories.push_back(BucketPath);
-
+			continue;
+		}
+		else if (BucketName.starts_with("[dropped]"))
+		{
+			BadBucketDirectories.push_back(BucketPath);
 			continue;
 		}
 
@@ -3987,7 +4004,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 	Work.Wait();
 }
 
-bool
+std::function<void()>
 ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
 {
 	ZEN_TRACE_CPU("Z$::DropBucket");
@@ -4005,33 +4022,72 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
 		return Bucket.Drop();
 	}
 
-	// Make sure we remove the folder even if we don't know about the bucket
 	std::filesystem::path BucketPath = m_RootDir;
 	BucketPath /= std::string(InBucket);
-	return cache::impl::MoveAndDeleteDirectory(BucketPath);
+	std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(BucketPath);
+	if (DroppedPath.empty())
+	{
+		return {};
+	}
+	else
+	{
+		return [DroppedPath = std::move(DroppedPath)]() {
+			std::error_code Ec;
+			(void)DeleteDirectories(DroppedPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message());
+			}
+		};
+	}
 }
 
-bool
+std::function<void()>
 ZenCacheDiskLayer::Drop()
 {
 	ZEN_TRACE_CPU("Z$::Drop");
 
-	RwLock::ExclusiveLockScope _(m_Lock);
-
-	std::vector<std::unique_ptr<CacheBucket>> Buckets;
-	Buckets.reserve(m_Buckets.size());
-	while (!m_Buckets.empty())
+	std::vector<std::function<void()>> PostDropOps;
 	{
-		const auto&	 It		= m_Buckets.begin();
-		CacheBucket& Bucket = *It->second;
-		m_DroppedBuckets.push_back(std::move(It->second));
-		m_Buckets.erase(It->first);
-		if (!Bucket.Drop())
+		RwLock::ExclusiveLockScope _(m_Lock);
+		PostDropOps.reserve(m_Buckets.size());
+		while (!m_Buckets.empty())
 		{
-			return false;
+			const auto&	 It		= m_Buckets.begin();
+			CacheBucket& Bucket = *It->second;
+			m_DroppedBuckets.push_back(std::move(It->second));
+			m_Buckets.erase(It->first);
+			if (std::function<void()> PostDropOp = Bucket.Drop(); !PostDropOp)
+			{
+				return {};
+			}
+			else
+			{
+				PostDropOps.emplace_back(std::move(PostDropOp));
+			}
 		}
 	}
-	return cache::impl::MoveAndDeleteDirectory(m_RootDir);
+
+	std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(m_RootDir);
+	if (DroppedPath.empty())
+	{
+		return {};
+	}
+	else
+	{
+		return [DroppedPath = std::move(DroppedPath), PostDropOps = std::move(PostDropOps)]() {
+			for (auto& PostDropOp : PostDropOps)
+			{
+				PostDropOp();
+			}
+			std::error_code Ec;
+			(void)DeleteDirectories(DroppedPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message());
+			}
+		};
+	}
 }
 
 void