Merge pull request #108 from EpicGames/de/drop-namespace

drop namespace

1 files changed, 230 insertions, 69 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 9ebec1e76..e4dbf390a 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -205,6 +205,36 @@ namespace {
 		return true;
 	}
 
+	bool MoveAndDeleteDirectory(const std::filesystem::path& Dir)
+	{
+		int DropIndex = 0;
+		do
+		{
+			if (!std::filesystem::exists(Dir))
+			{
+				return false;
+			}
+
+			std::string			  DroppedName		= fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex);
+			std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName;
+			if (std::filesystem::exists(DroppedBucketPath))
+			{
+				DropIndex++;
+				continue;
+			}
+
+			std::error_code Ec;
+			std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+			if (!Ec)
+			{
+				DeleteDirectories(DroppedBucketPath);
+				return true;
+			}
+			// TODO: Do we need to bail at some point?
+			zen::Sleep(100);
+		} while (true);
+	}
+
 }  // namespace
 
 namespace fs = std::filesystem;
@@ -342,6 +372,13 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket)
 	return AnyDropped;
 }
 
+bool
+ZenCacheNamespace::Drop()
+{
+	m_MemLayer.Drop();
+	return m_DiskLayer.Drop();
+}
+
 void
 ZenCacheNamespace::Flush()
 {
@@ -404,14 +441,14 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa
 {
 	RwLock::SharedLockScope _(m_Lock);
 
-	auto it = m_Buckets.find(std::string(InBucket));
+	auto It = m_Buckets.find(std::string(InBucket));
 
-	if (it == m_Buckets.end())
+	if (It == m_Buckets.end())
 	{
 		return false;
 	}
 
-	CacheBucket* Bucket = &it->second;
+	CacheBucket* Bucket = It->second.get();
 
 	_.ReleaseNow();
 
@@ -425,14 +462,15 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa
 void
 ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
 {
-	CacheBucket* Bucket = nullptr;
+	const auto	 BucketName = std::string(InBucket);
+	CacheBucket* Bucket		= nullptr;
 
 	{
 		RwLock::SharedLockScope _(m_Lock);
 
 		if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end())
 		{
-			Bucket = &It->second;
+			Bucket = It->second.get();
 		}
 	}
 
@@ -444,11 +482,12 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const
 
 		if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end())
 		{
-			Bucket = &It->second;
+			Bucket = It->second.get();
 		}
 		else
 		{
-			Bucket = &m_Buckets[std::string(InBucket)];
+			auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>());
+			Bucket			  = InsertResult.first->second.get();
 		}
 	}
 
@@ -458,11 +497,37 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const
 }
 
 bool
-ZenCacheMemoryLayer::DropBucket(std::string_view Bucket)
+ZenCacheMemoryLayer::DropBucket(std::string_view InBucket)
 {
 	RwLock::ExclusiveLockScope _(m_Lock);
 
-	return !!m_Buckets.erase(std::string(Bucket));
+	auto It = m_Buckets.find(std::string(InBucket));
+
+	if (It != m_Buckets.end())
+	{
+		CacheBucket& Bucket = *It->second;
+		m_DroppedBuckets.push_back(std::move(It->second));
+		m_Buckets.erase(It);
+		Bucket.Drop();
+		return true;
+	}
+	return false;
+}
+
+void
+ZenCacheMemoryLayer::Drop()
+{
+	RwLock::ExclusiveLockScope				  _(m_Lock);
+	std::vector<std::unique_ptr<CacheBucket>> Buckets;
+	Buckets.reserve(m_Buckets.size());
+	while (!m_Buckets.empty())
+	{
+		const auto&	 It		= m_Buckets.begin();
+		CacheBucket& Bucket = *It->second;
+		m_DroppedBuckets.push_back(std::move(It->second));
+		m_Buckets.erase(It->first);
+		Bucket.Drop();
+	}
 }
 
 void
@@ -472,7 +537,7 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
 
 	for (auto& Kv : m_Buckets)
 	{
-		Kv.second.Scrub(Ctx);
+		Kv.second->Scrub(Ctx);
 	}
 }
 
@@ -486,7 +551,7 @@ ZenCacheMemoryLayer::GatherAccessTimes(zen::access_tracking::AccessTimes& Access
 	for (auto& Kv : m_Buckets)
 	{
 		std::vector<KeyAccessTime>& Bucket = AccessTimes.Buckets[Kv.first];
-		Kv.second.GatherAccessTimes(Bucket);
+		Kv.second->GatherAccessTimes(Bucket);
 	}
 }
 
@@ -505,7 +570,7 @@ ZenCacheMemoryLayer::TotalSize() const
 
 	for (auto& Kv : m_Buckets)
 	{
-		TotalSize += Kv.second.TotalSize();
+		TotalSize += Kv.second->TotalSize();
 	}
 
 	return TotalSize;
@@ -570,6 +635,13 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue
 	m_TotalSize.fetch_add(Value.Value.GetSize(), std::memory_order::relaxed);
 }
 
+void
+ZenCacheMemoryLayer::CacheBucket::Drop()
+{
+	RwLock::ExclusiveLockScope _(m_BucketLock);
+	m_CacheMap.clear();
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)), m_BucketId(Oid::Zero)
@@ -1176,38 +1248,7 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue&
 	PutInlineCacheValue(HashKey, Value);
 }
 
-static bool
-DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view BucketName)
-{
-	int DropIndex = 0;
-	do
-	{
-		if (!std::filesystem::exists(BucketDir))
-		{
-			return false;
-		}
-
-		std::string			  DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, DropIndex);
-		std::filesystem::path DroppedBucketPath = BucketDir.parent_path() / DroppedBucketName;
-		if (std::filesystem::exists(DroppedBucketPath))
-		{
-			DropIndex++;
-			continue;
-		}
-
-		std::error_code Ec;
-		std::filesystem::rename(BucketDir, DroppedBucketPath, Ec);
-		if (!Ec)
-		{
-			DeleteDirectories(DroppedBucketPath);
-			return true;
-		}
-		// TODO: Do we need to bail at some point?
-		zen::Sleep(100);
-	} while (true);
-}
-
-void
+bool
 ZenCacheDiskLayer::CacheBucket::Drop()
 {
 	RwLock::ExclusiveLockScope _(m_IndexLock);
@@ -1221,9 +1262,10 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_BlockStore.Close();
 	m_SlogFile.Close();
 
-	DeleteBucketFromDisk(m_BucketDir, m_BucketName);
+	bool Deleted = MoveAndDeleteDirectory(m_BucketDir);
 
 	m_Index.clear();
+	return Deleted;
 }
 
 void
@@ -1948,11 +1990,11 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 	{
 		RwLock::SharedLockScope _(m_Lock);
 
-		auto it = m_Buckets.find(BucketName);
+		auto It = m_Buckets.find(BucketName);
 
-		if (it != m_Buckets.end())
+		if (It != m_Buckets.end())
 		{
-			Bucket = it->second.get();
+			Bucket = It->second.get();
 		}
 	}
 
@@ -1962,9 +2004,9 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 
 		RwLock::ExclusiveLockScope _(m_Lock);
 
-		if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end())
+		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 		{
-			Bucket = it->second.get();
+			Bucket = It->second.get();
 		}
 		else
 		{
@@ -1995,11 +2037,11 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 	{
 		RwLock::SharedLockScope _(m_Lock);
 
-		auto it = m_Buckets.find(BucketName);
+		auto It = m_Buckets.find(BucketName);
 
-		if (it != m_Buckets.end())
+		if (It != m_Buckets.end())
 		{
-			Bucket = it->second.get();
+			Bucket = It->second.get();
 		}
 	}
 
@@ -2009,9 +2051,9 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 
 		RwLock::ExclusiveLockScope _(m_Lock);
 
-		if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end())
+		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 		{
-			Bucket = it->second.get();
+			Bucket = It->second.get();
 		}
 		else
 		{
@@ -2072,23 +2114,42 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
 {
 	RwLock::ExclusiveLockScope _(m_Lock);
 
-	auto it = m_Buckets.find(std::string(InBucket));
+	auto It = m_Buckets.find(std::string(InBucket));
 
-	if (it != m_Buckets.end())
+	if (It != m_Buckets.end())
 	{
-		CacheBucket& Bucket = *it->second;
-		m_DroppedBuckets.push_back(std::move(it->second));
-		m_Buckets.erase(it);
-
-		Bucket.Drop();
+		CacheBucket& Bucket = *It->second;
+		m_DroppedBuckets.push_back(std::move(It->second));
+		m_Buckets.erase(It);
 
-		return true;
+		return Bucket.Drop();
 	}
 
 	// Make sure we remove the folder even if we don't know about the bucket
 	std::filesystem::path BucketPath = m_RootDir;
 	BucketPath /= std::string(InBucket);
-	return DeleteBucketFromDisk(BucketPath, InBucket);
+	return MoveAndDeleteDirectory(BucketPath);
+}
+
+bool
+ZenCacheDiskLayer::Drop()
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	std::vector<std::unique_ptr<CacheBucket>> Buckets;
+	Buckets.reserve(m_Buckets.size());
+	while (!m_Buckets.empty())
+	{
+		const auto&	 It		= m_Buckets.begin();
+		CacheBucket& Bucket = *It->second;
+		m_DroppedBuckets.push_back(std::move(It->second));
+		m_Buckets.erase(It->first);
+		if (!Bucket.Drop())
+		{
+			return false;
+		}
+	}
+	return MoveAndDeleteDirectory(m_RootDir);
 }
 
 void
@@ -2245,7 +2306,22 @@ ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket)
 	{
 		return Store->DropBucket(Bucket);
 	}
-	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}'", Namespace, Bucket);
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropBucket, bucket '{}'", Namespace, Bucket);
+	return false;
+}
+
+bool
+ZenCacheStore::DropNamespace(std::string_view InNamespace)
+{
+	RwLock::SharedLockScope _(m_NamespacesLock);
+	if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end())
+	{
+		ZenCacheNamespace& Namespace = *It->second;
+		m_DroppedNamespaces.push_back(std::move(It->second));
+		m_Namespaces.erase(It);
+		return Namespace.Drop();
+	}
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace);
 	return false;
 }
 
@@ -3196,25 +3272,110 @@ TEST_CASE("z$.drop.bucket")
 		Workers.ScheduleWork([&]() {
 			zen::Sleep(100);
 			Value1.Value = IoBuffer{};
-			Value1		 = GetValue(Zcs, Namespace, Bucket, Key1);
-			CHECK(!Value1.Value);
 			WorkComplete = true;
 		});
 		// On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket
 		// Our DropBucket execution blocks any incoming request from completing until we are done with the drop
-		Zcs.DropBucket(Namespace, Bucket);
+		CHECK(Zcs.DropBucket(Namespace, Bucket));
 		while (!WorkComplete)
 		{
 			zen::Sleep(1);
 		}
-		CHECK(!Value1.Value);
 
 		// Entire bucket should be dropped, but doing a request should will re-create the namespace but it must still be empty
+		Value1 = GetValue(Zcs, Namespace, Bucket, Key1);
+		CHECK(!Value1.Value);
 		ZenCacheValue Value2 = GetValue(Zcs, Namespace, Bucket, Key2);
 		CHECK(!Value2.Value);
 	}
 }
 
+TEST_CASE("z$.drop.namespace")
+{
+	using namespace testutils;
+
+	const auto CreateCacheValue = [](size_t Size) -> CbObject {
+		std::vector<uint8_t> Buf;
+		Buf.resize(Size);
+
+		CbObjectWriter Writer;
+		Writer.AddBinary("Binary"sv, Buf.data(), Buf.size());
+		return Writer.Save();
+	};
+
+	ScopedTemporaryDirectory TempDir;
+	CreateDirectories(TempDir.Path());
+
+	auto PutValue =
+		[&CreateCacheValue](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, size_t KeyIndex, size_t Size) {
+			// Create a cache record
+			IoHash	 Key		= CreateKey(KeyIndex);
+			CbObject CacheValue = CreateCacheValue(Size);
+
+			IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer();
+			Buffer.SetContentType(ZenContentType::kCbObject);
+
+			ZenCacheValue PutValue = {.Value = Buffer};
+			Zcs.Put(Namespace, Bucket, Key, PutValue);
+			return Key;
+		};
+	auto GetValue = [](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) {
+		ZenCacheValue GetValue;
+		Zcs.Get(Namespace, Bucket, Key, GetValue);
+		return GetValue;
+	};
+	WorkerThreadPool Workers(1);
+	{
+		CasGc		  Gc;
+		ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true});
+		const auto	  Bucket1	 = "teardrinker1"sv;
+		const auto	  Bucket2	 = "teardrinker2"sv;
+		const auto	  Namespace1 = "mynamespace1"sv;
+		const auto	  Namespace2 = "mynamespace2"sv;
+
+		IoHash Key1 = PutValue(Zcs, Namespace1, Bucket1, 42, 4096);
+		IoHash Key2 = PutValue(Zcs, Namespace1, Bucket2, 43, 2048);
+		IoHash Key3 = PutValue(Zcs, Namespace2, Bucket1, 44, 4096);
+		IoHash Key4 = PutValue(Zcs, Namespace2, Bucket2, 45, 2048);
+
+		ZenCacheValue Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1);
+		CHECK(Value1.Value);
+		ZenCacheValue Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2);
+		CHECK(Value2.Value);
+		ZenCacheValue Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3);
+		CHECK(Value3.Value);
+		ZenCacheValue Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4);
+		CHECK(Value4.Value);
+
+		std::atomic_bool WorkComplete = false;
+		Workers.ScheduleWork([&]() {
+			zen::Sleep(100);
+			Value1.Value = IoBuffer{};
+			Value2.Value = IoBuffer{};
+			Value3.Value = IoBuffer{};
+			Value4.Value = IoBuffer{};
+			WorkComplete = true;
+		});
+		// On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket
+		// Our DropBucket execution blocks any incoming request from completing until we are done with the drop
+		CHECK(Zcs.DropNamespace(Namespace1));
+		while (!WorkComplete)
+		{
+			zen::Sleep(1);
+		}
+
+		// Entire namespace should be dropped, but doing a request should will re-create the namespace but it must still be empty
+		Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1);
+		CHECK(!Value1.Value);
+		Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2);
+		CHECK(!Value2.Value);
+		Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3);
+		CHECK(Value3.Value);
+		Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4);
+		CHECK(Value4.Value);
+	}
+}
+
 TEST_CASE("z$.blocked.disklayer.put")
 {
 	ScopedTemporaryDirectory TempDir;