cleanup

1 files changed, 225 insertions, 31 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 2869191fd..075b7d408 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -232,7 +232,7 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 	WriteFile(Path, Object.GetBuffer().AsIoBuffer());
 }
 
-ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
+ZenCacheNamespace::ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir)
 : GcStorage(Gc)
 , GcContributor(Gc)
 , m_RootDir(RootDir)
@@ -248,12 +248,12 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
 #endif
 }
 
-ZenCacheStore::~ZenCacheStore()
+ZenCacheNamespace::~ZenCacheNamespace()
 {
 }
 
 bool
-ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
 {
 	ZEN_TRACE_CPU("Z$::Get");
 
@@ -291,7 +291,7 @@ ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheVal
 }
 
 void
-ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
@@ -327,7 +327,7 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
 }
 
 bool
-ZenCacheStore::DropBucket(std::string_view Bucket)
+ZenCacheNamespace::DropBucket(std::string_view Bucket)
 {
 	ZEN_INFO("dropping bucket '{}'", Bucket);
 
@@ -343,13 +343,13 @@ ZenCacheStore::DropBucket(std::string_view Bucket)
 }
 
 void
-ZenCacheStore::Flush()
+ZenCacheNamespace::Flush()
 {
 	m_DiskLayer.Flush();
 }
 
 void
-ZenCacheStore::Scrub(ScrubContext& Ctx)
+ZenCacheNamespace::Scrub(ScrubContext& Ctx)
 {
 	if (m_LastScrubTime == Ctx.ScrubTimestamp())
 	{
@@ -363,7 +363,7 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 }
 
 void
-ZenCacheStore::GatherReferences(GcContext& GcCtx)
+ZenCacheNamespace::GatherReferences(GcContext& GcCtx)
 {
 	Stopwatch  Timer;
 	const auto Guard =
@@ -377,14 +377,14 @@ ZenCacheStore::GatherReferences(GcContext& GcCtx)
 }
 
 void
-ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+ZenCacheNamespace::CollectGarbage(GcContext& GcCtx)
 {
 	m_MemLayer.Reset();
 	m_DiskLayer.CollectGarbage(GcCtx);
 }
 
 GcStorageSize
-ZenCacheStore::StorageSize() const
+ZenCacheNamespace::StorageSize() const
 {
 	return {.DiskSize = m_DiskLayer.TotalSize(), .MemorySize = m_MemLayer.TotalSize()};
 }
@@ -2098,6 +2098,200 @@ ZenCacheDiskLayer::TotalSize() const
 	return TotalSize;
 }
 
+//////////////////////////// ZenCacheStore
+
+const char* ZenCacheNamespaceDirPrefix = "ns_";
+
+namespace {
+
+	std::vector<std::string> FindExistingFolders(const std::filesystem::path& RootPath)
+	{
+		FileSystemTraversal Traversal;
+		struct Visitor : public FileSystemTraversal::TreeVisitor
+		{
+			virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override {}
+
+			virtual bool VisitDirectory(const std::filesystem::path&, const path_view& DirectoryName) override
+			{
+				std::string DirName8 = WideToUtf8(DirectoryName);
+				Dirs.push_back(DirName8);
+				return false;
+			}
+
+			std::vector<std::string> Dirs;
+		} Visit;
+
+		Traversal.TraverseFileSystem(RootPath, Visit);
+		return Visit.Dirs;
+	}
+
+}  // namespace
+
+ZenCacheStore::ZenCacheStore(std::filesystem::path BasePath, CasGc& Gc) : GcStorage(Gc), GcContributor(Gc)
+{
+	CreateDirectories(BasePath);
+	std::vector<std::string> ExistingFolders = FindExistingFolders(BasePath);
+
+	std::vector<std::string> LegacyBuckets;
+	std::vector<std::string> Namespaces;
+	for (const std::string& DirName : ExistingFolders)
+	{
+		if (DirName.starts_with(ZenCacheNamespaceDirPrefix))
+		{
+			Namespaces.push_back(DirName.substr(3));
+			continue;
+		}
+		LegacyBuckets.push_back(DirName);
+	}
+
+	ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size());
+
+	if (std::find(Namespaces.begin(), Namespaces.end(), "") == Namespaces.end())
+	{
+		ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size());
+		std::filesystem::path DefaultNamespaceFolder = BasePath / ZenCacheNamespaceDirPrefix;
+		CreateDirectories(DefaultNamespaceFolder);
+
+		// Move any non-namespace folders into the default namespace folder
+		for (const std::string& DirName : LegacyBuckets)
+		{
+			std::filesystem::path LegacyFolder = BasePath / DirName;
+			std::filesystem::path NewPath	   = DefaultNamespaceFolder / DirName;
+			std::filesystem::rename(LegacyFolder, NewPath);
+		}
+		Namespaces.push_back("");
+	}
+
+	for (const std::string& NamespaceName : Namespaces)
+	{
+		Ref<ZenCacheNamespace> Store = new ZenCacheNamespace(Gc, BasePath / (ZenCacheNamespaceDirPrefix + NamespaceName));
+		m_Namespaces[NamespaceName]	 = Store;
+	}
+}
+
+ZenCacheStore::~ZenCacheStore()
+{
+	m_Namespaces.clear();
+}
+
+bool
+ZenCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+	Ref<ZenCacheNamespace> Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return false;
+	}
+	return Store->Get(Bucket, HashKey, OutValue);
+}
+
+void
+ZenCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value)
+{
+	Ref<ZenCacheNamespace> Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return;
+	}
+	Store->Put(Bucket, HashKey, Value);
+}
+
+bool
+ZenCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket)
+{
+	Ref<ZenCacheNamespace> Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return false;
+	}
+	return Store->DropBucket(Bucket);
+}
+
+void
+ZenCacheStore::Flush()
+{
+	std::vector<Ref<ZenCacheNamespace>> Stores;
+	RwLock::SharedLockScope				_(m_NamespacesLock);
+	Stores.reserve(m_Namespaces.size());
+	for (const auto& Entry : m_Namespaces)
+	{
+		Stores.push_back(Entry.second);
+	}
+	_.ReleaseNow();
+	for (const Ref<ZenCacheNamespace>& Store : Stores)
+	{
+		Store->Flush();
+	}
+}
+
+void
+ZenCacheStore::Scrub(ScrubContext& Ctx)
+{
+	std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+	for (const Ref<ZenCacheNamespace>& Store : Stores)
+	{
+		Store->Scrub(Ctx);
+	}
+}
+
+Ref<ZenCacheNamespace>
+ZenCacheStore::GetStore(const std::string& Namespace)
+{
+	RwLock::SharedLockScope _(m_NamespacesLock);
+	if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end())
+	{
+		return It->second;
+	}
+	return nullptr;
+}
+
+std::vector<Ref<ZenCacheNamespace>>
+ZenCacheStore::GetAllStores() const
+{
+	std::vector<Ref<ZenCacheNamespace>> Stores;
+	RwLock::SharedLockScope				_(m_NamespacesLock);
+	Stores.reserve(m_Namespaces.size());
+	for (const auto& Entry : m_Namespaces)
+	{
+		Stores.push_back(Entry.second);
+	}
+	return Stores;
+}
+
+void
+ZenCacheStore::GatherReferences(GcContext& GcCtx)
+{
+	std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+	for (const Ref<ZenCacheNamespace>& Store : Stores)
+	{
+		Store->GatherReferences(GcCtx);
+	}
+}
+
+void
+ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+{
+	std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+	for (const Ref<ZenCacheNamespace>& Store : Stores)
+	{
+		Store->CollectGarbage(GcCtx);
+	}
+}
+
+GcStorageSize
+ZenCacheStore::StorageSize() const
+{
+	std::vector<Ref<ZenCacheNamespace>> Stores = GetAllStores();
+	GcStorageSize						Size;
+	for (const Ref<ZenCacheNamespace>& Store : Stores)
+	{
+		GcStorageSize StoreSize = Store->StorageSize();
+		Size.MemorySize += StoreSize.MemorySize;
+		Size.DiskSize += StoreSize.DiskSize;
+	}
+	return Size;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_WITH_TESTS
@@ -2136,7 +2330,7 @@ TEST_CASE("z$.store")
 
 	CasGc Gc;
 
-	ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+	ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 	const int kIterationCount = 100;
 
@@ -2189,8 +2383,8 @@ TEST_CASE("z$.size")
 		GcStorageSize CacheSize;
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256);
 
@@ -2209,8 +2403,8 @@ TEST_CASE("z$.size")
 		}
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2232,8 +2426,8 @@ TEST_CASE("z$.size")
 		GcStorageSize CacheSize;
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64);
 
@@ -2252,8 +2446,8 @@ TEST_CASE("z$.size")
 		}
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2290,9 +2484,9 @@ TEST_CASE("z$.gc")
 		};
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
-			const auto	  Bucket = "teardrinker"sv;
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+			const auto		  Bucket = "teardrinker"sv;
 
 			// Create a cache record
 			const IoHash   Key = CreateKey(42);
@@ -2328,7 +2522,7 @@ TEST_CASE("z$.gc")
 		// Expect timestamps to be serialized
 		{
 			CasGc				Gc;
-			ZenCacheStore		Zcs(Gc, TempDir.Path() / "cache");
+			ZenCacheNamespace	Zcs(Gc, TempDir.Path() / "cache");
 			std::vector<IoHash> Keep;
 
 			// Collect garbage with 1 hour max cache duration
@@ -2349,7 +2543,7 @@ TEST_CASE("z$.gc")
 	{
 		ScopedTemporaryDirectory TempDir;
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path() / "cache");
 		const auto				 Bucket		 = "fortysixandtwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -2397,7 +2591,7 @@ TEST_CASE("z$.gc")
 	{
 		ScopedTemporaryDirectory TempDir;
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path() / "cache");
 		const auto				 Bucket		 = "rightintwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -2490,7 +2684,7 @@ TEST_CASE("z$.legacyconversion")
 	const std::string Bucket = "rightintwo";
 	{
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path());
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path());
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
 		for (size_t i = 0; i < ChunkCount; i++)
@@ -2578,8 +2772,8 @@ TEST_CASE("z$.legacyconversion")
 	std::filesystem::remove(IndexPath);
 
 	{
-		CasGc		  Gc;
-		ZenCacheStore Zcs(Gc, TempDir.Path());
+		CasGc			  Gc;
+		ZenCacheNamespace Zcs(Gc, TempDir.Path());
 
 		for (size_t i = 0; i < ChunkCount; i += 2)
 		{
@@ -2639,9 +2833,9 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 
 		CreateDirectories(TempDir.Path());
 
-		WorkerThreadPool ThreadPool(4);
-		CasGc			 Gc;
-		ZenCacheStore	 Zcs(Gc, TempDir.Path());
+		WorkerThreadPool  ThreadPool(4);
+		CasGc			  Gc;
+		ZenCacheNamespace Zcs(Gc, TempDir.Path());
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;