Merge pull request #89 from EpicGames/de/namespacesv1.0.1.5

Add namespacecachestore layer to allow multiple structured cache namespaces

1 files changed, 195 insertions, 65 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 3e2692859..05c80c5bf 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -232,7 +232,7 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 	WriteFile(Path, Object.GetBuffer().AsIoBuffer());
 }
 
-ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
+ZenCacheNamespace::ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir)
 : GcStorage(Gc)
 , GcContributor(Gc)
 , m_RootDir(RootDir)
@@ -248,12 +248,12 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
 #endif
 }
 
-ZenCacheStore::~ZenCacheStore()
+ZenCacheNamespace::~ZenCacheNamespace()
 {
 }
 
 bool
-ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
 {
 	ZEN_TRACE_CPU("Z$::Get");
 
@@ -291,7 +291,7 @@ ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheVal
 }
 
 void
-ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
@@ -327,7 +327,7 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
 }
 
 bool
-ZenCacheStore::DropBucket(std::string_view Bucket)
+ZenCacheNamespace::DropBucket(std::string_view Bucket)
 {
 	ZEN_INFO("dropping bucket '{}'", Bucket);
 
@@ -343,13 +343,13 @@ ZenCacheStore::DropBucket(std::string_view Bucket)
 }
 
 void
-ZenCacheStore::Flush()
+ZenCacheNamespace::Flush()
 {
 	m_DiskLayer.Flush();
 }
 
 void
-ZenCacheStore::Scrub(ScrubContext& Ctx)
+ZenCacheNamespace::Scrub(ScrubContext& Ctx)
 {
 	if (m_LastScrubTime == Ctx.ScrubTimestamp())
 	{
@@ -363,7 +363,7 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 }
 
 void
-ZenCacheStore::GatherReferences(GcContext& GcCtx)
+ZenCacheNamespace::GatherReferences(GcContext& GcCtx)
 {
 	Stopwatch  Timer;
 	const auto Guard =
@@ -377,14 +377,14 @@ ZenCacheStore::GatherReferences(GcContext& GcCtx)
 }
 
 void
-ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+ZenCacheNamespace::CollectGarbage(GcContext& GcCtx)
 {
 	m_MemLayer.Reset();
 	m_DiskLayer.CollectGarbage(GcCtx);
 }
 
 GcStorageSize
-ZenCacheStore::StorageSize() const
+ZenCacheNamespace::StorageSize() const
 {
 	return {.DiskSize = m_DiskLayer.TotalSize(), .MemorySize = m_MemLayer.TotalSize()};
 }
@@ -1960,49 +1960,23 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 void
 ZenCacheDiskLayer::DiscoverBuckets()
 {
-	FileSystemTraversal Traversal;
-	struct Visitor : public FileSystemTraversal::TreeVisitor
-	{
-		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
-							   [[maybe_unused]] const path_view&			 File,
-							   [[maybe_unused]] uint64_t					 FileSize) override
-		{
-		}
-
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
-		{
-			Dirs.push_back((decltype(Dirs)::value_type)(DirectoryName));
-			return false;
-		}
-
-		std::vector<std::filesystem::path::string_type> Dirs;
-	} Visit;
-
-	Traversal.TraverseFileSystem(m_RootDir, Visit);
+	DirectoryContent DirContent;
+	GetDirectoryContent(m_RootDir, DirectoryContent::IncludeDirsFlag, DirContent);
 
 	// Initialize buckets
 
 	RwLock::ExclusiveLockScope _(m_Lock);
 
-	for (const auto& BucketName : Visit.Dirs)
+	for (const std::filesystem::path& BucketPath : DirContent.Directories)
 	{
+		std::string BucketName = PathToUtf8(BucketPath.stem());
 		// New bucket needs to be created
-
-#if ZEN_PLATFORM_WINDOWS
-		std::string BucketName8 = WideToUtf8(BucketName);
-#else
-		const auto& BucketName8 = BucketName;
-#endif
-
-		if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end())
+		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 		{
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.try_emplace(BucketName8, BucketName8);
-
-			std::filesystem::path BucketPath = m_RootDir;
-			BucketPath /= BucketName8;
+			auto InsertResult = m_Buckets.try_emplace(BucketName, BucketName);
 
 			CacheBucket& Bucket = InsertResult.first->second;
 
@@ -2010,11 +1984,11 @@ ZenCacheDiskLayer::DiscoverBuckets()
 
 			if (Bucket.IsOk())
 			{
-				ZEN_INFO("Discovered bucket '{}'", BucketName8);
+				ZEN_INFO("Discovered bucket '{}'", BucketName);
 			}
 			else
 			{
-				ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName8, m_RootDir);
+				ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
 
 				m_Buckets.erase(InsertResult.first);
 			}
@@ -2102,6 +2076,162 @@ ZenCacheDiskLayer::TotalSize() const
 	return TotalSize;
 }
 
+//////////////////////////// ZenCacheStore
+
+static constexpr std::string_view ZenCacheNamespaceDirPrefix = "ns_";
+
+ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc)
+{
+	CreateDirectories(BasePath);
+
+	DirectoryContent DirContent;
+	GetDirectoryContent(BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+
+	std::vector<std::string> LegacyBuckets;
+	std::vector<std::string> Namespaces;
+	for (const std::filesystem::path& DirPath : DirContent.Directories)
+	{
+		std::string DirName = PathToUtf8(DirPath.stem());
+		if (DirName.starts_with(ZenCacheNamespaceDirPrefix))
+		{
+			Namespaces.push_back(DirName.substr(3));
+			continue;
+		}
+		LegacyBuckets.push_back(DirName);
+	}
+
+	ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size());
+
+	if (std::find(Namespaces.begin(), Namespaces.end(), DefaultNamespace) == Namespaces.end())
+	{
+		ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size());
+		std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, DefaultNamespace);
+		CreateDirectories(DefaultNamespaceFolder);
+
+		// Move any non-namespace folders into the default namespace folder
+		for (const std::string& DirName : LegacyBuckets)
+		{
+			std::filesystem::path LegacyFolder = BasePath / DirName;
+			std::filesystem::path NewPath	   = DefaultNamespaceFolder / DirName;
+			std::error_code		  Ec;
+			std::filesystem::rename(LegacyFolder, NewPath, Ec);
+			if (Ec)
+			{
+				ZEN_ERROR("Unable to move '{}' to '{}', reason '{}'", LegacyFolder, NewPath, Ec.message());
+			}
+		}
+		Namespaces.push_back(std::string(DefaultNamespace));
+	}
+
+	for (const std::string& NamespaceName : Namespaces)
+	{
+		m_Namespaces[NamespaceName] =
+			std::make_unique<ZenCacheNamespace>(Gc, BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, NamespaceName));
+	}
+}
+
+ZenCacheStore::~ZenCacheStore()
+{
+	m_Namespaces.clear();
+}
+
+bool
+ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
+	{
+		return Store->Get(Bucket, HashKey, OutValue);
+	}
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString());
+	return false;
+}
+
+void
+ZenCacheStore::Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value)
+{
+	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
+	{
+		return Store->Put(Bucket, HashKey, Value);
+	}
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString());
+}
+
+bool
+ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket)
+{
+	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
+	{
+		return Store->DropBucket(Bucket);
+	}
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}'", Namespace, Bucket);
+	return false;
+}
+
+void
+ZenCacheStore::Flush()
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Flush(); });
+}
+
+void
+ZenCacheStore::Scrub(ScrubContext& Ctx)
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); });
+}
+
+ZenCacheNamespace*
+ZenCacheStore::GetNamespace(std::string_view Namespace)
+{
+	RwLock::SharedLockScope _(m_NamespacesLock);
+	if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end())
+	{
+		return It->second.get();
+	}
+	return nullptr;
+}
+
+void
+ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const
+{
+	std::vector<std::pair<std::string, ZenCacheNamespace&> > Namespaces;
+	{
+		RwLock::SharedLockScope _(m_NamespacesLock);
+		Namespaces.reserve(m_Namespaces.size());
+		for (const auto& Entry : m_Namespaces)
+		{
+			Namespaces.push_back({Entry.first, *Entry.second});
+		}
+	}
+	for (auto& Entry : Namespaces)
+	{
+		Callback(Entry.first, Entry.second);
+	}
+}
+
+void
+ZenCacheStore::GatherReferences(GcContext& GcCtx)
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.GatherReferences(GcCtx); });
+}
+
+void
+ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.CollectGarbage(GcCtx); });
+}
+
+GcStorageSize
+ZenCacheStore::StorageSize() const
+{
+	GcStorageSize Size;
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) {
+		GcStorageSize StoreSize = Store.StorageSize();
+		Size.MemorySize += StoreSize.MemorySize;
+		Size.DiskSize += StoreSize.DiskSize;
+	});
+	return Size;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_WITH_TESTS
@@ -2140,7 +2270,7 @@ TEST_CASE("z$.store")
 
 	CasGc Gc;
 
-	ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+	ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 	const int kIterationCount = 100;
 
@@ -2193,8 +2323,8 @@ TEST_CASE("z$.size")
 		GcStorageSize CacheSize;
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256);
 
@@ -2213,8 +2343,8 @@ TEST_CASE("z$.size")
 		}
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2236,8 +2366,8 @@ TEST_CASE("z$.size")
 		GcStorageSize CacheSize;
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64);
 
@@ -2256,8 +2386,8 @@ TEST_CASE("z$.size")
 		}
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2294,9 +2424,9 @@ TEST_CASE("z$.gc")
 		};
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
-			const auto	  Bucket = "teardrinker"sv;
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+			const auto		  Bucket = "teardrinker"sv;
 
 			// Create a cache record
 			const IoHash   Key = CreateKey(42);
@@ -2332,7 +2462,7 @@ TEST_CASE("z$.gc")
 		// Expect timestamps to be serialized
 		{
 			CasGc				Gc;
-			ZenCacheStore		Zcs(Gc, TempDir.Path() / "cache");
+			ZenCacheNamespace	Zcs(Gc, TempDir.Path() / "cache");
 			std::vector<IoHash> Keep;
 
 			// Collect garbage with 1 hour max cache duration
@@ -2353,7 +2483,7 @@ TEST_CASE("z$.gc")
 	{
 		ScopedTemporaryDirectory TempDir;
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path() / "cache");
 		const auto				 Bucket		 = "fortysixandtwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -2401,7 +2531,7 @@ TEST_CASE("z$.gc")
 	{
 		ScopedTemporaryDirectory TempDir;
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path() / "cache");
 		const auto				 Bucket		 = "rightintwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -2494,7 +2624,7 @@ TEST_CASE("z$.legacyconversion")
 	const std::string Bucket = "rightintwo";
 	{
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path());
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path());
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
 		for (size_t i = 0; i < ChunkCount; i++)
@@ -2582,8 +2712,8 @@ TEST_CASE("z$.legacyconversion")
 	std::filesystem::remove(IndexPath);
 
 	{
-		CasGc		  Gc;
-		ZenCacheStore Zcs(Gc, TempDir.Path());
+		CasGc			  Gc;
+		ZenCacheNamespace Zcs(Gc, TempDir.Path());
 
 		for (size_t i = 0; i < ChunkCount; i += 2)
 		{
@@ -2643,9 +2773,9 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 
 		CreateDirectories(TempDir.Path());
 
-		WorkerThreadPool ThreadPool(4);
-		CasGc			 Gc;
-		ZenCacheStore	 Zcs(Gc, TempDir.Path());
+		WorkerThreadPool  ThreadPool(4);
+		CasGc			  Gc;
+		ZenCacheNamespace Zcs(Gc, TempDir.Path());
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;