Add namespacecachestore layer to allow multiple structured cache namespaces

1 files changed, 207 insertions, 0 deletions

diff --git a/zenserver/cache/namespacecachestore.cpp b/zenserver/cache/namespacecachestore.cpp
new file mode 100644
index 000000000..82ac40c62
--- /dev/null
+++ b/zenserver/cache/namespacecachestore.cpp
@@ -0,0 +1,207 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "namespacecachestore.h"
+#include "structuredcachestore.h"
+
+namespace zen {
+
+const char* NamespaceDirPrefix = "ns_";
+
+NamespaceCacheStore::NamespaceCacheStore(std::filesystem::path BasePath, CasGc& Gc)
+: GcStorage(Gc)
+, GcContributor(Gc)
+, m_Log(logging::Get("namespacecachestore"))
+, m_BasePath(BasePath)
+{
+	CreateDirectories(m_BasePath);
+	std::vector<std::string> ExistingFolders = FindExistingFolders();
+
+	std::vector<std::string> LegacyBuckets;
+	std::vector<std::string> Namespaces;
+	for (const std::string& DirName : ExistingFolders)
+	{
+		if (DirName.starts_with(NamespaceDirPrefix))
+		{
+			Namespaces.push_back(DirName.substr(3));
+			continue;
+		}
+		LegacyBuckets.push_back(DirName);
+	}
+	if (Namespaces.empty() && !LegacyBuckets.empty())
+	{
+		// If we find no namespaces, but any unknown folders we assume we have a legacy folder
+		// and move any existing folders into a default namespace
+		std::filesystem::path DefaultfNamespaceFolder = m_BasePath / NamespaceDirPrefix;
+		CreateDirectories(DefaultfNamespaceFolder);
+		for (const std::string& DirName : LegacyBuckets)
+		{
+			std::filesystem::path LegacyFolder = m_BasePath / DirName;
+			std::filesystem::path NewPath	   = DefaultfNamespaceFolder / DirName;
+			std::filesystem::rename(LegacyFolder, NewPath);
+		}
+		Namespaces.push_back("");
+	}
+
+	for (const std::string& NamespaceName : Namespaces)
+	{
+		ZenCacheStore* Store		= new ZenCacheStore(Gc, m_BasePath / (NamespaceDirPrefix + NamespaceName));
+		m_Namespaces[NamespaceName] = Store;
+	}
+}
+
+NamespaceCacheStore::~NamespaceCacheStore()
+{
+	for (const auto& Entry : m_Namespaces)
+	{
+		delete Entry.second;
+	}
+	m_Namespaces.clear();
+}
+
+std::vector<std::string>
+NamespaceCacheStore::FindExistingFolders() const
+{
+	FileSystemTraversal Traversal;
+	struct Visitor : public FileSystemTraversal::TreeVisitor
+	{
+		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
+							   [[maybe_unused]] const path_view&			 File,
+							   [[maybe_unused]] uint64_t					 FileSize) override
+		{
+		}
+
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+		{
+			std::string DirName8 = WideToUtf8(DirectoryName);
+			Dirs.push_back(DirName8);
+			return false;
+		}
+
+		std::vector<std::string> Dirs;
+	} Visit;
+
+	Traversal.TraverseFileSystem(m_BasePath, Visit);
+	return Visit.Dirs;
+}
+
+bool
+NamespaceCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+	ZenCacheStore* Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return false;
+	}
+	return Store->Get(Bucket, HashKey, OutValue);
+}
+
+void
+NamespaceCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value)
+{
+	ZenCacheStore* Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return;
+	}
+	Store->Put(Bucket, HashKey, Value);
+}
+
+bool
+NamespaceCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket)
+{
+	ZenCacheStore* Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return false;
+	}
+	return Store->DropBucket(Bucket);
+}
+
+void
+NamespaceCacheStore::Flush()
+{
+	std::vector<ZenCacheStore*> Stores;
+	RwLock::SharedLockScope		_(m_NamespacesLock);
+	Stores.reserve(m_Namespaces.size());
+	for (const auto& Entry : m_Namespaces)
+	{
+		Stores.push_back(Entry.second);
+	}
+	_.ReleaseNow();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->Flush();
+	}
+}
+
+void
+NamespaceCacheStore::Scrub(ScrubContext& Ctx)
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->Scrub(Ctx);
+	}
+}
+
+ZenCacheStore*
+NamespaceCacheStore::GetStore(const std::string& Namespace)
+{
+	RwLock::SharedLockScope _(m_NamespacesLock);
+	if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end())
+	{
+		return It->second;
+	}
+	return nullptr;
+}
+
+std::vector<ZenCacheStore*>
+NamespaceCacheStore::GetAllStores() const
+{
+	std::vector<ZenCacheStore*> Stores;
+	RwLock::SharedLockScope		_(m_NamespacesLock);
+	Stores.reserve(m_Namespaces.size());
+	for (const auto& Entry : m_Namespaces)
+	{
+		Stores.push_back(Entry.second);
+	}
+	return Stores;
+}
+
+void
+NamespaceCacheStore::GatherReferences(GcContext& GcCtx)
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->GatherReferences(GcCtx);
+	}
+}
+
+void
+NamespaceCacheStore::CollectGarbage(GcContext& GcCtx)
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->CollectGarbage(GcCtx);
+	}
+}
+
+GcStorageSize
+NamespaceCacheStore::StorageSize() const
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	GcStorageSize				Size;
+	for (ZenCacheStore* Store : Stores)
+	{
+		GcStorageSize StoreSize = Store->StorageSize();
+		Size.MemorySize += StoreSize.MemorySize;
+		Size.DiskSize += StoreSize.DiskSize;
+	}
+	return Size;
+}
+
+}  // namespace zen