Merge pull request #89 from EpicGames/de/namespacesv1.0.1.5

Add namespacecachestore layer to allow multiple structured cache namespaces

9 files changed, 319 insertions, 162 deletions

diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 437741161..bd85f5a11 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -1022,6 +1022,40 @@ MaximizeOpenFileCount()
 #endif
 }
 
+void
+GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent)
+{
+	FileSystemTraversal Traversal;
+	struct Visitor : public FileSystemTraversal::TreeVisitor
+	{
+		Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
+
+		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
+							   [[maybe_unused]] const path_view&			 File,
+							   [[maybe_unused]] uint64_t					 FileSize) override
+		{
+			if (Flags & DirectoryContent::IncludeFilesFlag)
+			{
+				Content.Files.push_back(Parent / File);
+			}
+		}
+
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+		{
+			if (Flags & DirectoryContent::IncludeDirsFlag)
+			{
+				Content.Directories.push_back(Parent / DirectoryName);
+			}
+			return (Flags & DirectoryContent::RecursiveFlag) != 0;
+		}
+
+		const uint8_t	  Flags;
+		DirectoryContent& Content;
+	} Visit(Flags, OutContent);
+
+	Traversal.TraverseFileSystem(RootDir, Visit);
+}
+
 //////////////////////////////////////////////////////////////////////////
 //
 // Testing related code follows...
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index a6e76eaa0..6d07a79b4 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -169,6 +169,17 @@ public:
 	void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
 };
 
+struct DirectoryContent
+{
+	static const uint8_t			   IncludeDirsFlag	= 1u << 0;
+	static const uint8_t			   IncludeFilesFlag = 1u << 1;
+	static const uint8_t			   RecursiveFlag	= 1u << 2;
+	std::vector<std::filesystem::path> Files;
+	std::vector<std::filesystem::path> Directories;
+};
+
+void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent);
+
 //////////////////////////////////////////////////////////////////////////
 
 void filesystem_forcelink();  // internal
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index e1d9de976..0f16f6785 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -176,7 +176,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
 		case HttpVerb::kDelete:
 			// Drop bucket
 
-			if (m_CacheStore.DropBucket(Bucket))
+			if (m_CacheStore.DropBucket(ZenCacheStore::DefaultNamespace, Bucket))
 			{
 				return Request.WriteResponse(HttpResponseCode::OK);
 			}
@@ -225,7 +225,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 		return Request.WriteResponse(HttpResponseCode::OK);
 	}
 
-	if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, ClientResultValue))
+	if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) &&
+		m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue))
 	{
 		Success					   = true;
 		ZenContentType ContentType = ClientResultValue.Value.GetContentType();
@@ -350,7 +351,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 
 				if (Success && StoreLocal)
 				{
-					m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, ClientResultValue);
+					m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue);
 				}
 			}
 			else if (AcceptType == ZenContentType::kCbPackage)
@@ -404,7 +405,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 
 						if (StoreLocal)
 						{
-							m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue);
+							m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
 						}
 
 						BinaryWriter MemStream;
@@ -486,7 +487,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 	if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary)
 	{
 		ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType));
-		m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+		m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
 
 		if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote))
 		{
@@ -528,7 +529,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 				  ValidAttachments.size());
 
 		Body.SetContentType(ZenContentType::kCbObject);
-		m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+		m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
 
 		const bool IsPartialRecord = TotalCount != static_cast<int32_t>(ValidAttachments.size());
 
@@ -611,7 +612,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 		CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
 		CacheValue.Value.SetContentType(ZenContentType::kCbObject);
 
-		m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue);
+		m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
 
 		const bool IsPartialRecord = Count.Valid != Count.Total;
 
@@ -1013,7 +1014,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	CacheValue.Value = IoBuffer(Record.GetSize());
 	Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize()));
 	CacheValue.Value.SetContentType(ZenContentType::kCbObject);
-	m_CacheStore.Put(Request.Key.Bucket, Request.Key.Hash, CacheValue);
+	m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Request.Key.Bucket, Request.Key.Hash, CacheValue);
 
 	const bool IsPartialRecord = Count.Valid != Count.Total;
 
@@ -1098,7 +1099,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
 		bool		  FoundLocalInvalid		 = false;
 		ZenCacheValue RecordCacheValue;
 
-		if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && m_CacheStore.Get(Key.Bucket, Key.Hash, RecordCacheValue))
+		if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) &&
+			m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, RecordCacheValue))
 		{
 			Request.RecordCacheValue = std::move(RecordCacheValue.Value);
 			if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject)
@@ -1229,7 +1231,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
 				Request.RecordObject = ObjectBuffer;
 				if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal))
 				{
-					m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
+					m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
 				}
 				ParseValues(Request);
 				Request.UsedUpstream = true;
@@ -1386,7 +1388,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
 				{
 					IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer();
 					Value.SetContentType(ZenContentType::kCompressedBinary);
-					m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = Value});
+					m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Value});
 					TransferredSize = Chunk.GetCompressedSize();
 				}
 				Succeeded = true;
@@ -1400,7 +1402,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
 		else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal))
 		{
 			ZenCacheValue ExistingValue;
-			if (m_CacheStore.Get(Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType()))
+			if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, ExistingValue) &&
+				IsCompressedBinary(ExistingValue.Value.GetContentType()))
 			{
 				Succeeded = true;
 			}
@@ -1483,7 +1486,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
 		ZenCacheValue CacheValue;
 		if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal))
 		{
-			if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType()))
+			if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, CacheValue) &&
+				IsCompressedBinary(CacheValue.Value.GetContentType()))
 			{
 				Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value));
 			}
@@ -1547,7 +1551,10 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
 						// that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive
 						// for us to keep the data only on the upstream server.
 						// if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal))
-						m_CacheStore.Put(Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value});
+						m_CacheStore.Put(ZenCacheStore::DefaultNamespace,
+										 Request.Key.Bucket,
+										 Request.Key.Hash,
+										 ZenCacheValue{Params.Value});
 						ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}",
 								  ChunkRequest.Key.Bucket,
 								  ChunkRequest.Key.Hash,
@@ -1803,7 +1810,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector<CacheKeyRequest>&
 			if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal))
 			{
 				ZenCacheValue CacheValue;
-				if (m_CacheStore.Get(RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue))
+				if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue))
 				{
 					Record.Exists	  = true;
 					Record.CacheValue = std::move(CacheValue.Value);
@@ -1838,7 +1845,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector<CacheKeyRequest>&
 
 			if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal))
 			{
-				m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
+				m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
 			}
 		};
 		m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete));
@@ -1935,7 +1942,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vector<cache::detail::Chunk
 		if (!Request->Exists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal))
 		{
 			ZenCacheValue CacheValue;
-			if (m_CacheStore.Get(Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue))
+			if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue))
 			{
 				if (IsCompressedBinary(CacheValue.Value.GetContentType()))
 				{
@@ -2004,7 +2011,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vector<CacheChunkRequest
 					}
 					else
 					{
-						m_CacheStore.Put(Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value});
+						m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value});
 					}
 				}
 				if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 3e2692859..05c80c5bf 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -232,7 +232,7 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 	WriteFile(Path, Object.GetBuffer().AsIoBuffer());
 }
 
-ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
+ZenCacheNamespace::ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir)
 : GcStorage(Gc)
 , GcContributor(Gc)
 , m_RootDir(RootDir)
@@ -248,12 +248,12 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
 #endif
 }
 
-ZenCacheStore::~ZenCacheStore()
+ZenCacheNamespace::~ZenCacheNamespace()
 {
 }
 
 bool
-ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
 {
 	ZEN_TRACE_CPU("Z$::Get");
 
@@ -291,7 +291,7 @@ ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheVal
 }
 
 void
-ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
@@ -327,7 +327,7 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
 }
 
 bool
-ZenCacheStore::DropBucket(std::string_view Bucket)
+ZenCacheNamespace::DropBucket(std::string_view Bucket)
 {
 	ZEN_INFO("dropping bucket '{}'", Bucket);
 
@@ -343,13 +343,13 @@ ZenCacheStore::DropBucket(std::string_view Bucket)
 }
 
 void
-ZenCacheStore::Flush()
+ZenCacheNamespace::Flush()
 {
 	m_DiskLayer.Flush();
 }
 
 void
-ZenCacheStore::Scrub(ScrubContext& Ctx)
+ZenCacheNamespace::Scrub(ScrubContext& Ctx)
 {
 	if (m_LastScrubTime == Ctx.ScrubTimestamp())
 	{
@@ -363,7 +363,7 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 }
 
 void
-ZenCacheStore::GatherReferences(GcContext& GcCtx)
+ZenCacheNamespace::GatherReferences(GcContext& GcCtx)
 {
 	Stopwatch  Timer;
 	const auto Guard =
@@ -377,14 +377,14 @@ ZenCacheStore::GatherReferences(GcContext& GcCtx)
 }
 
 void
-ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+ZenCacheNamespace::CollectGarbage(GcContext& GcCtx)
 {
 	m_MemLayer.Reset();
 	m_DiskLayer.CollectGarbage(GcCtx);
 }
 
 GcStorageSize
-ZenCacheStore::StorageSize() const
+ZenCacheNamespace::StorageSize() const
 {
 	return {.DiskSize = m_DiskLayer.TotalSize(), .MemorySize = m_MemLayer.TotalSize()};
 }
@@ -1960,49 +1960,23 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 void
 ZenCacheDiskLayer::DiscoverBuckets()
 {
-	FileSystemTraversal Traversal;
-	struct Visitor : public FileSystemTraversal::TreeVisitor
-	{
-		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
-							   [[maybe_unused]] const path_view&			 File,
-							   [[maybe_unused]] uint64_t					 FileSize) override
-		{
-		}
-
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
-		{
-			Dirs.push_back((decltype(Dirs)::value_type)(DirectoryName));
-			return false;
-		}
-
-		std::vector<std::filesystem::path::string_type> Dirs;
-	} Visit;
-
-	Traversal.TraverseFileSystem(m_RootDir, Visit);
+	DirectoryContent DirContent;
+	GetDirectoryContent(m_RootDir, DirectoryContent::IncludeDirsFlag, DirContent);
 
 	// Initialize buckets
 
 	RwLock::ExclusiveLockScope _(m_Lock);
 
-	for (const auto& BucketName : Visit.Dirs)
+	for (const std::filesystem::path& BucketPath : DirContent.Directories)
 	{
+		std::string BucketName = PathToUtf8(BucketPath.stem());
 		// New bucket needs to be created
-
-#if ZEN_PLATFORM_WINDOWS
-		std::string BucketName8 = WideToUtf8(BucketName);
-#else
-		const auto& BucketName8 = BucketName;
-#endif
-
-		if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end())
+		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 		{
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.try_emplace(BucketName8, BucketName8);
-
-			std::filesystem::path BucketPath = m_RootDir;
-			BucketPath /= BucketName8;
+			auto InsertResult = m_Buckets.try_emplace(BucketName, BucketName);
 
 			CacheBucket& Bucket = InsertResult.first->second;
 
@@ -2010,11 +1984,11 @@ ZenCacheDiskLayer::DiscoverBuckets()
 
 			if (Bucket.IsOk())
 			{
-				ZEN_INFO("Discovered bucket '{}'", BucketName8);
+				ZEN_INFO("Discovered bucket '{}'", BucketName);
 			}
 			else
 			{
-				ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName8, m_RootDir);
+				ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
 
 				m_Buckets.erase(InsertResult.first);
 			}
@@ -2102,6 +2076,162 @@ ZenCacheDiskLayer::TotalSize() const
 	return TotalSize;
 }
 
+//////////////////////////// ZenCacheStore
+
+static constexpr std::string_view ZenCacheNamespaceDirPrefix = "ns_";
+
+ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc)
+{
+	CreateDirectories(BasePath);
+
+	DirectoryContent DirContent;
+	GetDirectoryContent(BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+
+	std::vector<std::string> LegacyBuckets;
+	std::vector<std::string> Namespaces;
+	for (const std::filesystem::path& DirPath : DirContent.Directories)
+	{
+		std::string DirName = PathToUtf8(DirPath.stem());
+		if (DirName.starts_with(ZenCacheNamespaceDirPrefix))
+		{
+			Namespaces.push_back(DirName.substr(3));
+			continue;
+		}
+		LegacyBuckets.push_back(DirName);
+	}
+
+	ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size());
+
+	if (std::find(Namespaces.begin(), Namespaces.end(), DefaultNamespace) == Namespaces.end())
+	{
+		ZEN_INFO("Moving #{} legacy buckets to anonymous namespace", LegacyBuckets.size());
+		std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, DefaultNamespace);
+		CreateDirectories(DefaultNamespaceFolder);
+
+		// Move any non-namespace folders into the default namespace folder
+		for (const std::string& DirName : LegacyBuckets)
+		{
+			std::filesystem::path LegacyFolder = BasePath / DirName;
+			std::filesystem::path NewPath	   = DefaultNamespaceFolder / DirName;
+			std::error_code		  Ec;
+			std::filesystem::rename(LegacyFolder, NewPath, Ec);
+			if (Ec)
+			{
+				ZEN_ERROR("Unable to move '{}' to '{}', reason '{}'", LegacyFolder, NewPath, Ec.message());
+			}
+		}
+		Namespaces.push_back(std::string(DefaultNamespace));
+	}
+
+	for (const std::string& NamespaceName : Namespaces)
+	{
+		m_Namespaces[NamespaceName] =
+			std::make_unique<ZenCacheNamespace>(Gc, BasePath / fmt::format("{}{}", ZenCacheNamespaceDirPrefix, NamespaceName));
+	}
+}
+
+ZenCacheStore::~ZenCacheStore()
+{
+	m_Namespaces.clear();
+}
+
+bool
+ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
+	{
+		return Store->Get(Bucket, HashKey, OutValue);
+	}
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString());
+	return false;
+}
+
+void
+ZenCacheStore::Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value)
+{
+	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
+	{
+		return Store->Put(Bucket, HashKey, Value);
+	}
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString());
+}
+
+bool
+ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket)
+{
+	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
+	{
+		return Store->DropBucket(Bucket);
+	}
+	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}'", Namespace, Bucket);
+	return false;
+}
+
+void
+ZenCacheStore::Flush()
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Flush(); });
+}
+
+void
+ZenCacheStore::Scrub(ScrubContext& Ctx)
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); });
+}
+
+ZenCacheNamespace*
+ZenCacheStore::GetNamespace(std::string_view Namespace)
+{
+	RwLock::SharedLockScope _(m_NamespacesLock);
+	if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end())
+	{
+		return It->second.get();
+	}
+	return nullptr;
+}
+
+void
+ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const
+{
+	std::vector<std::pair<std::string, ZenCacheNamespace&> > Namespaces;
+	{
+		RwLock::SharedLockScope _(m_NamespacesLock);
+		Namespaces.reserve(m_Namespaces.size());
+		for (const auto& Entry : m_Namespaces)
+		{
+			Namespaces.push_back({Entry.first, *Entry.second});
+		}
+	}
+	for (auto& Entry : Namespaces)
+	{
+		Callback(Entry.first, Entry.second);
+	}
+}
+
+void
+ZenCacheStore::GatherReferences(GcContext& GcCtx)
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.GatherReferences(GcCtx); });
+}
+
+void
+ZenCacheStore::CollectGarbage(GcContext& GcCtx)
+{
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.CollectGarbage(GcCtx); });
+}
+
+GcStorageSize
+ZenCacheStore::StorageSize() const
+{
+	GcStorageSize Size;
+	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) {
+		GcStorageSize StoreSize = Store.StorageSize();
+		Size.MemorySize += StoreSize.MemorySize;
+		Size.DiskSize += StoreSize.DiskSize;
+	});
+	return Size;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_WITH_TESTS
@@ -2140,7 +2270,7 @@ TEST_CASE("z$.store")
 
 	CasGc Gc;
 
-	ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+	ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 	const int kIterationCount = 100;
 
@@ -2193,8 +2323,8 @@ TEST_CASE("z$.size")
 		GcStorageSize CacheSize;
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256);
 
@@ -2213,8 +2343,8 @@ TEST_CASE("z$.size")
 		}
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2236,8 +2366,8 @@ TEST_CASE("z$.size")
 		GcStorageSize CacheSize;
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64);
 
@@ -2256,8 +2386,8 @@ TEST_CASE("z$.size")
 		}
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -2294,9 +2424,9 @@ TEST_CASE("z$.gc")
 		};
 
 		{
-			CasGc		  Gc;
-			ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
-			const auto	  Bucket = "teardrinker"sv;
+			CasGc			  Gc;
+			ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
+			const auto		  Bucket = "teardrinker"sv;
 
 			// Create a cache record
 			const IoHash   Key = CreateKey(42);
@@ -2332,7 +2462,7 @@ TEST_CASE("z$.gc")
 		// Expect timestamps to be serialized
 		{
 			CasGc				Gc;
-			ZenCacheStore		Zcs(Gc, TempDir.Path() / "cache");
+			ZenCacheNamespace	Zcs(Gc, TempDir.Path() / "cache");
 			std::vector<IoHash> Keep;
 
 			// Collect garbage with 1 hour max cache duration
@@ -2353,7 +2483,7 @@ TEST_CASE("z$.gc")
 	{
 		ScopedTemporaryDirectory TempDir;
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path() / "cache");
 		const auto				 Bucket		 = "fortysixandtwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -2401,7 +2531,7 @@ TEST_CASE("z$.gc")
 	{
 		ScopedTemporaryDirectory TempDir;
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path() / "cache");
 		const auto				 Bucket		 = "rightintwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -2494,7 +2624,7 @@ TEST_CASE("z$.legacyconversion")
 	const std::string Bucket = "rightintwo";
 	{
 		CasGc					 Gc;
-		ZenCacheStore			 Zcs(Gc, TempDir.Path());
+		ZenCacheNamespace		 Zcs(Gc, TempDir.Path());
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
 		for (size_t i = 0; i < ChunkCount; i++)
@@ -2582,8 +2712,8 @@ TEST_CASE("z$.legacyconversion")
 	std::filesystem::remove(IndexPath);
 
 	{
-		CasGc		  Gc;
-		ZenCacheStore Zcs(Gc, TempDir.Path());
+		CasGc			  Gc;
+		ZenCacheNamespace Zcs(Gc, TempDir.Path());
 
 		for (size_t i = 0; i < ChunkCount; i += 2)
 		{
@@ -2643,9 +2773,9 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 
 		CreateDirectories(TempDir.Path());
 
-		WorkerThreadPool ThreadPool(4);
-		CasGc			 Gc;
-		ZenCacheStore	 Zcs(Gc, TempDir.Path());
+		WorkerThreadPool  ThreadPool(4);
+		CasGc			  Gc;
+		ZenCacheNamespace Zcs(Gc, TempDir.Path());
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 0c2a7c0b2..881285bc9 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -322,11 +322,11 @@ private:
 	ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
 };
 
-class ZenCacheStore final : public GcStorage, public GcContributor
+class ZenCacheNamespace final : public RefCounted, public GcStorage, public GcContributor
 {
 public:
-	ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir);
-	~ZenCacheStore();
+	ZenCacheNamespace(CasGc& Gc, const std::filesystem::path& RootDir);
+	~ZenCacheNamespace();
 
 	bool				  Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
 	void				  Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
@@ -349,8 +349,36 @@ private:
 	std::unique_ptr<ZenCacheTracker> m_AccessTracker;
 #endif
 
-	ZenCacheStore(const ZenCacheStore&) = delete;
-	ZenCacheStore& operator=(const ZenCacheStore&) = delete;
+	ZenCacheNamespace(const ZenCacheNamespace&) = delete;
+	ZenCacheNamespace& operator=(const ZenCacheNamespace&) = delete;
+};
+
+class ZenCacheStore final : public GcStorage, public GcContributor
+{
+public:
+	static constexpr std::string_view DefaultNamespace = "";
+
+	ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath);
+	~ZenCacheStore();
+
+	bool Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+	void Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+	bool DropBucket(std::string_view Namespace, std::string_view Bucket);
+	void Flush();
+	void Scrub(ScrubContext& Ctx);
+
+	virtual void		  GatherReferences(GcContext& GcCtx) override;
+	virtual void		  CollectGarbage(GcContext& GcCtx) override;
+	virtual GcStorageSize StorageSize() const override;
+
+private:
+	ZenCacheNamespace* GetNamespace(std::string_view Namespace);
+	void			   IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const;
+
+	typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NameSpaceMap;
+
+	mutable RwLock m_NamespacesLock;
+	NameSpaceMap   m_Namespaces;
 };
 
 void z$_forcelink();
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index aceb2df00..d18ae9e1a 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -804,29 +804,12 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId)
 void
 ProjectStore::Project::DiscoverOplogs()
 {
-	FileSystemTraversal Traversal;
-	struct Visitor : public FileSystemTraversal::TreeVisitor
-	{
-		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
-							   [[maybe_unused]] const path_view&			 File,
-							   [[maybe_unused]] uint64_t					 FileSize) override
-		{
-		}
-
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
-		{
-			Dirs.push_back(PathToUtf8(DirectoryName));
-			return false;
-		}
-
-		std::vector<std::string> Dirs;
-	} Visit;
-
-	Traversal.TraverseFileSystem(m_OplogStoragePath, Visit);
+	DirectoryContent DirContent;
+	GetDirectoryContent(m_OplogStoragePath, DirectoryContent::IncludeDirsFlag, DirContent);
 
-	for (const std::string& Dir : Visit.Dirs)
+	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
-		OpenOplog(Dir);
+		OpenOplog(PathToUtf8(DirPath.stem()));
 	}
 }
 
@@ -900,34 +883,18 @@ ProjectStore::BasePathForProject(std::string_view ProjectId)
 void
 ProjectStore::DiscoverProjects()
 {
-	FileSystemTraversal Traversal;
-	struct Visitor : public FileSystemTraversal::TreeVisitor
-	{
-		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
-							   [[maybe_unused]] const path_view&			 File,
-							   [[maybe_unused]] uint64_t					 FileSize) override
-		{
-		}
-
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
-		{
-			Dirs.push_back(PathToUtf8(DirectoryName));
-			return false;
-		}
-
-		std::vector<std::string> Dirs;
-	} Visit;
-
 	if (!std::filesystem::exists(m_ProjectBasePath))
 	{
 		return;
 	}
 
-	Traversal.TraverseFileSystem(m_ProjectBasePath, Visit);
+	DirectoryContent DirContent;
+	GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
 
-	for (const auto& Dir : Visit.Dirs)
+	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
-		Project* Project = OpenProject(Dir);
+		std::string DirName = PathToUtf8(DirPath.stem());
+		Project*	Project = OpenProject(DirName);
 
 		if (Project)
 		{
diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h
index 2edc6dc49..1deaf00a5 100644
--- a/zenserver/upstream/upstreamapply.h
+++ b/zenserver/upstream/upstreamapply.h
@@ -23,7 +23,7 @@ class CbObjectWriter;
 class CidStore;
 class CloudCacheTokenProvider;
 class WorkerThreadPool;
-class ZenCacheStore;
+class ZenCacheNamespace;
 struct CloudCacheClientOptions;
 struct UpstreamAuthConfig;
 
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index dba80faa9..c870e0773 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -1517,7 +1517,7 @@ private:
 		ZenCacheValue		  CacheValue;
 		std::vector<IoBuffer> Payloads;
 
-		if (!m_CacheStore.Get(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue))
+		if (!m_CacheStore.Get(ZenCacheStore::DefaultNamespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue))
 		{
 			ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", CacheRecord.Key.Bucket, CacheRecord.Key.Hash);
 			return;
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index 1946169c4..d490678b5 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -162,7 +162,7 @@ BlockStore::Initialize(const std::filesystem::path&			  BlocksBasePath,
 					{
 						continue;
 					}
-					std::string FileName = Path.stem().string();
+					std::string FileName = PathToUtf8(Path.stem());
 					uint32_t	BlockIndex;
 					bool		OK = ParseHexNumber(FileName, BlockIndex);
 					if (!OK)
@@ -1053,35 +1053,15 @@ namespace {
 
 	std::vector<std::filesystem::path> GetDirectoryContent(std::filesystem::path RootDir, bool Files, bool Directories)
 	{
-		FileSystemTraversal Traversal;
-		struct Visitor : public FileSystemTraversal::TreeVisitor
-		{
-			virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
-			{
-				if (Files)
-				{
-					Items.push_back(Parent / File);
-				}
-			}
-
-			virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& Dir) override
-			{
-				if (Directories)
-				{
-					Items.push_back(Parent / Dir);
-				}
-				return true;
-			}
-
-			bool							   Files;
-			bool							   Directories;
-			std::vector<std::filesystem::path> Items;
-		} Visit;
-		Visit.Files		  = Files;
-		Visit.Directories = Directories;
-
-		Traversal.TraverseFileSystem(RootDir, Visit);
-		return Visit.Items;
+		DirectoryContent DirectoryContent;
+		GetDirectoryContent(RootDir,
+							DirectoryContent::RecursiveFlag | (Files ? DirectoryContent::IncludeFilesFlag : 0) |
+								(Directories ? DirectoryContent::IncludeDirsFlag : 0),
+							DirectoryContent);
+		std::vector<std::filesystem::path> Result;
+		Result.insert(Result.end(), DirectoryContent.Directories.begin(), DirectoryContent.Directories.end());
+		Result.insert(Result.end(), DirectoryContent.Files.begin(), DirectoryContent.Files.end());
+		return Result;
 	};
 
 	static IoBuffer CreateChunk(uint64_t Size)