Add namespacecachestore layer to allow multiple structured cache namespaces

7 files changed, 299 insertions, 39 deletions

diff --git a/zenserver/cache/namespacecachestore.cpp b/zenserver/cache/namespacecachestore.cpp
new file mode 100644
index 000000000..82ac40c62
--- /dev/null
+++ b/zenserver/cache/namespacecachestore.cpp
@@ -0,0 +1,207 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "namespacecachestore.h"
+#include "structuredcachestore.h"
+
+namespace zen {
+
+const char* NamespaceDirPrefix = "ns_";
+
+NamespaceCacheStore::NamespaceCacheStore(std::filesystem::path BasePath, CasGc& Gc)
+: GcStorage(Gc)
+, GcContributor(Gc)
+, m_Log(logging::Get("namespacecachestore"))
+, m_BasePath(BasePath)
+{
+	CreateDirectories(m_BasePath);
+	std::vector<std::string> ExistingFolders = FindExistingFolders();
+
+	std::vector<std::string> LegacyBuckets;
+	std::vector<std::string> Namespaces;
+	for (const std::string& DirName : ExistingFolders)
+	{
+		if (DirName.starts_with(NamespaceDirPrefix))
+		{
+			Namespaces.push_back(DirName.substr(3));
+			continue;
+		}
+		LegacyBuckets.push_back(DirName);
+	}
+	if (Namespaces.empty() && !LegacyBuckets.empty())
+	{
+		// If we find no namespaces, but any unknown folders we assume we have a legacy folder
+		// and move any existing folders into a default namespace
+		std::filesystem::path DefaultfNamespaceFolder = m_BasePath / NamespaceDirPrefix;
+		CreateDirectories(DefaultfNamespaceFolder);
+		for (const std::string& DirName : LegacyBuckets)
+		{
+			std::filesystem::path LegacyFolder = m_BasePath / DirName;
+			std::filesystem::path NewPath	   = DefaultfNamespaceFolder / DirName;
+			std::filesystem::rename(LegacyFolder, NewPath);
+		}
+		Namespaces.push_back("");
+	}
+
+	for (const std::string& NamespaceName : Namespaces)
+	{
+		ZenCacheStore* Store		= new ZenCacheStore(Gc, m_BasePath / (NamespaceDirPrefix + NamespaceName));
+		m_Namespaces[NamespaceName] = Store;
+	}
+}
+
+NamespaceCacheStore::~NamespaceCacheStore()
+{
+	for (const auto& Entry : m_Namespaces)
+	{
+		delete Entry.second;
+	}
+	m_Namespaces.clear();
+}
+
+std::vector<std::string>
+NamespaceCacheStore::FindExistingFolders() const
+{
+	FileSystemTraversal Traversal;
+	struct Visitor : public FileSystemTraversal::TreeVisitor
+	{
+		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
+							   [[maybe_unused]] const path_view&			 File,
+							   [[maybe_unused]] uint64_t					 FileSize) override
+		{
+		}
+
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+		{
+			std::string DirName8 = WideToUtf8(DirectoryName);
+			Dirs.push_back(DirName8);
+			return false;
+		}
+
+		std::vector<std::string> Dirs;
+	} Visit;
+
+	Traversal.TraverseFileSystem(m_BasePath, Visit);
+	return Visit.Dirs;
+}
+
+bool
+NamespaceCacheStore::Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+	ZenCacheStore* Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return false;
+	}
+	return Store->Get(Bucket, HashKey, OutValue);
+}
+
+void
+NamespaceCacheStore::Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value)
+{
+	ZenCacheStore* Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return;
+	}
+	Store->Put(Bucket, HashKey, Value);
+}
+
+bool
+NamespaceCacheStore::DropBucket(const std::string& Namespace, std::string_view Bucket)
+{
+	ZenCacheStore* Store = GetStore(Namespace);
+	if (!Store)
+	{
+		return false;
+	}
+	return Store->DropBucket(Bucket);
+}
+
+void
+NamespaceCacheStore::Flush()
+{
+	std::vector<ZenCacheStore*> Stores;
+	RwLock::SharedLockScope		_(m_NamespacesLock);
+	Stores.reserve(m_Namespaces.size());
+	for (const auto& Entry : m_Namespaces)
+	{
+		Stores.push_back(Entry.second);
+	}
+	_.ReleaseNow();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->Flush();
+	}
+}
+
+void
+NamespaceCacheStore::Scrub(ScrubContext& Ctx)
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->Scrub(Ctx);
+	}
+}
+
+ZenCacheStore*
+NamespaceCacheStore::GetStore(const std::string& Namespace)
+{
+	RwLock::SharedLockScope _(m_NamespacesLock);
+	if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end())
+	{
+		return It->second;
+	}
+	return nullptr;
+}
+
+std::vector<ZenCacheStore*>
+NamespaceCacheStore::GetAllStores() const
+{
+	std::vector<ZenCacheStore*> Stores;
+	RwLock::SharedLockScope		_(m_NamespacesLock);
+	Stores.reserve(m_Namespaces.size());
+	for (const auto& Entry : m_Namespaces)
+	{
+		Stores.push_back(Entry.second);
+	}
+	return Stores;
+}
+
+void
+NamespaceCacheStore::GatherReferences(GcContext& GcCtx)
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->GatherReferences(GcCtx);
+	}
+}
+
+void
+NamespaceCacheStore::CollectGarbage(GcContext& GcCtx)
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	for (ZenCacheStore* Store : Stores)
+	{
+		Store->CollectGarbage(GcCtx);
+	}
+}
+
+GcStorageSize
+NamespaceCacheStore::StorageSize() const
+{
+	std::vector<ZenCacheStore*> Stores = GetAllStores();
+	GcStorageSize				Size;
+	for (ZenCacheStore* Store : Stores)
+	{
+		GcStorageSize StoreSize = Store->StorageSize();
+		Size.MemorySize += StoreSize.MemorySize;
+		Size.DiskSize += StoreSize.DiskSize;
+	}
+	return Size;
+}
+
+}  // namespace zen
diff --git a/zenserver/cache/namespacecachestore.h b/zenserver/cache/namespacecachestore.h
new file mode 100644
index 000000000..85f4150bf
--- /dev/null
+++ b/zenserver/cache/namespacecachestore.h
@@ -0,0 +1,49 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+#include <zencore/refcount.h>
+
+#include <zenstore/gc.h>
+
+#include <string>
+
+namespace zen {
+
+class ScrubContext;
+class ZenCacheStore;
+struct ZenCacheValue;
+
+/** NamespaceCache Store
+ */
+class NamespaceCacheStore : public RefCounted, public GcStorage, public GcContributor
+{
+public:
+	NamespaceCacheStore(std::filesystem::path BasePath, CasGc& Gc);
+	~NamespaceCacheStore();
+
+	bool Get(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+	void Put(const std::string& Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+	bool DropBucket(const std::string& Namespace, std::string_view Bucket);
+	void Flush();
+	void Scrub(ScrubContext& Ctx);
+
+	spdlog::logger&				 Log() { return m_Log; }
+	const std::filesystem::path& BasePath() const { return m_BasePath; }
+
+	virtual void		  GatherReferences(GcContext& GcCtx) override;
+	virtual void		  CollectGarbage(GcContext& GcCtx) override;
+	virtual GcStorageSize StorageSize() const override;
+
+private:
+	std::vector<std::string>						FindExistingFolders() const;
+	ZenCacheStore*									GetStore(const std::string& Namespace);
+	std::vector<ZenCacheStore*>						GetAllStores() const;
+	spdlog::logger&									m_Log;
+	std::filesystem::path							m_BasePath;
+	mutable RwLock									m_NamespacesLock;
+	std::unordered_map<std::string, ZenCacheStore*> m_Namespaces;
+};
+
+}  // namespace zen
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index e1d9de976..276c99081 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -21,6 +21,7 @@
 
 //#include "cachekey.h"
 #include "monitoring/httpstats.h"
+#include "namespacecachestore.h"
 #include "structuredcachestore.h"
 #include "upstream/jupiter.h"
 #include "upstream/upstreamcache.h"
@@ -72,13 +73,13 @@ struct PutRequestData
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore&	  InCacheStore,
-													   CidStore&		  InCidStore,
-													   HttpStatsService&  StatsService,
-													   HttpStatusService& StatusService,
-													   UpstreamCache&	  UpstreamCache)
+HttpStructuredCacheService::HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore,
+													   CidStore&			InCidStore,
+													   HttpStatsService&	StatsService,
+													   HttpStatusService&	StatusService,
+													   UpstreamCache&		UpstreamCache)
 : m_Log(logging::Get("cache"))
-, m_CacheStore(InCacheStore)
+, m_CacheStore(InNamespaceCacheStore)
 , m_StatsService(StatsService)
 , m_StatusService(StatusService)
 , m_CidStore(InCidStore)
@@ -176,7 +177,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
 		case HttpVerb::kDelete:
 			// Drop bucket
 
-			if (m_CacheStore.DropBucket(Bucket))
+			if (m_CacheStore.DropBucket("", Bucket))
 			{
 				return Request.WriteResponse(HttpResponseCode::OK);
 			}
@@ -225,7 +226,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 		return Request.WriteResponse(HttpResponseCode::OK);
 	}
 
-	if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, ClientResultValue))
+	if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) && m_CacheStore.Get("", Ref.BucketSegment, Ref.HashKey, ClientResultValue))
 	{
 		Success					   = true;
 		ZenContentType ContentType = ClientResultValue.Value.GetContentType();
@@ -350,7 +351,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 
 				if (Success && StoreLocal)
 				{
-					m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, ClientResultValue);
+					m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, ClientResultValue);
 				}
 			}
 			else if (AcceptType == ZenContentType::kCbPackage)
@@ -404,7 +405,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 
 						if (StoreLocal)
 						{
-							m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue);
+							m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, CacheValue);
 						}
 
 						BinaryWriter MemStream;
@@ -486,7 +487,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 	if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary)
 	{
 		ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType));
-		m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+		m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, {.Value = Body});
 
 		if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote))
 		{
@@ -528,7 +529,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 				  ValidAttachments.size());
 
 		Body.SetContentType(ZenContentType::kCbObject);
-		m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+		m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, {.Value = Body});
 
 		const bool IsPartialRecord = TotalCount != static_cast<int32_t>(ValidAttachments.size());
 
@@ -611,7 +612,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 		CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
 		CacheValue.Value.SetContentType(ZenContentType::kCbObject);
 
-		m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue);
+		m_CacheStore.Put("", Ref.BucketSegment, Ref.HashKey, CacheValue);
 
 		const bool IsPartialRecord = Count.Valid != Count.Total;
 
@@ -1013,7 +1014,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	CacheValue.Value = IoBuffer(Record.GetSize());
 	Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize()));
 	CacheValue.Value.SetContentType(ZenContentType::kCbObject);
-	m_CacheStore.Put(Request.Key.Bucket, Request.Key.Hash, CacheValue);
+	m_CacheStore.Put("", Request.Key.Bucket, Request.Key.Hash, CacheValue);
 
 	const bool IsPartialRecord = Count.Valid != Count.Total;
 
@@ -1098,7 +1099,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
 		bool		  FoundLocalInvalid		 = false;
 		ZenCacheValue RecordCacheValue;
 
-		if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) && m_CacheStore.Get(Key.Bucket, Key.Hash, RecordCacheValue))
+		if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) &&
+			m_CacheStore.Get("", Key.Bucket, Key.Hash, RecordCacheValue))
 		{
 			Request.RecordCacheValue = std::move(RecordCacheValue.Value);
 			if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject)
@@ -1229,7 +1231,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
 				Request.RecordObject = ObjectBuffer;
 				if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal))
 				{
-					m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
+					m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
 				}
 				ParseValues(Request);
 				Request.UsedUpstream = true;
@@ -1386,7 +1388,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
 				{
 					IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer();
 					Value.SetContentType(ZenContentType::kCompressedBinary);
-					m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = Value});
+					m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = Value});
 					TransferredSize = Chunk.GetCompressedSize();
 				}
 				Succeeded = true;
@@ -1400,7 +1402,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
 		else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal))
 		{
 			ZenCacheValue ExistingValue;
-			if (m_CacheStore.Get(Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType()))
+			if (m_CacheStore.Get("", Key.Bucket, Key.Hash, ExistingValue) && IsCompressedBinary(ExistingValue.Value.GetContentType()))
 			{
 				Succeeded = true;
 			}
@@ -1483,7 +1485,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
 		ZenCacheValue CacheValue;
 		if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal))
 		{
-			if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType()))
+			if (m_CacheStore.Get("", Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType()))
 			{
 				Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value));
 			}
@@ -1547,7 +1549,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
 						// that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive
 						// for us to keep the data only on the upstream server.
 						// if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal))
-						m_CacheStore.Put(Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value});
+						m_CacheStore.Put("", Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value});
 						ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}",
 								  ChunkRequest.Key.Bucket,
 								  ChunkRequest.Key.Hash,
@@ -1803,7 +1805,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector<CacheKeyRequest>&
 			if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal))
 			{
 				ZenCacheValue CacheValue;
-				if (m_CacheStore.Get(RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue))
+				if (m_CacheStore.Get("", RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue))
 				{
 					Record.Exists	  = true;
 					Record.CacheValue = std::move(CacheValue.Value);
@@ -1838,7 +1840,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector<CacheKeyRequest>&
 
 			if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal))
 			{
-				m_CacheStore.Put(Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
+				m_CacheStore.Put("", Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
 			}
 		};
 		m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete));
@@ -1935,7 +1937,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vector<cache::detail::Chunk
 		if (!Request->Exists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal))
 		{
 			ZenCacheValue CacheValue;
-			if (m_CacheStore.Get(Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue))
+			if (m_CacheStore.Get("", Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue))
 			{
 				if (IsCompressedBinary(CacheValue.Value.GetContentType()))
 				{
@@ -2004,7 +2006,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vector<CacheChunkRequest
 					}
 					else
 					{
-						m_CacheStore.Put(Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value});
+						m_CacheStore.Put("", Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value});
 					}
 				}
 				if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index 00c4260aa..7e9847838 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -25,7 +25,7 @@ class CbObjectView;
 struct PutRequestData;
 class ScrubContext;
 class UpstreamCache;
-class ZenCacheStore;
+class NamespaceCacheStore;
 enum class CachePolicy : uint32_t;
 
 namespace cache::detail {
@@ -64,11 +64,11 @@ namespace cache::detail {
 class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider
 {
 public:
-	HttpStructuredCacheService(ZenCacheStore&	  InCacheStore,
-							   CidStore&		  InCidStore,
-							   HttpStatsService&  StatsService,
-							   HttpStatusService& StatusService,
-							   UpstreamCache&	  UpstreamCache);
+	HttpStructuredCacheService(NamespaceCacheStore& InNamespaceCacheStore,
+							   CidStore&			InCidStore,
+							   HttpStatsService&	StatsService,
+							   HttpStatusService&	StatusService,
+							   UpstreamCache&		UpstreamCache);
 	~HttpStructuredCacheService();
 
 	virtual const char* BaseUri() const override;
@@ -140,7 +140,7 @@ private:
 
 	spdlog::logger&			 Log() { return m_Log; }
 	spdlog::logger&			 m_Log;
-	ZenCacheStore&			 m_CacheStore;
+	NamespaceCacheStore&	 m_CacheStore;
 	HttpStatsService&		 m_StatsService;
 	HttpStatusService&		 m_StatusService;
 	CidStore&				 m_CidStore;
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index dba80faa9..49f384774 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -19,6 +19,7 @@
 #include <zenstore/cidstore.h>
 
 #include <auth/authmgr.h>
+#include "cache/namespacecachestore.h"
 #include "cache/structuredcache.h"
 #include "cache/structuredcachestore.h"
 #include "diag/logging.h"
@@ -1173,7 +1174,7 @@ namespace detail {
 class UpstreamCacheImpl final : public UpstreamCache
 {
 public:
-	UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
+	UpstreamCacheImpl(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore)
 	: m_Log(logging::Get("upstream"))
 	, m_Options(Options)
 	, m_CacheStore(CacheStore)
@@ -1517,7 +1518,7 @@ private:
 		ZenCacheValue		  CacheValue;
 		std::vector<IoBuffer> Payloads;
 
-		if (!m_CacheStore.Get(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue))
+		if (!m_CacheStore.Get("", CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue))
 		{
 			ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", CacheRecord.Key.Bucket, CacheRecord.Key.Hash);
 			return;
@@ -1687,7 +1688,7 @@ private:
 
 	spdlog::logger&								   m_Log;
 	UpstreamCacheOptions						   m_Options;
-	ZenCacheStore&								   m_CacheStore;
+	NamespaceCacheStore&						   m_CacheStore;
 	CidStore&									   m_CidStore;
 	UpstreamQueue								   m_UpstreamQueue;
 	std::shared_mutex							   m_EndpointsMutex;
@@ -1712,7 +1713,7 @@ UpstreamEndpoint::CreateJupiterEndpoint(const CloudCacheClientOptions& Options,
 }
 
 std::unique_ptr<UpstreamCache>
-UpstreamCache::Create(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
+UpstreamCache::Create(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore)
 {
 	return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore);
 }
diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h
index 6f18b3119..54386e80d 100644
--- a/zenserver/upstream/upstreamcache.h
+++ b/zenserver/upstream/upstreamcache.h
@@ -24,7 +24,7 @@ class CbObjectView;
 class CbPackage;
 class CbObjectWriter;
 class CidStore;
-class ZenCacheStore;
+class NamespaceCacheStore;
 struct CloudCacheClientOptions;
 class CloudCacheTokenProvider;
 struct ZenStructuredCacheClientOptions;
@@ -206,7 +206,7 @@ public:
 
 	virtual void GetStatus(CbObjectWriter& CbO) = 0;
 
-	static std::unique_ptr<UpstreamCache> Create(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore);
+	static std::unique_ptr<UpstreamCache> Create(const UpstreamCacheOptions& Options, NamespaceCacheStore& CacheStore, CidStore& CidStore);
 };
 
 }  // namespace zen
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index abaec888a..9b7083312 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -102,6 +102,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #include "admin/admin.h"
 #include "auth/authmgr.h"
 #include "auth/authservice.h"
+#include "cache/namespacecachestore.h"
 #include "cache/structuredcache.h"
 #include "cache/structuredcachestore.h"
 #include "compute/function.h"
@@ -611,7 +612,7 @@ private:
 	zen::GcScheduler								 m_GcScheduler{m_CasGc};
 	std::unique_ptr<zen::CasStore>					 m_CasStore{zen::CreateCasStore(m_CasGc)};
 	std::unique_ptr<zen::CidStore>					 m_CidStore;
-	std::unique_ptr<zen::ZenCacheStore>				 m_CacheStore;
+	std::unique_ptr<zen::NamespaceCacheStore>		 m_CacheStore;
 	zen::CasScrubber								 m_Scrubber{*m_CasStore};
 	zen::HttpTestService							 m_TestService;
 	zen::HttpTestingService							 m_TestingService;
@@ -755,7 +756,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 	using namespace std::literals;
 
 	ZEN_INFO("instantiating structured cache service");
-	m_CacheStore = std::make_unique<ZenCacheStore>(m_CasGc, m_DataRoot / "cache");
+	m_CacheStore = std::make_unique<NamespaceCacheStore>(m_DataRoot / "cache", m_CasGc);
 
 	const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig;