per namespace/project cas prep refactor (#470)

- Refactor so we can have more than one cas store for project store and cache.
- Refactor `UpstreamCacheClient` so it is not tied to a specific CidStore
- Refactor scrub to keep the GC interface ScrubStorage function separate from scrub accessor functions (renamed to Scrub).
- Refactor storage size to keep GC interface StorageSize function separate from size accessor functions (renamed to TotalSize)
- Refactor cache storage so  `ZenCacheDiskLayer::CacheBucket` implements GcStorage interface rather than `ZenCacheNamespace`

1 files changed, 134 insertions, 33 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 317a419eb..9600133f3 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -235,13 +235,11 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpProjectService::HttpProjectService(CidStore&		  Store,
-									   ProjectStore*	  Projects,
+HttpProjectService::HttpProjectService(ProjectStore*	  Projects,
 									   HttpStatusService& StatusService,
 									   HttpStatsService&  StatsService,
 									   AuthMgr&			  AuthMgr)
 : m_Log(logging::Get("project"))
-, m_CidStore(Store)
 , m_ProjectStore(Projects)
 , m_StatusService(StatusService)
 , m_StatsService(StatsService)
@@ -407,8 +405,45 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 {
 	ZEN_TRACE_CPU("ProjectService::Stats");
 
-	const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
-	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
+	bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true";
+
+	const GcStorageSize StoreSize			 = m_ProjectStore->StorageSize();
+	uint64_t			TotalChunkHitCount	 = 0;
+	uint64_t			TotalChunkMissCount	 = 0;
+	uint64_t			TotalChunkWriteCount = 0;
+	CidStoreSize		TotalCidSize;
+
+	tsl::robin_map<CidStore*, std::string> UniqueStores;
+	{
+		m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) {
+			CidStore* Store = &Project.GetCidStore();
+			if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
+			{
+				UniqueStores.insert_or_assign(Store, Project.Identifier);
+			}
+			else
+			{
+				UniqueStores.insert_or_assign(Store, std::string{});
+			}
+		});
+
+		for (auto It : UniqueStores)
+		{
+			CidStore* ChunkStore = It.first;
+
+			CidStoreStats ChunkStoreStats = ChunkStore->Stats();
+			CidStoreSize  ChunkStoreSize  = ChunkStore->TotalSize();
+
+			TotalChunkHitCount += ChunkStoreStats.HitCount;
+			TotalChunkMissCount += ChunkStoreStats.MissCount;
+			TotalChunkWriteCount += ChunkStoreStats.WriteCount;
+
+			TotalCidSize.TinySize += ChunkStoreSize.TinySize;
+			TotalCidSize.SmallSize += ChunkStoreSize.SmallSize;
+			TotalCidSize.LargeSize += ChunkStoreSize.LargeSize;
+			TotalCidSize.TotalSize += ChunkStoreSize.TotalSize;
+		}
+	}
 
 	CbObjectWriter Cbo;
 
@@ -460,12 +495,66 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 	{
 		Cbo.BeginObject("size");
 		{
-			Cbo << "tiny" << CidSize.TinySize;
-			Cbo << "small" << CidSize.SmallSize;
-			Cbo << "large" << CidSize.LargeSize;
-			Cbo << "total" << CidSize.TotalSize;
+			Cbo << "tiny" << TotalCidSize.TinySize;
+			Cbo << "small" << TotalCidSize.SmallSize;
+			Cbo << "large" << TotalCidSize.LargeSize;
+			Cbo << "total" << TotalCidSize.TotalSize;
 		}
 		Cbo.EndObject();
+
+		if (ShowCidStoreStats)
+		{
+			Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
+			const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
+			Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
+
+			Cbo.BeginObject("store");
+
+			auto OutputStats = [&](CidStore& ChunkStore) {
+				CidStoreStats StoreStats = ChunkStore.Stats();
+				Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
+				const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
+				Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
+				EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
+				EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
+			};
+
+			if (UniqueStores.size() > 1)
+			{
+				Cbo.BeginArray("projects");
+				for (auto It : UniqueStores)
+				{
+					CidStore*		   ChunkStore	  = It.first;
+					const std::string& ProjectId	  = It.second;
+					CidStoreSize	   ChunkStoreSize = ChunkStore->TotalSize();
+
+					Cbo.BeginObject();
+					{
+						Cbo << "project" << ProjectId;
+						Cbo.BeginObject("stats");
+						OutputStats(*ChunkStore);
+						Cbo.EndObject();
+
+						Cbo.BeginObject("size");
+						{
+							Cbo << "tiny" << ChunkStoreSize.TinySize;
+							Cbo << "small" << ChunkStoreSize.SmallSize;
+							Cbo << "large" << ChunkStoreSize.LargeSize;
+							Cbo << "total" << ChunkStoreSize.TotalSize;
+						}
+						Cbo.EndObject();
+					}
+					Cbo.EndObject();
+				}
+				Cbo.EndArray();	 // projects
+			}
+			else if (UniqueStores.size() != 0)
+			{
+				CidStore& ChunkStore = *UniqueStores.begin()->first;
+				OutputStats(ChunkStore);
+			}
+			Cbo.EndObject();
+		}
 	}
 	Cbo.EndObject();
 
@@ -1125,6 +1214,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchOplog(OplogId);
 
+	CidStore& ChunkStore = Project->GetCidStore();
+
 	ProjectStore::Oplog& Oplog = *FoundLog;
 
 	IoBuffer Payload = HttpReq.ReadPayload();
@@ -1137,7 +1228,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	std::vector<IoHash> MissingChunks;
 
 	CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
-		if (m_CidStore.ContainsChunk(Hash))
+		if (ChunkStore.ContainsChunk(Hash))
 		{
 			// Return null attachment as we already have it, no point in reading it and storing it again
 			return {};
@@ -1393,6 +1484,8 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchOplog(OplogId);
 
+	CidStore& ChunkStore = Project->GetCidStore();
+
 	ProjectStore::Oplog& Oplog = *FoundLog;
 
 	if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
@@ -1407,7 +1500,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 
 				Op.IterateAttachments([&](CbFieldView FieldView) {
 					const IoHash AttachmentHash = FieldView.AsAttachment();
-					IoBuffer	 Payload		= m_CidStore.FindChunkByCid(AttachmentHash);
+					IoBuffer	 Payload		= ChunkStore.FindChunkByCid(AttachmentHash);
 					if (Payload)
 					{
 						switch (Payload.GetContentType())
@@ -2036,11 +2129,14 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+			CidStore& ChunkStore = Project.GetCidStore();
+
 			Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
-				Oplog.IterateOplogWithKey(
-					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
-						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
-					});
+				Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t	 LSN,
+																														const Oid&	 Key,
+																														CbObjectView Op) {
+					CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+				});
 			});
 		});
 
@@ -2054,8 +2150,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
 			m_ProjectStore->DiscoverProjects();
 
 			m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
-				std::vector<std::string> OpLogs = Project.ScanForOplogs();
-				CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+				CidStore&				 ChunkStore = Project.GetCidStore();
+				std::vector<std::string> OpLogs		= Project.ScanForOplogs();
+				CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
 			});
 		}
 		Cbo.EndArray();
@@ -2084,7 +2181,8 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
-	ProjectStore::Project& Project = *FoundProject.Get();
+	ProjectStore::Project& Project	  = *FoundProject.Get();
+	CidStore&			   ChunkStore = Project.GetCidStore();
 
 	if (CSV)
 	{
@@ -2092,10 +2190,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
-			Oplog.IterateOplogWithKey(
-				[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
-					CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
-				});
+			Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t	 LSN,
+																													const Oid&	 Key,
+																													CbObjectView Op) {
+				CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+			});
 		});
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
@@ -2105,7 +2204,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 		std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
 		Cbo.BeginArray("projects");
 		{
-			CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+			CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2141,16 +2240,17 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Project& Project = *FoundProject.Get();
-	ProjectStore::Oplog&   Oplog   = *FoundLog;
+	ProjectStore::Project& Project	  = *FoundProject.Get();
+	CidStore&			   ChunkStore = Project.GetCidStore();
+	ProjectStore::Oplog&   Oplog	  = *FoundLog;
 	if (CSV)
 	{
 		ExtendableStringBuilder<4096> CSVWriter;
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		Oplog.IterateOplogWithKey(
-			[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
-				CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+			[this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+				CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
 			});
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
@@ -2159,7 +2259,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		CbObjectWriter Cbo;
 		Cbo.BeginArray("oplogs");
 		{
-			CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+			CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2204,9 +2304,10 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 									 fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId));
 	}
 
-	const Oid			   ObjId   = Oid::FromHexString(OpId);
-	ProjectStore::Project& Project = *FoundProject.Get();
-	ProjectStore::Oplog&   Oplog   = *FoundLog;
+	const Oid			   ObjId	  = Oid::FromHexString(OpId);
+	ProjectStore::Project& Project	  = *FoundProject.Get();
+	CidStore&			   ChunkStore = Project.GetCidStore();
+	ProjectStore::Oplog&   Oplog	  = *FoundLog;
 
 	std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId);
 	if (!Op.has_value())
@@ -2224,7 +2325,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		ExtendableStringBuilder<4096> CSVWriter;
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
-		CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
+		CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
 	else
@@ -2232,7 +2333,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		CbObjectWriter Cbo;
 		Cbo.BeginArray("ops");
 		{
-			CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
+			CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());