revert multi-cid store (#475)

1 files changed, 33 insertions, 134 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 9600133f3..317a419eb 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -235,11 +235,13 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpProjectService::HttpProjectService(ProjectStore*	  Projects,
+HttpProjectService::HttpProjectService(CidStore&		  Store,
+									   ProjectStore*	  Projects,
 									   HttpStatusService& StatusService,
 									   HttpStatsService&  StatsService,
 									   AuthMgr&			  AuthMgr)
 : m_Log(logging::Get("project"))
+, m_CidStore(Store)
 , m_ProjectStore(Projects)
 , m_StatusService(StatusService)
 , m_StatsService(StatsService)
@@ -405,45 +407,8 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 {
 	ZEN_TRACE_CPU("ProjectService::Stats");
 
-	bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true";
-
-	const GcStorageSize StoreSize			 = m_ProjectStore->StorageSize();
-	uint64_t			TotalChunkHitCount	 = 0;
-	uint64_t			TotalChunkMissCount	 = 0;
-	uint64_t			TotalChunkWriteCount = 0;
-	CidStoreSize		TotalCidSize;
-
-	tsl::robin_map<CidStore*, std::string> UniqueStores;
-	{
-		m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) {
-			CidStore* Store = &Project.GetCidStore();
-			if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
-			{
-				UniqueStores.insert_or_assign(Store, Project.Identifier);
-			}
-			else
-			{
-				UniqueStores.insert_or_assign(Store, std::string{});
-			}
-		});
-
-		for (auto It : UniqueStores)
-		{
-			CidStore* ChunkStore = It.first;
-
-			CidStoreStats ChunkStoreStats = ChunkStore->Stats();
-			CidStoreSize  ChunkStoreSize  = ChunkStore->TotalSize();
-
-			TotalChunkHitCount += ChunkStoreStats.HitCount;
-			TotalChunkMissCount += ChunkStoreStats.MissCount;
-			TotalChunkWriteCount += ChunkStoreStats.WriteCount;
-
-			TotalCidSize.TinySize += ChunkStoreSize.TinySize;
-			TotalCidSize.SmallSize += ChunkStoreSize.SmallSize;
-			TotalCidSize.LargeSize += ChunkStoreSize.LargeSize;
-			TotalCidSize.TotalSize += ChunkStoreSize.TotalSize;
-		}
-	}
+	const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
 
 	CbObjectWriter Cbo;
 
@@ -495,66 +460,12 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 	{
 		Cbo.BeginObject("size");
 		{
-			Cbo << "tiny" << TotalCidSize.TinySize;
-			Cbo << "small" << TotalCidSize.SmallSize;
-			Cbo << "large" << TotalCidSize.LargeSize;
-			Cbo << "total" << TotalCidSize.TotalSize;
+			Cbo << "tiny" << CidSize.TinySize;
+			Cbo << "small" << CidSize.SmallSize;
+			Cbo << "large" << CidSize.LargeSize;
+			Cbo << "total" << CidSize.TotalSize;
 		}
 		Cbo.EndObject();
-
-		if (ShowCidStoreStats)
-		{
-			Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
-			const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
-			Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
-
-			Cbo.BeginObject("store");
-
-			auto OutputStats = [&](CidStore& ChunkStore) {
-				CidStoreStats StoreStats = ChunkStore.Stats();
-				Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
-				const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
-				Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
-				EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
-				EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
-			};
-
-			if (UniqueStores.size() > 1)
-			{
-				Cbo.BeginArray("projects");
-				for (auto It : UniqueStores)
-				{
-					CidStore*		   ChunkStore	  = It.first;
-					const std::string& ProjectId	  = It.second;
-					CidStoreSize	   ChunkStoreSize = ChunkStore->TotalSize();
-
-					Cbo.BeginObject();
-					{
-						Cbo << "project" << ProjectId;
-						Cbo.BeginObject("stats");
-						OutputStats(*ChunkStore);
-						Cbo.EndObject();
-
-						Cbo.BeginObject("size");
-						{
-							Cbo << "tiny" << ChunkStoreSize.TinySize;
-							Cbo << "small" << ChunkStoreSize.SmallSize;
-							Cbo << "large" << ChunkStoreSize.LargeSize;
-							Cbo << "total" << ChunkStoreSize.TotalSize;
-						}
-						Cbo.EndObject();
-					}
-					Cbo.EndObject();
-				}
-				Cbo.EndArray();	 // projects
-			}
-			else if (UniqueStores.size() != 0)
-			{
-				CidStore& ChunkStore = *UniqueStores.begin()->first;
-				OutputStats(ChunkStore);
-			}
-			Cbo.EndObject();
-		}
 	}
 	Cbo.EndObject();
 
@@ -1214,8 +1125,6 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchOplog(OplogId);
 
-	CidStore& ChunkStore = Project->GetCidStore();
-
 	ProjectStore::Oplog& Oplog = *FoundLog;
 
 	IoBuffer Payload = HttpReq.ReadPayload();
@@ -1228,7 +1137,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	std::vector<IoHash> MissingChunks;
 
 	CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
-		if (ChunkStore.ContainsChunk(Hash))
+		if (m_CidStore.ContainsChunk(Hash))
 		{
 			// Return null attachment as we already have it, no point in reading it and storing it again
 			return {};
@@ -1484,8 +1393,6 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchOplog(OplogId);
 
-	CidStore& ChunkStore = Project->GetCidStore();
-
 	ProjectStore::Oplog& Oplog = *FoundLog;
 
 	if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
@@ -1500,7 +1407,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 
 				Op.IterateAttachments([&](CbFieldView FieldView) {
 					const IoHash AttachmentHash = FieldView.AsAttachment();
-					IoBuffer	 Payload		= ChunkStore.FindChunkByCid(AttachmentHash);
+					IoBuffer	 Payload		= m_CidStore.FindChunkByCid(AttachmentHash);
 					if (Payload)
 					{
 						switch (Payload.GetContentType())
@@ -2129,14 +2036,11 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
-			CidStore& ChunkStore = Project.GetCidStore();
-
 			Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
-				Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t	 LSN,
-																														const Oid&	 Key,
-																														CbObjectView Op) {
-					CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
-				});
+				Oplog.IterateOplogWithKey(
+					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+					});
 			});
 		});
 
@@ -2150,9 +2054,8 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
 			m_ProjectStore->DiscoverProjects();
 
 			m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
-				CidStore&				 ChunkStore = Project.GetCidStore();
-				std::vector<std::string> OpLogs		= Project.ScanForOplogs();
-				CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+				std::vector<std::string> OpLogs = Project.ScanForOplogs();
+				CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
 			});
 		}
 		Cbo.EndArray();
@@ -2181,8 +2084,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
-	ProjectStore::Project& Project	  = *FoundProject.Get();
-	CidStore&			   ChunkStore = Project.GetCidStore();
+	ProjectStore::Project& Project = *FoundProject.Get();
 
 	if (CSV)
 	{
@@ -2190,11 +2092,10 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
-			Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t	 LSN,
-																													const Oid&	 Key,
-																													CbObjectView Op) {
-				CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
-			});
+			Oplog.IterateOplogWithKey(
+				[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+					CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+				});
 		});
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
@@ -2204,7 +2105,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 		std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
 		Cbo.BeginArray("projects");
 		{
-			CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+			CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2240,17 +2141,16 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Project& Project	  = *FoundProject.Get();
-	CidStore&			   ChunkStore = Project.GetCidStore();
-	ProjectStore::Oplog&   Oplog	  = *FoundLog;
+	ProjectStore::Project& Project = *FoundProject.Get();
+	ProjectStore::Oplog&   Oplog   = *FoundLog;
 	if (CSV)
 	{
 		ExtendableStringBuilder<4096> CSVWriter;
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		Oplog.IterateOplogWithKey(
-			[this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
-				CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+			[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+				CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
 			});
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
@@ -2259,7 +2159,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		CbObjectWriter Cbo;
 		Cbo.BeginArray("oplogs");
 		{
-			CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+			CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2304,10 +2204,9 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 									 fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId));
 	}
 
-	const Oid			   ObjId	  = Oid::FromHexString(OpId);
-	ProjectStore::Project& Project	  = *FoundProject.Get();
-	CidStore&			   ChunkStore = Project.GetCidStore();
-	ProjectStore::Oplog&   Oplog	  = *FoundLog;
+	const Oid			   ObjId   = Oid::FromHexString(OpId);
+	ProjectStore::Project& Project = *FoundProject.Get();
+	ProjectStore::Oplog&   Oplog   = *FoundLog;
 
 	std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId);
 	if (!Op.has_value())
@@ -2325,7 +2224,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		ExtendableStringBuilder<4096> CSVWriter;
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
-		CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
+		CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
 	else
@@ -2333,7 +2232,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		CbObjectWriter Cbo;
 		Cbo.BeginArray("ops");
 		{
-			CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
+			CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());