oplog and cache stats (#244)

* basic oplog stats
* add GetValueStats to cache store
* RwLock::ExclusiveLockScope -> RwLock::SharedLockScope
* add rawhash and attachment count to CacheValueStats
* added cache-stats and project-stats commands
* add cast to make Mac overload detection happy
* fix accept type in cache-stats command
* Add options to project-stats command
* use resource paths for stats in project store
* use resource paths for stats in cache
* fix cache-info and project-info url discriminator
* more control over details$ output
* cleanup
* changelog

1 files changed, 520 insertions, 1 deletions

diff --git a/zenserver/projectstore/projectstore.cpp b/zenserver/projectstore/projectstore.cpp
index 5aa8cad26..db5cae503 100644
--- a/zenserver/projectstore/projectstore.cpp
+++ b/zenserver/projectstore/projectstore.cpp
@@ -201,6 +201,204 @@ namespace {
 									  : fmt::format("{}. Reason: '{}'", Result.Text, Result.Reason)};
 	}
 
+	void CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter)
+	{
+		if (AttachmentDetails)
+		{
+			CSVWriter << "Project, Oplog, LSN, Key, Cid, Size";
+		}
+		else if (Details)
+		{
+			CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize";
+		}
+		else
+		{
+			CSVWriter << "Project, Oplog, Key";
+		}
+	}
+
+	void CSVWriteOp(CidStore&		   CidStore,
+					std::string_view   ProjectId,
+					std::string_view   OplogId,
+					bool			   Details,
+					bool			   AttachmentDetails,
+					int				   LSN,
+					const Oid&		   Key,
+					CbObject		   Op,
+					StringBuilderBase& CSVWriter)
+	{
+		StringBuilder<32> KeyStringBuilder;
+		Key.ToString(KeyStringBuilder);
+		const std::string_view KeyString = KeyStringBuilder.ToView();
+
+		SharedBuffer Buffer = Op.GetBuffer();
+		if (AttachmentDetails)
+		{
+			Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) {
+				const IoHash AttachmentHash = FieldView.AsAttachment();
+				IoBuffer	 Attachment		= CidStore.FindChunkByCid(AttachmentHash);
+				CSVWriter << "\r\n"
+						  << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << AttachmentHash.ToHexString()
+						  << ", " << gsl::narrow<uint64_t>(Attachment.GetSize());
+			});
+		}
+		else if (Details)
+		{
+			uint64_t AttachmentCount = 0;
+			size_t	 AttachmentsSize = 0;
+			Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
+				const IoHash AttachmentHash = FieldView.AsAttachment();
+				AttachmentCount++;
+				IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+				AttachmentsSize += Attachment.GetSize();
+			});
+			CSVWriter << "\r\n"
+					  << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Buffer.GetSize())
+					  << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
+		}
+		else
+		{
+			CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString;
+		}
+	};
+
+	void CbWriteOp(CidStore&	   CidStore,
+				   bool			   Details,
+				   bool			   OpDetails,
+				   bool			   AttachmentDetails,
+				   int			   LSN,
+				   const Oid&	   Key,
+				   CbObject		   Op,
+				   CbObjectWriter& CbWriter)
+	{
+		CbWriter.BeginObject();
+		{
+			SharedBuffer Buffer = Op.GetBuffer();
+			CbWriter.AddObjectId("key", Key);
+			if (Details)
+			{
+				CbWriter.AddInteger("lsn", LSN);
+				CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Buffer.GetSize()));
+			}
+			if (AttachmentDetails)
+			{
+				CbWriter.BeginArray("attachments");
+				Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) {
+					const IoHash AttachmentHash = FieldView.AsAttachment();
+					CbWriter.BeginObject();
+					{
+						IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+						CbWriter.AddString("cid", AttachmentHash.ToHexString());
+						CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Attachment.GetSize()));
+					}
+					CbWriter.EndObject();
+				});
+				CbWriter.EndArray();
+			}
+			else if (Details)
+			{
+				uint64_t AttachmentCount = 0;
+				size_t	 AttachmentsSize = 0;
+				Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
+					const IoHash AttachmentHash = FieldView.AsAttachment();
+					AttachmentCount++;
+					IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+					AttachmentsSize += Attachment.GetSize();
+				});
+				if (AttachmentCount > 0)
+				{
+					CbWriter.AddInteger("attachments", AttachmentCount);
+					CbWriter.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
+				}
+			}
+			if (OpDetails)
+			{
+				CbWriter.BeginObject("op");
+				for (const CbFieldView& Field : Op)
+				{
+					if (!Field.HasName())
+					{
+						CbWriter.AddField(Field);
+						continue;
+					}
+					std::string_view FieldName = Field.GetName();
+					CbWriter.AddField(FieldName, Field);
+				}
+				CbWriter.EndObject();
+			}
+		}
+		CbWriter.EndObject();
+	};
+
+	void CbWriteOplogOps(CidStore&			  CidStore,
+						 ProjectStore::Oplog& Oplog,
+						 bool				  Details,
+						 bool				  OpDetails,
+						 bool				  AttachmentDetails,
+						 CbObjectWriter&	  Cbo)
+	{
+		Cbo.BeginArray("ops");
+		{
+			Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+				CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
+			});
+		}
+		Cbo.EndArray();
+	}
+
+	void CbWriteOplog(CidStore&			   CidStore,
+					  ProjectStore::Oplog& Oplog,
+					  bool				   Details,
+					  bool				   OpDetails,
+					  bool				   AttachmentDetails,
+					  CbObjectWriter&	   Cbo)
+	{
+		Cbo.BeginObject();
+		{
+			Cbo.AddString("name", Oplog.OplogId());
+			CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+		}
+		Cbo.EndObject();
+	}
+
+	void CbWriteOplogs(CidStore&				CidStore,
+					   ProjectStore::Project&	Project,
+					   std::vector<std::string> OpLogs,
+					   bool						Details,
+					   bool						OpDetails,
+					   bool						AttachmentDetails,
+					   CbObjectWriter&			Cbo)
+	{
+		Cbo.BeginArray("oplogs");
+		{
+			for (const std::string& OpLogId : OpLogs)
+			{
+				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId);
+				if (Oplog != nullptr)
+				{
+					CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+				}
+			}
+		}
+		Cbo.EndArray();
+	}
+
+	void CbWriteProject(CidStore&				 CidStore,
+						ProjectStore::Project&	 Project,
+						std::vector<std::string> OpLogs,
+						bool					 Details,
+						bool					 OpDetails,
+						bool					 AttachmentDetails,
+						CbObjectWriter&			 Cbo)
+	{
+		Cbo.BeginObject();
+		{
+			Cbo.AddString("name", Project.Identifier);
+			CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+		}
+		Cbo.EndObject();
+	}
+
 }  // namespace
 
 //////////////////////////////////////////////////////////////////////////
@@ -681,6 +879,69 @@ ProjectStore::Oplog::IterateOplog(std::function<void(CbObject)>&& Handler)
 	m_Storage->ReplayLog(Entries, [&](CbObject Op) { Handler(Op); });
 }
 
+void
+ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbObject)>&& Handler)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+	if (!m_Storage)
+	{
+		return;
+	}
+
+	std::vector<size_t>			   EntryIndexes;
+	std::vector<OplogEntryAddress> Entries;
+	std::vector<Oid>			   Keys;
+	std::vector<int>			   LSNs;
+	Entries.reserve(m_LatestOpMap.size());
+	EntryIndexes.reserve(m_LatestOpMap.size());
+	Keys.reserve(m_LatestOpMap.size());
+	LSNs.reserve(m_LatestOpMap.size());
+
+	for (const auto& Kv : m_LatestOpMap)
+	{
+		const auto AddressEntry = m_OpAddressMap.find(Kv.second);
+		ZEN_ASSERT(AddressEntry != m_OpAddressMap.end());
+
+		Entries.push_back(AddressEntry->second);
+		Keys.push_back(Kv.first);
+		LSNs.push_back(Kv.second);
+		EntryIndexes.push_back(EntryIndexes.size());
+	}
+
+	std::sort(EntryIndexes.begin(), EntryIndexes.end(), [&Entries](const size_t& Lhs, const size_t& Rhs) {
+		const OplogEntryAddress& LhsEntry = Entries[Lhs];
+		const OplogEntryAddress& RhsEntry = Entries[Rhs];
+		return LhsEntry.Offset < RhsEntry.Offset;
+	});
+	std::vector<OplogEntryAddress> SortedEntries;
+	SortedEntries.reserve(EntryIndexes.size());
+	for (size_t Index : EntryIndexes)
+	{
+		SortedEntries.push_back(Entries[Index]);
+	}
+
+	size_t EntryIndex = 0;
+	m_Storage->ReplayLog(SortedEntries, [&](CbObject Op) {
+		Handler(LSNs[EntryIndex], Keys[EntryIndex], Op);
+		EntryIndex++;
+	});
+}
+
+int
+ProjectStore::Oplog::GetOpIndexByKey(const Oid& Key)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+	if (!m_Storage)
+	{
+		return {};
+	}
+	if (const auto LatestOp = m_LatestOpMap.find(Key); LatestOp != m_LatestOpMap.end())
+	{
+		return LatestOp->second;
+	}
+	return -1;
+}
+
 std::optional<CbObject>
 ProjectStore::Oplog::GetOpByKey(const Oid& Key)
 {
@@ -2240,14 +2501,17 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, AuthMgr& AuthMgr)
+HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
 : m_Log(logging::Get("project"))
 , m_CidStore(Store)
 , m_ProjectStore(Projects)
+, m_StatsService(StatsService)
 , m_AuthMgr(AuthMgr)
 {
 	using namespace std::literals;
 
+	m_StatsService.RegisterHandler("prj", *this);
+
 	m_Router.AddPattern("project", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("log", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("op", "([[:digit:]]+?)");
@@ -3174,10 +3438,231 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
 		},
 		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"details\\$",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+					Project.IterateOplogs([&](ProjectStore::Oplog& Oplog) {
+						Oplog.IterateOplogWithKey(
+							[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+								CSVWriteOp(m_CidStore,
+										   Project.Identifier,
+										   Oplog.OplogId(),
+										   Details,
+										   AttachmentDetails,
+										   LSN,
+										   Key,
+										   Op,
+										   CSVWriter);
+							});
+					});
+				});
+
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter Cbo;
+				Cbo.BeginArray("projects");
+				{
+					m_ProjectStore->DiscoverProjects();
+
+					m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+						std::vector<std::string> OpLogs = Project.ScanForOplogs();
+						CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+					});
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"details\\$/{project}",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq	 = Req.ServerRequest();
+			const auto&		   ProjectId = Req.GetCapture(1);
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+			if (!FoundProject)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			ProjectStore::Project& Project = *FoundProject.Get();
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				FoundProject->IterateOplogs([&](ProjectStore::Oplog& Oplog) {
+					Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int		  LSN,
+																											   const Oid& Key,
+																											   CbObject	  Op) {
+						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+					});
+				});
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter			 Cbo;
+				std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
+				Cbo.BeginArray("projects");
+				{
+					CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"details\\$/{project}/{log}",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq	 = Req.ServerRequest();
+			const auto&		   ProjectId = Req.GetCapture(1);
+			const auto&		   OplogId	 = Req.GetCapture(2);
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+			if (!FoundProject)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+
+			if (!FoundLog)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Project& Project = *FoundProject.Get();
+			ProjectStore::Oplog&   Oplog   = *FoundLog;
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				Oplog.IterateOplogWithKey(
+					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+					});
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter Cbo;
+				Cbo.BeginArray("oplogs");
+				{
+					CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"details\\$/{project}/{log}/{chunk}",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq	 = Req.ServerRequest();
+			const auto&		   ProjectId = Req.GetCapture(1);
+			const auto&		   OplogId	 = Req.GetCapture(2);
+			const auto&		   ChunkId	 = Req.GetCapture(3);
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+			if (!FoundProject)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+
+			if (!FoundLog)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
+			{
+				return HttpReq.WriteResponse(
+					HttpResponseCode::BadRequest,
+					HttpContentType::kText,
+					fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId));
+			}
+
+			const Oid			   ObjId   = Oid::FromHexString(ChunkId);
+			ProjectStore::Project& Project = *FoundProject.Get();
+			ProjectStore::Oplog&   Oplog   = *FoundLog;
+
+			int LSN = Oplog.GetOpIndexByKey(ObjId);
+			if (LSN == -1)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			std::optional<CbObject> Op = Oplog.GetOpByIndex(LSN);
+			if (!Op.has_value())
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter);
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter Cbo;
+				Cbo.BeginArray("ops");
+				{
+					CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo);
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
 }
 
 HttpProjectService::~HttpProjectService()
 {
+	m_StatsService.UnregisterHandler("prj", *this);
 }
 
 const char*
@@ -3195,6 +3680,40 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
 	}
 }
 
+void
+HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+	const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
+
+	CbObjectWriter Cbo;
+	Cbo.BeginObject("store");
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "disk" << StoreSize.DiskSize;
+			Cbo << "memory" << StoreSize.MemorySize;
+		}
+		Cbo.EndObject();
+	}
+	Cbo.EndObject();
+
+	Cbo.BeginObject("cid");
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "tiny" << CidSize.TinySize;
+			Cbo << "small" << CidSize.SmallSize;
+			Cbo << "large" << CidSize.LargeSize;
+			Cbo << "total" << CidSize.TotalSize;
+		}
+		Cbo.EndObject();
+	}
+	Cbo.EndObject();
+
+	return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_WITH_TESTS