new in-memory storage strategy for oplogs (#490)

- Improvement: Revised project oplog in-memory representation which reduces load times and memory usage

1 files changed, 52 insertions, 48 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 11ef067c4..48086b863 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -46,15 +46,15 @@ CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter)
 }
 
 void
-CSVWriteOp(CidStore&		  CidStore,
-		   std::string_view	  ProjectId,
-		   std::string_view	  OplogId,
-		   bool				  Details,
-		   bool				  AttachmentDetails,
-		   uint32_t			  LSN,
-		   const Oid&		  Key,
-		   CbObjectView		  Op,
-		   StringBuilderBase& CSVWriter)
+CSVWriteOp(CidStore&					   CidStore,
+		   std::string_view				   ProjectId,
+		   std::string_view				   OplogId,
+		   bool							   Details,
+		   bool							   AttachmentDetails,
+		   ProjectStore::LogSequenceNumber LSN,
+		   const Oid&					   Key,
+		   CbObjectView					   Op,
+		   StringBuilderBase&			   CSVWriter)
 {
 	StringBuilder<32> KeyStringBuilder;
 	Key.ToString(KeyStringBuilder);
@@ -66,8 +66,8 @@ CSVWriteOp(CidStore&		  CidStore,
 			const IoHash AttachmentHash = FieldView.AsAttachment();
 			IoBuffer	 Attachment		= CidStore.FindChunkByCid(AttachmentHash);
 			CSVWriter << "\r\n"
-					  << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << AttachmentHash.ToHexString() << ", "
-					  << gsl::narrow<uint64_t>(Attachment.GetSize());
+					  << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << AttachmentHash.ToHexString()
+					  << ", " << gsl::narrow<uint64_t>(Attachment.GetSize());
 		});
 	}
 	else if (Details)
@@ -81,8 +81,8 @@ CSVWriteOp(CidStore&		  CidStore,
 			AttachmentsSize += Attachment.GetSize();
 		});
 		CSVWriter << "\r\n"
-				  << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Op.GetSize()) << ", "
-				  << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
+				  << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Op.GetSize())
+				  << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
 	}
 	else
 	{
@@ -94,21 +94,21 @@ CSVWriteOp(CidStore&		  CidStore,
 
 namespace {
 
-	void CbWriteOp(CidStore&	   CidStore,
-				   bool			   Details,
-				   bool			   OpDetails,
-				   bool			   AttachmentDetails,
-				   uint32_t		   LSN,
-				   const Oid&	   Key,
-				   CbObjectView	   Op,
-				   CbObjectWriter& CbWriter)
+	void CbWriteOp(CidStore&					   CidStore,
+				   bool							   Details,
+				   bool							   OpDetails,
+				   bool							   AttachmentDetails,
+				   ProjectStore::LogSequenceNumber LSN,
+				   const Oid&					   Key,
+				   CbObjectView					   Op,
+				   CbObjectWriter&				   CbWriter)
 	{
 		CbWriter.BeginObject();
 		{
 			CbWriter.AddObjectId("key", Key);
 			if (Details)
 			{
-				CbWriter.AddInteger("lsn", LSN);
+				CbWriter.AddInteger("lsn", LSN.Number);
 				CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Op.GetSize()));
 			}
 			if (AttachmentDetails)
@@ -170,10 +170,11 @@ namespace {
 	{
 		Cbo.BeginArray("ops");
 		{
-			Oplog.IterateOplogWithKey(
-				[&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
-					CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
-				});
+			Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](ProjectStore::LogSequenceNumber LSN,
+																							   const Oid&					   Key,
+																							   CbObjectView					   Op) {
+				CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
+			});
 		}
 		Cbo.EndArray();
 	}
@@ -1240,10 +1241,9 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 
 	// Write core to oplog
 
-	size_t		   AttachmentCount = Package.GetAttachments().size();
-	const uint32_t OpLsn		   = Oplog.AppendNewOplogEntry(Package);
-
-	if (OpLsn == ProjectStore::Oplog::kInvalidOp)
+	size_t								  AttachmentCount = Package.GetAttachments().size();
+	const ProjectStore::LogSequenceNumber OpLsn			  = Oplog.AppendNewOplogEntry(Package);
+	if (!OpLsn)
 	{
 		m_ProjectStats.BadRequestCount++;
 		return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
@@ -1257,7 +1257,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	}
 
 	m_ProjectStats.OpWriteCount++;
-	ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString());
+	ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn.Number, NiceBytes(Payload.Size()), Core["key"sv].AsString());
 	HttpReq.WriteResponse(HttpResponseCode::Created);
 }
 
@@ -1307,8 +1307,8 @@ HttpProjectService::HandleOplogValidateRequest(HttpRouterRequest& Req)
 	CbObjectWriter Writer;
 	Writer << "HasMissingData" << !Result.IsEmpty();
 	Writer << "OpCount" << Result.OpCount;
-	Writer << "LSNLow" << Result.LSNLow;
-	Writer << "LSNHigh" << Result.LSNHigh;
+	Writer << "LSNLow" << Result.LSNLow.Number;
+	Writer << "LSNHigh" << Result.LSNHigh.Number;
 	if (!Result.MissingFiles.empty())
 	{
 		Writer.BeginArray("MissingFiles");
@@ -1407,7 +1407,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 
 	if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
 	{
-		if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value()))
+		if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(ProjectStore::LogSequenceNumber(OpId.value())))
 		{
 			CbObject& Op = MaybeOp.value();
 			if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage)
@@ -1741,7 +1741,7 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 			{
 				const OplogReferencedSet& ReferencedSet = MaybeReferencedSet.value();
 				FoundLog->IterateOplogWithKey(
-					[this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](uint32_t /* LSN */,
+					[this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](ProjectStore::LogSequenceNumber /* LSN */,
 																						 const Oid&	  Key,
 																						 CbObjectView Op) {
 						if (!ReferencedSet.Contains(Key))
@@ -2091,7 +2091,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
 		m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
 			Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
 				Oplog.IterateOplogWithKey(
-					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN,
+																					 const Oid&						 Key,
+																					 CbObjectView					 Op) {
 						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
 					});
 			});
@@ -2145,10 +2147,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
-			Oplog.IterateOplogWithKey(
-				[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
-					CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
-				});
+			Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN,
+																									   const Oid&					   Key,
+																									   CbObjectView					   Op) {
+				CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+			});
 		});
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
@@ -2201,10 +2204,11 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		ExtendableStringBuilder<4096> CSVWriter;
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
-		Oplog.IterateOplogWithKey(
-			[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
-				CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
-			});
+		Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN,
+																								   const Oid&					   Key,
+																								   CbObjectView					   Op) {
+			CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+		});
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
 	else
@@ -2266,8 +2270,8 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
-	std::optional<uint32_t> LSN = Oplog.GetOpIndexByKey(ObjId);
-	if (!LSN.has_value())
+	ProjectStore::LogSequenceNumber LSN = Oplog.GetOpIndexByKey(ObjId);
+	if (!LSN)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
@@ -2277,7 +2281,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		ExtendableStringBuilder<4096> CSVWriter;
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
-		CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
+		CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter);
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
 	else
@@ -2285,7 +2289,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		CbObjectWriter Cbo;
 		Cbo.BeginArray("ops");
 		{
-			CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
+			CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());