add compacting of oplogs as part of GC (#106)

* add compacting of oplogs as part of GC
* force retain of LSN unless we have less than 16 miln entries left

1 files changed, 12 insertions, 12 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 6de91cf94..28506145b 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -210,7 +210,7 @@ namespace {
 		{
 			for (const std::string& OpLogId : OpLogs)
 			{
-				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId);
+				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false);
 				if (Oplog != nullptr)
 				{
 					CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
@@ -487,7 +487,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -964,7 +964,7 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1042,7 +1042,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1184,7 +1184,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1281,13 +1281,14 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 	{
 		case HttpVerb::kGet:
 			{
-				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
+				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
 				if (!OplogIt)
 				{
 					return HttpReq.WriteResponse(HttpResponseCode::NotFound,
 												 HttpContentType::kText,
 												 fmt::format("oplog {} not found in project {}", OplogId, ProjectId));
 				}
+
 				Project->TouchOplog(OplogId);
 
 				ProjectStore::Oplog& Log = *OplogIt;
@@ -1296,7 +1297,6 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 				Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str()
 				   << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount" << Log.OplogCount()
 				   << "expired"sv << Project->IsExpired(GcClock::TimePoint::min(), Log);
-
 				HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save());
 
 				m_ProjectStats.OpLogReadCount++;
@@ -1315,7 +1315,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 					OplogMarkerPath = Params["gcpath"sv].AsString();
 				}
 
-				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
+				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false);
 				if (!OplogIt)
 				{
 					if (!Project->NewOplog(OplogId, OplogMarkerPath))
@@ -1352,7 +1352,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 					OplogMarkerPath = Params["gcpath"sv].AsString();
 				}
 
-				ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+				ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false);
 				if (!FoundLog)
 				{
 					if (!Project->NewOplog(OplogId, OplogMarkerPath))
@@ -1420,7 +1420,7 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1880,7 +1880,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1935,7 +1935,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);