oplog prep gc fix (#216)

- Added option gc-validation to zenserver that does a check for missing references in all oplog post full GC. Enabled by default.
- Feature: Added option gc-validation to zen gc command to control reference validation. Enabled by default.
- Added more details in post GC log.
- Fixed race condition in oplog writes which could cause used attachments to be incorrectly removed by GC

1 files changed, 141 insertions, 16 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 1b45e66f3..710cbe5a7 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -292,6 +292,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		HttpVerb::kPost);
 
 	m_Router.RegisterRoute(
+		"{project}/oplog/{log}/validate",
+		[this](HttpRouterRequest& Req) { HandleOplogValidateRequest(Req); },
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/{op}",
 		[this](HttpRouterRequest& Req) { HandleOpLogOpRequest(Req); },
 		HttpVerb::kGet);
@@ -962,28 +967,25 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
 	IoBuffer Payload	   = HttpReq.ReadPayload();
 	CbObject RequestObject = LoadCompactBinaryObject(Payload);
 
-	std::vector<IoHash> NeedList;
-
-	for (auto Entry : RequestObject["have"sv])
+	std::vector<IoHash> ChunkList;
+	CbArrayView			HaveList = RequestObject["have"sv].AsArrayView();
+	ChunkList.reserve(HaveList.Num());
+	for (auto& Entry : HaveList)
 	{
-		const IoHash FileHash = Entry.AsHash();
-
-		if (!m_CidStore.ContainsChunk(FileHash))
-		{
-			ZEN_DEBUG("prep - NEED: {}", FileHash);
-
-			NeedList.push_back(FileHash);
-		}
+		ChunkList.push_back(Entry.AsHash());
 	}
 
+	std::vector<IoHash> NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2));
+
 	CbObjectWriter Cbo;
 	Cbo.BeginArray("need");
-
-	for (const IoHash& Hash : NeedList)
 	{
-		Cbo << Hash;
+		for (const IoHash& Hash : NeedList)
+		{
+			ZEN_DEBUG("prep - NEED: {}", Hash);
+			Cbo << Hash;
+		}
 	}
-
 	Cbo.EndArray();
 	CbObject Response = Cbo.Save();
 
@@ -1043,9 +1045,12 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 
 	bool				IsValid = true;
 	std::vector<IoHash> MissingChunks;
+	std::vector<IoHash> ReferencedChunks;
 
 	CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
-		Oplog.CaptureAddedAttachments(std::vector<IoHash>{Hash});
+		// We want to add all chunks here so we can properly clear them from the 'prep' call where we retained them earlier
+		ReferencedChunks.push_back(Hash);
+
 		if (m_CidStore.ContainsChunk(Hash))
 		{
 			// Return null attachment as we already have it, no point in reading it and storing it again
@@ -1146,12 +1151,132 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	}
 	m_ProjectStats.ChunkWriteCount += AttachmentCount;
 
+	// Once we stored the op, we no longer need to retain any chunks this op references
+	FoundLog->RemovePendingChunkReferences(ReferencedChunks);
+
 	m_ProjectStats.OpWriteCount++;
 	ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString());
 	HttpReq.WriteResponse(HttpResponseCode::Created);
 }
 
 void
+HttpProjectService::HandleOplogValidateRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("ProjectService::OplogOpNew");
+
+	using namespace std::literals;
+
+	HttpServerRequest& HttpReq = Req.ServerRequest();
+
+	if (!m_ProjectStore->AreDiskWritesAllowed())
+	{
+		return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+	}
+
+	const auto& ProjectId = Req.GetCapture(1);
+	const auto& OplogId	  = Req.GetCapture(2);
+
+	Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+	if (!Project)
+	{
+		return HttpReq.WriteResponse(HttpResponseCode::NotFound, ZenContentType::kText, fmt::format("Project '{}' not found", ProjectId));
+	}
+	Project->TouchProject();
+
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
+	if (!FoundLog)
+	{
+		return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+									 ZenContentType::kText,
+									 fmt::format("Oplog '{}' not found in project '{}'", OplogId, ProjectId));
+	}
+	Project->TouchOplog(OplogId);
+
+	ProjectStore::Oplog& Oplog = *FoundLog;
+
+	std::atomic_bool							  CancelFlag = false;
+	ProjectStore::Oplog::ValidationResult		  Result	 = Oplog.Validate(CancelFlag);
+	tsl::robin_map<Oid, std::string, Oid::Hasher> KeyNameLookup;
+	KeyNameLookup.reserve(Result.OpKeys.size());
+	for (const auto& It : Result.OpKeys)
+	{
+		KeyNameLookup.insert_or_assign(It.first, It.second);
+	}
+	CbObjectWriter Writer;
+	Writer << "HasMissingData" << !Result.IsEmpty();
+	Writer << "OpCount" << Result.OpCount;
+	Writer << "LSNLow" << Result.LSNLow;
+	Writer << "LSNHigh" << Result.LSNHigh;
+	if (!Result.MissingFiles.empty())
+	{
+		Writer.BeginArray("MissingFiles");
+		for (const auto& MissingFile : Result.MissingFiles)
+		{
+			Writer.BeginObject();
+			{
+				Writer << "Key" << MissingFile.first;
+				Writer << "KeyName" << KeyNameLookup[MissingFile.first];
+				Writer << "Id" << MissingFile.second.Id;
+				Writer << "Hash" << MissingFile.second.Hash;
+				Writer << "ServerPath" << MissingFile.second.ServerPath;
+				Writer << "ClientPath" << MissingFile.second.ClientPath;
+			}
+			Writer.EndObject();
+		}
+		Writer.EndArray();
+	}
+	if (!Result.MissingChunks.empty())
+	{
+		Writer.BeginArray("MissingChunks");
+		for (const auto& MissingChunk : Result.MissingChunks)
+		{
+			Writer.BeginObject();
+			{
+				Writer << "Key" << MissingChunk.first;
+				Writer << "KeyName" << KeyNameLookup[MissingChunk.first];
+				Writer << "Id" << MissingChunk.second.Id;
+				Writer << "Hash" << MissingChunk.second.Hash;
+			}
+			Writer.EndObject();
+		}
+		Writer.EndArray();
+	}
+	if (!Result.MissingMetas.empty())
+	{
+		Writer.BeginArray("MissingMetas");
+		for (const auto& MissingMeta : Result.MissingMetas)
+		{
+			Writer.BeginObject();
+			{
+				Writer << "Key" << MissingMeta.first;
+				Writer << "KeyName" << KeyNameLookup[MissingMeta.first];
+				Writer << "Id" << MissingMeta.second.Id;
+				Writer << "Hash" << MissingMeta.second.Hash;
+			}
+			Writer.EndObject();
+		}
+		Writer.EndArray();
+	}
+	if (!Result.MissingAttachments.empty())
+	{
+		Writer.BeginArray("MissingAttachments");
+		for (const auto& MissingMeta : Result.MissingAttachments)
+		{
+			Writer.BeginObject();
+			{
+				Writer << "Key" << MissingMeta.first;
+				Writer << "KeyName" << KeyNameLookup[MissingMeta.first];
+				Writer << "Hash" << MissingMeta.second;
+			}
+			Writer.EndObject();
+		}
+		Writer.EndArray();
+	}
+	CbObject Response = Writer.Save();
+	HttpReq.WriteResponse(HttpResponseCode::OK, Response);
+}
+
+void
 HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 {
 	ZEN_TRACE_CPU("ProjectService::OplogOp");