Add ability to read the oplog's ReferencedSet, as written by the cook… (#190)v5.5.9-pre7

Add ability to read the oplog's ReferencedSet, as written by the cooker, from the ReferencedSet op. Filter oplog entries requests by the ReferencedSet, if trim_by_referencedset parameter is present.. Add -trim=true/false parameter to oplog-mirror command, default to true, to request the trimmed/not trimmed oplog.
Helper functions: Add paging to IterateOpLogWithKey. Add unit tests for IterateOpLog functions. Move OpKeyStringAsOid from httpprojectstore into projectstore.

1 files changed, 48 insertions, 27 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index cffd2569f..5f4f44e31 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -2,6 +2,7 @@
 
 #include "httpprojectstore.h"
 
+#include "oplogreferencedset.h"
 #include "projectstore.h"
 
 #include <zencore/compactbinarybuilder.h>
@@ -12,27 +13,10 @@
 #include <zencore/logging.h>
 #include <zencore/stream.h>
 #include <zencore/trace.h>
+#include <zenstore/zenstore.h>
 
 namespace zen {
 
-Oid
-OpKeyStringAsOId(std::string_view OpKey)
-{
-	using namespace std::literals;
-
-	CbObjectWriter Writer;
-	Writer << "key"sv << OpKey;
-
-	XXH3_128Stream KeyHasher;
-	Writer.Save()["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
-	XXH3_128 KeyHash = KeyHasher.GetHash();
-
-	Oid OpId;
-	memcpy(OpId.OidBits, &KeyHash, sizeof(OpId.OidBits));
-
-	return OpId;
-}
-
 void
 CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter)
 {
@@ -1401,6 +1385,28 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 	}
 }
 
+std::optional<OplogReferencedSet>
+LoadReferencedSet(ProjectStore::Oplog& Log)
+{
+	using namespace std::literals;
+
+	Oid						ReferencedSetOplogId = OpKeyStringAsOid(OplogReferencedSet::ReferencedSetOplogKey);
+	std::optional<CbObject> ReferencedSetOp		 = Log.GetOpByKey(ReferencedSetOplogId);
+	if (!ReferencedSetOp)
+	{
+		return std::optional<OplogReferencedSet>();
+	}
+	// We expect only a single file in the "files" array; get the chunk for the first file
+	CbFieldView FileField = *(*ReferencedSetOp)["files"sv].AsArrayView().CreateViewIterator();
+	Oid			ChunkId	  = FileField.AsObjectView()["id"sv].AsObjectId();
+	if (ChunkId == Oid::Zero)
+	{
+		return std::optional<OplogReferencedSet>();
+	}
+
+	return OplogReferencedSet::LoadFromChunk(Log.FindChunk(ChunkId));
+}
+
 void
 HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 {
@@ -1456,7 +1462,7 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 
 		if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty())
 		{
-			Oid						OpKeyId = OpKeyStringAsOId(OpKey);
+			Oid						OpKeyId = OpKeyStringAsOid(OpKey);
 			std::optional<CbObject> Op		= FoundLog->GetOpByKey(OpKeyId);
 
 			if (Op.has_value())
@@ -1477,6 +1483,11 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 		}
 		else
 		{
+			std::optional<OplogReferencedSet> ReferencedSet;
+			if (auto TrimString = Params.GetValue("trim_by_referencedset"); TrimString == "true")
+			{
+				ReferencedSet = LoadReferencedSet(*FoundLog);
+			}
 			Response.BeginArray("entries"sv);
 
 			ProjectStore::Oplog::Paging EntryPaging;
@@ -1495,14 +1506,24 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 				}
 			}
 
-			if (FieldNamesFilter.empty())
-			{
-				FoundLog->IterateOplog([&Response](CbObjectView Op) { Response << Op; }, EntryPaging);
-			}
-			else
-			{
-				FoundLog->IterateOplog([this, &Response, &FilterObject](CbObjectView Op) { Response << FilterObject(Op); }, EntryPaging);
-			}
+			bool ShouldFilterFields = !FieldNamesFilter.empty();
+			FoundLog->IterateOplogWithKey(
+				[this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](uint32_t /* LSN */, const Oid& Key, CbObjectView Op) {
+					if (ReferencedSet && !ReferencedSet->Contains(Key, Op["key"].AsString()))
+					{
+						return;
+					}
+
+					if (ShouldFilterFields)
+					{
+						Response << FilterObject(Op);
+					}
+					else
+					{
+						Response << Op;
+					}
+				},
+				EntryPaging);
 
 			Response.EndArray();
 		}