Add ability to read the oplog's ReferencedSet, as written by the cook… (#190)v5.5.9-pre7

Add ability to read the oplog's ReferencedSet, as written by the cooker, from the ReferencedSet op. Filter oplog entries requests by the ReferencedSet, if trim_by_referencedset parameter is present.. Add -trim=true/false parameter to oplog-mirror command, default to true, to request the trimmed/not trimmed oplog.
Helper functions: Add paging to IterateOpLogWithKey. Add unit tests for IterateOpLog functions. Move OpKeyStringAsOid from httpprojectstore into projectstore.

1 files changed, 110 insertions, 0 deletions

diff --git a/src/zenserver/projectstore/oplogreferencedset.cpp b/src/zenserver/projectstore/oplogreferencedset.cpp
new file mode 100644
index 000000000..c6bfa0b98
--- /dev/null
+++ b/src/zenserver/projectstore/oplogreferencedset.cpp
@@ -0,0 +1,110 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "oplogreferencedset.h"
+
+#include "projectstore.h"
+
+#include <zencore/iobuffer.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+
+namespace zen {
+
+std::optional<OplogReferencedSet>
+OplogReferencedSet::LoadFromChunk(const IoBuffer& ChunkData)
+{
+	using namespace std::literals;
+
+	if (!ChunkData || ChunkData.Size() == 0)
+	{
+		return std::optional<OplogReferencedSet>();
+	}
+	std::string_view ChunkText(reinterpret_cast<const char*>(ChunkData.Data()), ChunkData.Size());
+
+	AsciiSet	TrimWhitespace(" \t\r\n");
+	const char* FirstNonComment = nullptr;
+	uint64_t	Version			= 0;
+
+	// Parse the initial comment block: the leading block of empty or comment lines.
+	ForEachStrTok(ChunkText, '\n', [&Version, &FirstNonComment, &TrimWhitespace](std::string_view Line) {
+		Line = AsciiSet::TrimSuffixWith(AsciiSet::TrimPrefixWith(Line, TrimWhitespace), TrimWhitespace);
+		if (Line.empty())
+		{
+			return true;  // empty line, keep reading
+		}
+		if (!Line.starts_with('#'))
+		{
+			FirstNonComment = Line.data();
+			return false;  // non-comment line, stop
+		}
+
+		// Comment line in the header block of comments in the file. Interpret it if it is a version line.
+		// Skip past the '#' and whitespace.
+		Line.remove_prefix(1);
+		Line = AsciiSet::TrimPrefixWith(Line, TrimWhitespace);
+
+		// Parse "# Version <uint64>".
+		constexpr std::string_view VersionStr("Version "sv);
+		if (Line.starts_with(VersionStr))
+		{
+			std::string_view VersionToken = Line;
+			VersionToken.remove_prefix(VersionStr.length());
+			VersionToken.remove_suffix(AsciiSet::TrimPrefixWithout(VersionToken, TrimWhitespace).length());
+			std::optional<uint64_t> ParsedVersion = ParseInt<uint64_t>(VersionToken);
+			if (ParsedVersion)
+			{
+				Version = *ParsedVersion;
+			}
+		}
+		return true;
+	});
+
+	// Report no referencedset if the version is below our minimum version.
+	constexpr uint64_t MinSupportedVersion = 1;
+	if (Version < MinSupportedVersion)
+	{
+		ZEN_INFO("ReferencedSet is below the minimum supported version, ignoring it. Version: {}, minimum version: {}.",
+				 Version,
+				 MinSupportedVersion);
+		return std::optional<OplogReferencedSet>();
+	}
+
+	// Parse the remaining lines after the leading comment block.
+	ChunkText.remove_prefix(FirstNonComment ? FirstNonComment - ChunkText.data() : ChunkText.length());
+
+	OplogReferencedSet Result;
+	ForEachStrTok(ChunkText, '\n', [&Result, &TrimWhitespace](std::string_view Line) {
+		Line = AsciiSet::TrimSuffixWith(AsciiSet::TrimPrefixWith(Line, TrimWhitespace), TrimWhitespace);
+		if (!Line.empty() && !Line.starts_with('#'))
+		{
+			Result.Emplace(OpKeyStringAsOid(Line));
+		}
+		return true;
+	});
+	return std::optional<OplogReferencedSet>(std::move(Result));
+}
+
+void
+OplogReferencedSet::Emplace(Oid OplogId)
+{
+	Set.emplace(OplogId);
+}
+
+void
+OplogReferencedSet::Clear()
+{
+	Set.clear();
+}
+
+bool
+OplogReferencedSet::Contains(Oid OplogId, std::string_view OplogKey)
+{
+	// A referencedset always includes all non-package keys
+	if (OplogKey.empty() || !OplogKey.starts_with("/"))
+	{
+		return true;
+	}
+	return Set.contains(OplogId);
+}
+
+}  // namespace zen