faster oplog entries with referenceset (#488)

- Improvement: Faster project store `/entries` endpoint, 10-15% faster when using a reference set to limit entries
  fix missing space after task name in pretty progressbar

7 files changed, 88 insertions, 57 deletions

diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 11c915e9a..b7c0e0b7f 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -2090,6 +2090,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 	if (HttpClient::Response Response =
 			Http.Get(fmt::format("/prj/{}/oplog/{}/entries"sv, m_ProjectName, m_OplogName), HttpClient::KeyValueMap(), Parameters))
 	{
+		ZEN_CONSOLE("Fetched oplog in {}", NiceTimeSpanMs(uint64_t(Response.ElapsedSeconds * 1000.0)));
 		if (CbObject ResponseObject = Response.AsObject())
 		{
 			std::unique_ptr<ProgressBar> EmitProgressBar;
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index db63624ce..200c8e4b4 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -519,7 +519,7 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
 
 		ExtendableStringBuilder<256> OutputBuilder;
 
-		OutputBuilder << "\r" << Task << PercentString;
+		OutputBuilder << "\r" << Task << " " << PercentString;
 		if (OutputBuilder.Size() + 1 < ConsoleColumns)
 		{
 			size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1);
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 237dc097e..11ef067c4 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -1712,13 +1712,6 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 		}
 		else
 		{
-			std::optional<OplogReferencedSet> ReferencedSet;
-			if (auto TrimString = Params.GetValue("trim_by_referencedset"); TrimString == "true")
-			{
-				ReferencedSet = LoadReferencedSet(*Project, *FoundLog);
-			}
-			Response.BeginArray("entries"sv);
-
 			ProjectStore::Oplog::Paging EntryPaging;
 			if (std::string_view Param = Params.GetValue("start"); !Param.empty())
 			{
@@ -1735,24 +1728,56 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 				}
 			}
 
+			std::optional<OplogReferencedSet> MaybeReferencedSet;
+			if (auto TrimString = Params.GetValue("trim_by_referencedset"); TrimString == "true")
+			{
+				MaybeReferencedSet = LoadReferencedSet(*Project, *FoundLog);
+			}
+			Response.BeginArray("entries"sv);
+
 			bool ShouldFilterFields = !FieldNamesFilter.empty();
-			FoundLog->IterateOplogWithKey(
-				[this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](uint32_t /* LSN */, const Oid& Key, CbObjectView Op) {
-					if (ReferencedSet && !ReferencedSet->Contains(Key, Op["key"].AsString()))
-					{
-						return;
-					}
 
-					if (ShouldFilterFields)
-					{
-						Response << FilterObject(Op);
-					}
-					else
-					{
-						Response << Op;
-					}
-				},
-				EntryPaging);
+			if (MaybeReferencedSet)
+			{
+				const OplogReferencedSet& ReferencedSet = MaybeReferencedSet.value();
+				FoundLog->IterateOplogWithKey(
+					[this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](uint32_t /* LSN */,
+																						 const Oid&	  Key,
+																						 CbObjectView Op) {
+						if (!ReferencedSet.Contains(Key))
+						{
+							if (!OplogReferencedSet::IsNonPackage(Op["key"].AsString()))
+							{
+								return;
+							}
+						}
+
+						if (ShouldFilterFields)
+						{
+							Response << FilterObject(Op);
+						}
+						else
+						{
+							Response << Op;
+						}
+					},
+					EntryPaging);
+			}
+			else
+			{
+				FoundLog->IterateOplog(
+					[this, &Response, &FilterObject, ShouldFilterFields](CbObjectView Op) {
+						if (ShouldFilterFields)
+						{
+							Response << FilterObject(Op);
+						}
+						else
+						{
+							Response << Op;
+						}
+					},
+					EntryPaging);
+			}
 
 			Response.EndArray();
 		}
diff --git a/src/zenserver/projectstore/oplogreferencedset.cpp b/src/zenserver/projectstore/oplogreferencedset.cpp
index c6bfa0b98..b5cbc6f4b 100644
--- a/src/zenserver/projectstore/oplogreferencedset.cpp
+++ b/src/zenserver/projectstore/oplogreferencedset.cpp
@@ -63,7 +63,7 @@ OplogReferencedSet::LoadFromChunk(const IoBuffer& ChunkData)
 	constexpr uint64_t MinSupportedVersion = 1;
 	if (Version < MinSupportedVersion)
 	{
-		ZEN_INFO("ReferencedSet is below the minimum supported version, ignoring it. Version: {}, minimum version: {}.",
+		ZEN_WARN("ReferencedSet is below the minimum supported version, ignoring it. Version: {}, minimum version: {}.",
 				 Version,
 				 MinSupportedVersion);
 		return std::optional<OplogReferencedSet>();
@@ -72,12 +72,14 @@ OplogReferencedSet::LoadFromChunk(const IoBuffer& ChunkData)
 	// Parse the remaining lines after the leading comment block.
 	ChunkText.remove_prefix(FirstNonComment ? FirstNonComment - ChunkText.data() : ChunkText.length());
 
+	eastl::fixed_vector<uint8_t, 256> TmpBuffer;
+
 	OplogReferencedSet Result;
-	ForEachStrTok(ChunkText, '\n', [&Result, &TrimWhitespace](std::string_view Line) {
+	ForEachStrTok(ChunkText, '\n', [&Result, &TrimWhitespace, &TmpBuffer](std::string_view Line) {
 		Line = AsciiSet::TrimSuffixWith(AsciiSet::TrimPrefixWith(Line, TrimWhitespace), TrimWhitespace);
 		if (!Line.empty() && !Line.starts_with('#'))
 		{
-			Result.Emplace(OpKeyStringAsOid(Line));
+			Result.Set.emplace(OpKeyStringAsOid(Line, TmpBuffer));
 		}
 		return true;
 	});
@@ -85,26 +87,9 @@ OplogReferencedSet::LoadFromChunk(const IoBuffer& ChunkData)
 }
 
 void
-OplogReferencedSet::Emplace(Oid OplogId)
-{
-	Set.emplace(OplogId);
-}
-
-void
 OplogReferencedSet::Clear()
 {
 	Set.clear();
 }
 
-bool
-OplogReferencedSet::Contains(Oid OplogId, std::string_view OplogKey)
-{
-	// A referencedset always includes all non-package keys
-	if (OplogKey.empty() || !OplogKey.starts_with("/"))
-	{
-		return true;
-	}
-	return Set.contains(OplogId);
-}
-
 }  // namespace zen
diff --git a/src/zenserver/projectstore/oplogreferencedset.h b/src/zenserver/projectstore/oplogreferencedset.h
index 297fd29d5..dcc156060 100644
--- a/src/zenserver/projectstore/oplogreferencedset.h
+++ b/src/zenserver/projectstore/oplogreferencedset.h
@@ -6,7 +6,10 @@
 
 #include <optional>
 #include <string_view>
-#include <unordered_set>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
@@ -24,8 +27,12 @@ class IoBuffer;
 class OplogReferencedSet
 {
 public:
-	void Emplace(Oid OplogId);
-	bool Contains(Oid OplogId, std::string_view OplogKey);
+	inline bool		   Contains(const Oid& OplogId) const { return Set.contains(OplogId); }
+	static inline bool IsNonPackage(std::string_view OplogKey)
+	{
+		// A referencedset always includes all non-package keys
+		return OplogKey.empty() || !OplogKey.starts_with('/');
+	}
 	void Clear();
 
 	static std::optional<OplogReferencedSet> LoadFromChunk(const IoBuffer& ChunkData);
@@ -33,7 +40,7 @@ public:
 	static constexpr std::string_view ReferencedSetOplogKey = "ReferencedSet";
 
 private:
-	std::unordered_set<Oid> Set;
+	tsl::robin_set<Oid, Oid::Hasher> Set;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 5277d689f..322af5e69 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -6996,14 +6996,8 @@ ProjectStore::CreateReferenceValidators(GcCtx& Ctx)
 Oid
 OpKeyStringAsOid(std::string_view OpKey)
 {
-	using namespace std::literals;
-
-	CbObjectWriter Writer;
-	Writer << "key"sv << OpKey;
-
-	const Oid OpId = ComputeOpKey(Writer.Save());
-
-	return OpId;
+	eastl::fixed_vector<uint8_t, 512> Buffer;
+	return OpKeyStringAsOid(OpKey, Buffer);
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 9f7de82b6..9d68d3e5e 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -2,7 +2,7 @@
 
 #pragma once
 
-#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
 #include <zencore/uid.h>
 #include <zencore/xxhash.h>
 #include <zenhttp/httpserver.h>
@@ -531,8 +531,27 @@ private:
 	friend class ProjectStoreReferenceChecker;
 };
 
+Oid ComputeOpKey(const CbObjectView& Op);
+
 Oid OpKeyStringAsOid(std::string_view OpKey);
 
+template<typename T>
+Oid
+OpKeyStringAsOid(std::string_view OpKey, T& TmpBuffer)
+{
+	using namespace std::literals;
+
+	CbObjectWriter Writer;
+	Writer << "key"sv << OpKey;
+	Writer.Finalize();
+	TmpBuffer.resize(Writer.GetSaveSize());
+	MutableMemoryView SaveBuffer(MutableMemoryView(TmpBuffer.data(), TmpBuffer.size()));
+
+	const Oid OpId = ComputeOpKey(Writer.Save(SaveBuffer).AsObjectView());
+
+	return OpId;
+}
+
 void prj_forcelink();
 
 }  // namespace zen