gc unused refactor (#165)

* optimize IoHash and OId comparisions
* refactor filtering of unused references
* add attachment filtering to gc

2 files changed, 31 insertions, 28 deletions

diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 1eeb5637a..cd336c715 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -416,6 +416,16 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 				GcParams.SingleThreaded = Param == "true"sv;
 			}
 
+			if (auto Param = Params.GetValue("referencehashlow"); Param.empty() == false)
+			{
+				GcParams.AttachmentRangeMin = IoHash::FromHexString(Param);
+			}
+
+			if (auto Param = Params.GetValue("referencehashhigh"); Param.empty() == false)
+			{
+				GcParams.AttachmentRangeMax = IoHash::FromHexString(Param);
+			}
+
 			const bool Started = m_GcScheduler.TriggerGc(GcParams);
 
 			CbObjectWriter Response;
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index fb1385266..be50a03e2 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -5410,15 +5410,18 @@ public:
 
 			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
+		FilterReferences(Ctx, m_References);
 	}
 
-	virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override
+	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
 	{
-		ZEN_TRACE_CPU("Store::RemoveUsedReferencesFromSet");
+		ZEN_TRACE_CPU("Store::GetUnusedReferences");
 
 		auto Log = [&Ctx]() { return Ctx.Logger; };
 
-		size_t	   InitialCount = IoCids.size();
+		size_t InitialCount = IoCids.size();
+		size_t UsedCount	= InitialCount;
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
@@ -5427,21 +5430,14 @@ public:
 			}
 			ZEN_INFO("GCV2: projectstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}",
 					 "projectstore",
-					 InitialCount - IoCids.size(),
+					 UsedCount,
 					 InitialCount,
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		for (const IoHash& ReferenceHash : m_References)
-		{
-			if (IoCids.erase(ReferenceHash) == 1)
-			{
-				if (IoCids.empty())
-				{
-					return;
-				}
-			}
-		}
+		std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids);
+		UsedCount						   = IoCids.size() - UnusedReferences.size();
+		return UnusedReferences;
 	}
 
 private:
@@ -5548,6 +5544,7 @@ public:
 
 		Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		m_OplogAccessTime = m_Project->LastOplogAccessTime(m_OplogId);
+		FilterReferences(Ctx, m_References);
 	}
 
 	virtual void UpdateLockedState(GcCtx& Ctx) override
@@ -5595,13 +5592,15 @@ public:
 		}
 	}
 
-	virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override
+	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
 	{
-		ZEN_TRACE_CPU("Store::Oplog::RemoveUsedReferencesFromSet");
+		ZEN_TRACE_CPU("Store::Oplog::GetUnusedReferences");
 
 		auto Log = [&Ctx]() { return Ctx.Logger; };
 
-		size_t	   InitialCount = IoCids.size();
+		const size_t InitialCount = IoCids.size();
+		size_t		 UsedCount	  = InitialCount;
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
@@ -5610,24 +5609,18 @@ public:
 			}
 			ZEN_INFO("GCV2: projectstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {} from {}/{}",
 					 m_OplogBasePath,
-					 InitialCount - IoCids.size(),
+					 UsedCount,
 					 InitialCount,
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 					 m_Project->Identifier,
 					 m_OplogId);
 		});
 
-		for (const IoHash& ReferenceHash : m_References)
-		{
-			if (IoCids.erase(ReferenceHash) == 1)
-			{
-				if (IoCids.empty())
-				{
-					return;
-				}
-			}
-		}
+		std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids);
+		UsedCount						   = IoCids.size() - UnusedReferences.size();
+		return UnusedReferences;
 	}
+
 	ProjectStore&			   m_ProjectStore;
 	Ref<ProjectStore::Project> m_Project;
 	std::string				   m_OplogId;