gc unused refactor (#165)

* optimize IoHash and OId comparisions
* refactor filtering of unused references
* add attachment filtering to gc

1 files changed, 27 insertions, 27 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 63f6d708a..417b63fb4 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -1262,7 +1262,9 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept
 						}
 						if (m_TrackedReferences && HashKeyAndReferences.size() > 1)
 						{
-							m_TrackedReferences->insert(HashKeyAndReferences.begin() + 1, HashKeyAndReferences.end());
+							m_TrackedReferences->insert(m_TrackedReferences->end(),
+														HashKeyAndReferences.begin() + 1,
+														HashKeyAndReferences.end());
 						}
 						if (auto It = m_Index.find(HashKey); It != m_Index.end())
 						{
@@ -2963,7 +2965,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 	}
 	if (m_TrackedReferences)
 	{
-		m_TrackedReferences->insert(References.begin(), References.end());
+		m_TrackedReferences->insert(m_TrackedReferences->end(), References.begin(), References.end());
 	}
 
 	PayloadIndex EntryIndex = {};
@@ -3130,7 +3132,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash&		 HashKey,
 								}
 								if (m_TrackedReferences)
 								{
-									m_TrackedReferences->insert(References.begin(), References.end());
+									m_TrackedReferences->insert(m_TrackedReferences->end(), References.begin(), References.end());
 								}
 								if (auto It = m_Index.find(HashKey); It != m_Index.end())
 								{
@@ -3782,17 +3784,19 @@ public:
 			}
 			ZEN_INFO("GCV2: cachebucket [PRECACHE] '{}': found {} references in {}",
 					 m_CacheBucket.m_BucketDir,
-					 m_References.size(),
+					 m_PrecachedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<HashSet>(); });
+		m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<std::vector<IoHash>>(); });
 
-		bool Continue = m_CacheBucket.GetReferences(Ctx, /*StateIsAlreadyLocked*/ false, m_References);
+		bool Continue = m_CacheBucket.GetReferences(Ctx, /*StateIsAlreadyLocked*/ false, m_PrecachedReferences);
 		if (!Continue)
 		{
 			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
+			return;
 		}
+		FilterReferences(Ctx, m_PrecachedReferences);
 	}
 
 	virtual void UpdateLockedState(GcCtx& Ctx) override
@@ -3809,32 +3813,32 @@ public:
 			}
 			ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}",
 					 m_CacheBucket.m_BucketDir,
-					 m_References.size(),
+					 m_PrecachedReferences.size() + m_AddedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		if (Ctx.IsCancelledFlag.load())
 		{
-			m_References = {};
+			m_PrecachedReferences = {};
 			m_CacheBucket.m_TrackedReferences.reset();
 			return;
 		}
 
 		ZEN_ASSERT(m_CacheBucket.m_TrackedReferences);
 
-		HashSet& AddedReferences(*m_CacheBucket.m_TrackedReferences);
-		m_References.reserve(m_References.size() + AddedReferences.size());
-		m_References.insert(m_References.end(), AddedReferences.begin(), AddedReferences.end());
-		AddedReferences = {};
+		m_AddedReferences = std::move(*m_CacheBucket.m_TrackedReferences);
+		FilterReferences(Ctx, m_AddedReferences);
 	}
 
-	virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override
+	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
 	{
-		ZEN_TRACE_CPU("Z$::Bucket::RemoveUsedReferencesFromSet");
+		ZEN_TRACE_CPU("Z$::Bucket::GetUnusedReferences");
 
 		auto Log = [&Ctx]() { return Ctx.Logger; };
 
-		size_t	   InitialCount = IoCids.size();
+		const size_t InitialCount = IoCids.size();
+		size_t		 UsedCount	  = InitialCount;
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
@@ -3843,24 +3847,20 @@ public:
 			}
 			ZEN_INFO("GCV2: cachebucket [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}",
 					 m_CacheBucket.m_BucketDir,
-					 InitialCount - IoCids.size(),
+					 UsedCount,
 					 InitialCount,
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		for (const IoHash& ReferenceHash : m_References)
-		{
-			if (IoCids.erase(ReferenceHash) == 1)
-			{
-				if (IoCids.empty())
-				{
-					return;
-				}
-			}
-		}
+		std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_PrecachedReferences, IoCids);
+		UnusedReferences				   = KeepUnusedReferences(m_AddedReferences, UnusedReferences);
+		UsedCount						   = IoCids.size() - UnusedReferences.size();
+		return UnusedReferences;
 	}
+
 	CacheBucket&		m_CacheBucket;
-	std::vector<IoHash> m_References;
+	std::vector<IoHash> m_PrecachedReferences;
+	std::vector<IoHash> m_AddedReferences;
 };
 
 std::vector<GcReferenceChecker*>