Make sure we monitor for new project, oplogs, namespaces and buckets during GCv2 (#93)

- Bugfix: Make sure we monitor and include new project/oplogs created during GCv2
- Bugfix: Make sure we monitor and include new namespaces/cache buckets created during GCv2

3 files changed, 72 insertions, 10 deletions

diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 9dee4d3f7..537f4396a 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -197,6 +197,12 @@ public:
 
 	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
 
+	std::vector<RwLock::SharedLockScope> GetGcReferencerLocks();
+
+	void					 EnableUpdateCapture();
+	void					 DisableUpdateCapture();
+	std::vector<std::string> GetCapturedBuckets();
+
 #if ZEN_WITH_TESTS
 	void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
 #endif	// ZEN_WITH_TESTS
@@ -227,6 +233,8 @@ public:
 		void	 ScrubStorage(ScrubContext& Ctx);
 		void	 GatherReferences(GcContext& GcCtx);
 		void	 CollectGarbage(GcContext& GcCtx);
+		RwLock::SharedLockScope GetGcReferencerLock();
+		bool					GetReferencesLocked(GcCtx& Ctx, std::vector<IoHash>& OutReferences);
 
 		inline GcStorageSize StorageSize() const
 		{
@@ -461,12 +469,16 @@ private:
 	mutable RwLock												  m_Lock;
 	std::unordered_map<std::string, std::unique_ptr<CacheBucket>> m_Buckets;
 	std::vector<std::unique_ptr<CacheBucket>>					  m_DroppedBuckets;
+	mutable RwLock												  m_UpdateCaptureLock;
+	uint32_t													  m_UpdateCaptureRefCounter = 0;
+	std::unique_ptr<std::vector<std::string>>					  m_CapturedBuckets;
 
 	ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
 	ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
 
 	friend class DiskBucketStoreCompactor;
 	friend class DiskBucketReferenceChecker;
+	friend class CacheStoreReferenceChecker;
 };
 
 }  // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h
index 7460d01ce..9160db667 100644
--- a/src/zenstore/include/zenstore/cache/structuredcachestore.h
+++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h
@@ -48,6 +48,7 @@ class JobQueue;
    projects from each other.
 
  */
+
 class ZenCacheNamespace final : public GcStorage, public GcContributor
 {
 public:
@@ -118,6 +119,11 @@ public:
 
 	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
 
+	std::vector<RwLock::SharedLockScope> GetGcReferencerLocks();
+
+	void EnableUpdateCapture();
+	void DisableUpdateCapture();
+
 #if ZEN_WITH_TESTS
 	void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
 #endif	// ZEN_WITH_TESTS
@@ -137,6 +143,8 @@ private:
 
 	ZenCacheNamespace(const ZenCacheNamespace&) = delete;
 	ZenCacheNamespace& operator=(const ZenCacheNamespace&) = delete;
+
+	friend class CacheStoreReferenceChecker;
 };
 
 /** Cache store interface
@@ -145,7 +153,7 @@ private:
 
   */
 
-class ZenCacheStore final : public RefCounted, public StatsProvider
+class ZenCacheStore final : public RefCounted, public StatsProvider, public GcReferencer, public GcReferenceLocker
 {
 public:
 	static constexpr std::string_view DefaultNamespace =
@@ -271,6 +279,16 @@ public:
 	// StatsProvider
 	virtual void ReportMetrics(StatsMetrics& Statsd) override;
 
+	virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override;
+
+	virtual std::string						 GetGcName(GcCtx& Ctx) override;
+	virtual GcStoreCompactor*				 RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
+	virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
+
+	void					 EnableUpdateCapture();
+	void					 DisableUpdateCapture();
+	std::vector<std::string> GetCapturedNamespaces();
+
 private:
 	const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const;
 	ZenCacheNamespace*		 GetNamespace(std::string_view Namespace);
@@ -283,6 +301,9 @@ private:
 	mutable RwLock									m_NamespacesLock;
 	NamespaceMap									m_Namespaces;
 	std::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces;
+	mutable RwLock									m_UpdateCaptureLock;
+	uint32_t										m_UpdateCaptureRefCounter = 0;
+	std::unique_ptr<std::vector<std::string>>		m_CapturedNamespaces;
 
 	GcManager&			  m_Gc;
 	JobQueue&			  m_JobQueue;
@@ -314,6 +335,8 @@ private:
 	std::thread				   m_AsyncLoggingThread;
 	std::atomic_bool		   m_WriteLogEnabled;
 	std::atomic_bool		   m_AccessLogEnabled;
+
+	friend class CacheStoreReferenceChecker;
 };
 
 void structured_cachestore_forcelink();
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 5262c6d2e..c3a71baa6 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -88,7 +88,7 @@ struct GcReferencerStats
 
 	std::chrono::milliseconds CreateReferenceCheckersMS = {};
 	std::chrono::milliseconds PreCacheStateMS			= {};
-	std::chrono::milliseconds LockStateMS				= {};
+	std::chrono::milliseconds UpdateLockedStateMS		= {};
 	std::chrono::milliseconds ElapsedMS					= {};
 };
 
@@ -115,6 +115,7 @@ struct GcResult
 	std::chrono::milliseconds CreateReferenceCheckersMS = {};
 	std::chrono::milliseconds PreCacheStateMS			= {};
 	std::chrono::milliseconds LockStateMS				= {};
+	std::chrono::milliseconds UpdateLockedStateMS		= {};
 
 	std::chrono::milliseconds CreateReferencePrunersMS = {};
 	std::chrono::milliseconds RemoveUnreferencedDataMS = {};
@@ -177,13 +178,19 @@ public:
 
 	virtual std::string GetGcName(GcCtx& Ctx) = 0;
 
+	// Read as much of the current state - nothing is locked for you here so you need to lock as appropriate
 	virtual void PreCache(GcCtx& Ctx) = 0;
 
-	// Lock the state and make sure no references changes, usually a read-lock is taken until the destruction
-	// of the instance. Called once before any calls to RemoveUsedReferencesFromSet
-	// The implementation should be as fast as possible as LockState is part of a stop the world (from changes)
-	// until all instances of GcReferenceChecker are deleted
-	virtual void LockState(GcCtx& Ctx) = 0;
+	// Update the state after all ReferenceCheckers has completed PreCache and all ReferenceLockers has
+	// completed their LockState operation.
+	// At this stage all data that UpdateLockedState needs to touch should be locked by the ReferenceLockers.
+	// *IMPORTANT* Do *not* take any locks (shared or exclusive) in this code.
+	// This is because we need to acquire the locks in an ordered manner and not end up in a deadlock due to other code
+	// trying to get exclusive locks halfway through our execution.
+	// Called once before any calls to RemoveUsedReferencesFromSet.
+	// The implementation should be as fast as possible as UpdateLockedState is part of a stop the world (from changes)
+	// until all instances of GcReferenceChecker UpdateLockedState are completed
+	virtual void UpdateLockedState(GcCtx& Ctx) = 0;
 
 	// Go through IoCids and see which ones are referenced. If it is the reference must be removed from IoCids
 	// This function should use pre-cached information on what is referenced as we are in stop the world mode
@@ -191,6 +198,22 @@ public:
 };
 
 /**
+ * @brief An interface to implement a lock for Stop The World (from writing new data)
+ *
+ * This interface is registered/unregistered to GcManager vua AddGcReferenceLocker() and RemoveGcReferenceLockerr()
+ */
+class GcReferenceLocker
+{
+public:
+	virtual ~GcReferenceLocker() = default;
+
+	// Take all the locks needed to execute UpdateLockedState for the all the GcReferenceChecker in your domain
+	// Once all the GcReferenceChecker has executed UpdateLockedState and RemoveUsedReferencesFromSet for all
+	// domains has completed, the locks will be disposed and writes are allowed once again
+	virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) = 0;
+};
+
+/**
  * @brief Interface to handle GC of data that references Cid data
  *
  * This interface is registered/unregistered to GcManager vua AddGcReferencer() and RemoveGcReferencer()
@@ -203,7 +226,7 @@ protected:
 public:
 	virtual std::string GetGcName(GcCtx& Ctx) = 0;
 
-	// Remove expired data based on either GcCtx::Settings CacheExpireTime/ProjectExpireTime
+	// Remove expired data based on either GcCtx::Settings CacheExpireTime or ProjectExpireTime
 	virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) = 0;
 
 	// Create 0-n GcReferenceChecker for this GcReferencer. Caller will manage lifetime of
@@ -350,6 +373,9 @@ public:
 	void AddGcReferencer(GcReferencer& Referencer);
 	void RemoveGcReferencer(GcReferencer& Referencer);
 
+	void AddGcReferenceLocker(GcReferenceLocker& ReferenceLocker);
+	void RemoveGcReferenceLocker(GcReferenceLocker& ReferenceLocker);
+
 	void AddGcReferenceStore(GcReferenceStore& ReferenceStore);
 	void RemoveGcReferenceStore(GcReferenceStore& ReferenceStore);
 
@@ -382,8 +408,9 @@ private:
 	CidStore*					m_CidStore		   = nullptr;
 	const DiskWriteBlocker*		m_DiskWriteBlocker = nullptr;
 
-	std::vector<GcReferencer*>	   m_GcReferencers;
-	std::vector<GcReferenceStore*> m_GcReferenceStores;
+	std::vector<GcReferencer*>		m_GcReferencers;
+	std::vector<GcReferenceLocker*> m_GcReferencerLockers;
+	std::vector<GcReferenceStore*>	m_GcReferenceStores;
 
 	std::atomic_bool m_CancelGC{false};
 };