cache reference tracking (#455)

- Feature: Add caching of referenced CId content for structured cache records, this avoid disk thrashing when gathering references for GC
  - disabled by default, enable with `--cache-reference-cache-enabled`
- Improvement: Faster collection of referenced CId content in project store

1 files changed, 32 insertions, 11 deletions

diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index 62163100d..7e05430a2 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -93,6 +93,7 @@ public:
 	struct Configuration
 	{
 		std::filesystem::path RootDir;
+		bool				  EnableReferenceCaching;
 	};
 
 	struct BucketInfo
@@ -130,11 +131,11 @@ public:
 		std::vector<NamedBucketStats> BucketStats;
 	};
 
-	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
+	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir, bool EnableReferenceCaching);
 	~ZenCacheDiskLayer();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
-	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 	bool Drop();
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
@@ -160,12 +161,12 @@ private:
 	  */
 	struct CacheBucket
 	{
-		CacheBucket(std::string BucketName);
+		CacheBucket(std::string BucketName, bool EnableReferenceCaching);
 		~CacheBucket();
 
 		bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
 		bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
-		void Put(const IoHash& HashKey, const ZenCacheValue& Value);
+		void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		bool Drop();
 		void Flush();
 		void ScrubStorage(ScrubContext& Ctx);
@@ -191,6 +192,7 @@ private:
 		Oid					  m_BucketId;
 		uint64_t			  m_LargeObjectThreshold = 128 * 1024;
 		std::atomic_bool	  m_IsFlushing{};
+		const bool			  m_EnableReferenceCaching = false;
 
 		// These files are used to manage storage of small objects for this bucket
 
@@ -199,6 +201,9 @@ private:
 
 #pragma pack(push)
 #pragma pack(1)
+		static const size_t UnknownReferencesIndex = (size_t)-1;
+		static const size_t NoReferencesIndex	   = (size_t)-2;
+
 		struct BucketPayload
 		{
 			DiskLocation Location;	// 12
@@ -218,16 +223,19 @@ private:
 		metrics::RequestStats m_GetOps;
 
 		mutable RwLock			   m_IndexLock;
+		IndexMap				   m_Index;
 		std::vector<AccessTime>	   m_AccessTimes;
 		std::vector<BucketPayload> m_Payloads;
-		IndexMap				   m_Index;
-
-		std::atomic_uint64_t m_TotalStandaloneSize{};
+		std::vector<size_t>		   m_FirstReferenceIndex;
+		std::vector<IoHash>		   m_ReferenceHashes;
+		std::vector<size_t>		   m_NextReferenceHashesIndexes;
+		size_t					   m_ReferenceCount = 0;
+		std::atomic_uint64_t	   m_TotalStandaloneSize{};
 
 		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
-		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
-		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
+		IoBuffer GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const;
+		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
 		void	 MakeIndexSnapshot();
 		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
@@ -236,13 +244,25 @@ private:
 		CbObject MakeManifest(IndexMap&& Index, std::vector<AccessTime>&& AccessTimes, const std::vector<BucketPayload>& Payloads);
 		void	 SaveManifest(CbObject&& Manifest);
 		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, size_t Index) const;
+		void							CompactReferences(RwLock::ExclusiveLockScope&);
+		void		SetReferences(RwLock::ExclusiveLockScope&, std::size_t& FirstReferenceIndex, std::span<IoHash> References);
+		void		RemoveReferences(RwLock::ExclusiveLockScope&, std::size_t& FirstReferenceIndex);
+		inline bool GetReferences(RwLock::SharedLockScope&, std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
+		{
+			return LockedGetReferences(FirstReferenceIndex, OutReferences);
+		}
+		inline bool GetReferences(RwLock::ExclusiveLockScope&, std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
+		{
+			return LockedGetReferences(FirstReferenceIndex, OutReferences);
+		}
+		size_t AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key);
+		bool   LockedGetReferences(std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
 		// These locks are here to avoid contention on file creation, therefore it's sufficient
 		// that we take the same lock for the same hash
 		//
 		// These locks are small and should really be spaced out so they don't share cache lines,
 		// but we don't currently access them at particularly high frequency so it should not be
 		// an issue in practice
-
 		mutable RwLock m_ShardedLocks[256];
 		inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
 	};
@@ -251,6 +271,7 @@ private:
 	mutable RwLock												  m_Lock;
 	std::unordered_map<std::string, std::unique_ptr<CacheBucket>> m_Buckets;  // TODO: make this case insensitive
 	std::vector<std::unique_ptr<CacheBucket>>					  m_DroppedBuckets;
+	const bool													  m_EnableReferenceCaching;
 
 	ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
 	ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;