reduce cachebucket mem (#509)

* reduce memory footprint for disk cache separate dense arrays for rawhash+rawsize and memcache buffer
* don't write RawHash/RawSize for buckets with no such metadata
* helper functions
* make index into metadata and cached payload type safe
* helper functions for memcached

1 files changed, 94 insertions, 33 deletions

diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index ded8f2e70..4b9d4ed1e 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -231,20 +231,72 @@ private:
 
 #pragma pack(push)
 #pragma pack(1)
-		static const size_t UnknownReferencesIndex = (size_t)-1;
-		static const size_t NoReferencesIndex	   = (size_t)-2;
+		struct MetaDataIndex
+		{
+			size_t Index = std::numeric_limits<size_t>::max();
+
+			operator bool() const { return Index != std::numeric_limits<size_t>::max(); };
+			MetaDataIndex() = default;
+			explicit MetaDataIndex(size_t Index) : Index(Index) {}
+						operator size_t() const { return Index; };
+			inline auto operator<=>(const MetaDataIndex& Other) const = default;
+		};
+
+		struct MemCachedIndex
+		{
+			uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+			operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+			MemCachedIndex() = default;
+			explicit MemCachedIndex(uint32_t Index) : Index(Index) {}
+						operator size_t() const { return Index; };
+			inline auto operator<=>(const MemCachedIndex& Other) const = default;
+		};
+
+		struct ReferenceIndex
+		{
+			size_t Index = std::numeric_limits<size_t>::max();
+
+			static const ReferenceIndex Unknown() { return ReferenceIndex{std::numeric_limits<size_t>::max()}; }
+			static const ReferenceIndex None() { return ReferenceIndex{std::numeric_limits<size_t>::max() - 1}; }
+
+			ReferenceIndex() = default;
+			explicit ReferenceIndex(size_t Index) : Index(Index) {}
+						operator size_t() const { return Index; };
+						operator bool() const { return Index != std::numeric_limits<size_t>::max(); };
+			inline auto operator<=>(const ReferenceIndex& Other) const = default;
+		};
+
+		struct PayloadIndex
+		{
+			size_t Index = std::numeric_limits<size_t>::max();
+
+			operator bool() const { return Index != std::numeric_limits<size_t>::max(); };
+			PayloadIndex() = default;
+			explicit PayloadIndex(size_t Index) : Index(Index) {}
+						operator size_t() const { return Index; };
+			inline auto operator<=>(const PayloadIndex& Other) const = default;
+		};
 
 		struct BucketPayload
 		{
-			DiskLocation Location;	// 12
-			uint64_t	 RawSize;	// 8
-			IoHash		 RawHash;	// 20
+			DiskLocation   Location;   // 12
+			MetaDataIndex  MetaData;   // 8
+			MemCachedIndex MemCached;  // 4
+		};
+		struct BucketMetaData
+		{
+			uint64_t RawSize = 0;  // 8
+			IoHash	 RawHash;	   // 20
+
+			operator bool() const { return RawSize != 0 || RawHash != IoHash::Zero; };
 		};
 #pragma pack(pop)
-		static_assert(sizeof(BucketPayload) == 40u);
+		static_assert(sizeof(BucketPayload) == 24u);
+		static_assert(sizeof(BucketMetaData) == 28u);
 		static_assert(sizeof(AccessTime) == 4u);
 
-		using IndexMap = tsl::robin_map<IoHash, size_t, IoHash::Hasher>;
+		using IndexMap = tsl::robin_map<IoHash, PayloadIndex, IoHash::Hasher>;
 
 		std::atomic<uint64_t> m_DiskHitCount;
 		std::atomic<uint64_t> m_DiskMissCount;
@@ -255,17 +307,20 @@ private:
 		metrics::RequestStats m_PutOps;
 		metrics::RequestStats m_GetOps;
 
-		mutable RwLock			   m_IndexLock;
-		IndexMap				   m_Index;
-		std::vector<AccessTime>	   m_AccessTimes;
-		std::vector<BucketPayload> m_Payloads;
-		std::vector<IoBuffer>	   m_CachedPayloads;
-		std::vector<size_t>		   m_FirstReferenceIndex;
-		std::vector<IoHash>		   m_ReferenceHashes;
-		std::vector<size_t>		   m_NextReferenceHashesIndexes;
-		size_t					   m_ReferenceCount = 0;
-		std::atomic_uint64_t	   m_StandaloneSize{};
-		std::atomic_uint64_t	   m_MemCachedSize{};
+		mutable RwLock				m_IndexLock;
+		IndexMap					m_Index;
+		std::vector<AccessTime>		m_AccessTimes;
+		std::vector<BucketPayload>	m_Payloads;
+		std::vector<BucketMetaData> m_MetaDatas;
+		std::vector<MetaDataIndex>	m_FreeMetaDatas;
+		std::vector<IoBuffer>		m_MemCachedPayloads;
+		std::vector<MemCachedIndex> m_FreeMemCachedPayloads;
+		std::vector<ReferenceIndex> m_FirstReferenceIndex;
+		std::vector<IoHash>			m_ReferenceHashes;
+		std::vector<ReferenceIndex> m_NextReferenceHashesIndexes;
+		size_t						m_ReferenceCount = 0;
+		std::atomic_uint64_t		m_StandaloneSize{};
+		std::atomic_uint64_t		m_MemCachedSize{};
 
 		virtual std::string						 GetGcName(GcCtx& Ctx) override;
 		virtual void							 RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
@@ -282,28 +337,34 @@ private:
 		void	 OpenLog(const bool IsNew);
 		CbObject MakeManifest(IndexMap&& Index, std::vector<AccessTime>&& AccessTimes, const std::vector<BucketPayload>& Payloads);
 		void	 SaveManifest(CbObject&& Manifest);
-		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, size_t Index) const;
+		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, PayloadIndex Index) const;
 		void							CompactReferences(RwLock::ExclusiveLockScope&);
-		void		SetReferences(RwLock::ExclusiveLockScope&, std::size_t& FirstReferenceIndex, std::span<IoHash> References);
-		void		RemoveReferences(RwLock::ExclusiveLockScope&, std::size_t& FirstReferenceIndex);
-		inline bool GetReferences(RwLock::SharedLockScope&, std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
+		void		SetReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex, std::span<IoHash> References);
+		void		RemoveReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex);
+		inline bool GetReferences(RwLock::SharedLockScope&, ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
 		{
 			return LockedGetReferences(FirstReferenceIndex, OutReferences);
 		}
-		inline bool GetReferences(RwLock::ExclusiveLockScope&, std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
+		inline bool GetReferences(RwLock::ExclusiveLockScope&, ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
 		{
 			return LockedGetReferences(FirstReferenceIndex, OutReferences);
 		}
-		size_t AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key);
-		bool   LockedGetReferences(std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
-		void   ClearReferenceCache();
-
-		void CompactState(std::vector<BucketPayload>& TmpPayloads,
-						  std::vector<AccessTime>&	  TmpAccessTimes,
-						  std::vector<IoBuffer>&	  TmpCachedPayloads,
-						  std::vector<size_t>&		  TmpFirstReferenceIndex,
-						  IndexMap&					  TmpIndex,
-						  RwLock::ExclusiveLockScope& IndexLock);
+		ReferenceIndex AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key);
+		bool		   LockedGetReferences(ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
+		void		   ClearReferenceCache();
+		void		   SetMetaData(BucketPayload& Payload, const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData);
+		void		   RemoveMetaData(BucketPayload& Payload);
+		BucketMetaData GetMetaData(const BucketPayload& Payload) const;
+		void		   SetMemCachedData(BucketPayload& Payload, IoBuffer& MemCachedData);
+		size_t		   RemoveMemCachedData(BucketPayload& Payload);
+
+		void CompactState(std::vector<BucketPayload>&  Payloads,
+						  std::vector<AccessTime>&	   AccessTimes,
+						  std::vector<BucketMetaData>& MetaDatas,
+						  std::vector<IoBuffer>&	   MemCachedPayloads,
+						  std::vector<ReferenceIndex>& FirstReferenceIndex,
+						  IndexMap&					   Index,
+						  RwLock::ExclusiveLockScope&  IndexLock);
 
 		void AddMemCacheUsage(uint64_t ValueSize)
 		{