From cda7cb764af09d90c5a1e5fd2a4e55f43e59581a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 21 Apr 2023 09:22:03 +0200 Subject: oplog and cache stats (#244) * basic oplog stats * add GetValueStats to cache store * RwLock::ExclusiveLockScope -> RwLock::SharedLockScope * add rawhash and attachment count to CacheValueStats * added cache-stats and project-stats commands * add cast to make Mac overload detection happy * fix accept type in cache-stats command * Add options to project-stats command * use resource paths for stats in project store * use resource paths for stats in cache * fix cache-info and project-info url discriminator * more control over details$ output * cleanup * changelog --- zenserver/cache/structuredcachestore.h | 63 ++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 14 deletions(-) (limited to 'zenserver/cache/structuredcachestore.h') diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index fe83f3c97..2ef4c4dcb 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -67,6 +67,31 @@ struct ZenCacheValue IoHash RawHash = IoHash::Zero; }; +struct CacheValueDetails +{ + struct ValueDetails + { + uint64_t Size; + uint64_t RawSize; + IoHash RawHash; + GcClock::Tick LastAccess{}; + std::vector Attachments; + ZenContentType ContentType; + }; + + struct BucketDetails + { + std::unordered_map Values; + }; + + struct NamespaceDetails + { + std::unordered_map Buckets; + }; + + std::unordered_map Namespaces; +}; + ////////////////////////////////////////////////////////////////////////// #pragma pack(push) @@ -301,6 +326,8 @@ public: Info GetInfo() const; std::optional GetBucketInfo(std::string_view Bucket) const; + CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const; + private: /** A cache bucket manages a single directory containing metadata and data for that bucket @@ -323,6 +350,8 @@ private: inline uint64_t TotalSize() const { return m_TotalStandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(); } uint64_t EntryCount() const; + CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const; + private: const uint64_t MaxBlockSize = 1ull << 30; uint64_t m_PayloadAlignment = 1ull << 4; @@ -360,18 +389,18 @@ private: std::atomic_uint64_t m_TotalStandaloneSize{}; - void BuildPath(PathBuilderBase& Path, const IoHash& HashKey); - void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); - IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey); - void PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); - IoBuffer GetInlineCacheValue(const DiskLocation& Loc); - void MakeIndexSnapshot(); - uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion); - uint64_t ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition); - uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition); - void OpenLog(const bool IsNew); - void SaveManifest(); - + void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const; + void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); + IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const; + void PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); + IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const; + void MakeIndexSnapshot(); + uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion); + uint64_t ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition); + uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition); + void OpenLog(const bool IsNew); + void SaveManifest(); + CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, size_t Index) const; // These locks are here to avoid contention on file creation, therefore it's sufficient // that we take the same lock for the same hash // @@ -379,8 +408,8 @@ private: // but we don't currently access them at particularly high frequency so it should not be // an issue in practice - RwLock m_ShardedLocks[256]; - inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; } + mutable RwLock m_ShardedLocks[256]; + inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; } }; std::filesystem::path m_RootDir; @@ -429,6 +458,8 @@ public: Info GetInfo() const; std::optional GetBucketInfo(std::string_view Bucket) const; + CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const; + private: std::filesystem::path m_RootDir; ZenCacheMemoryLayer m_MemLayer; @@ -476,6 +507,10 @@ public: void Flush(); void Scrub(ScrubContext& Ctx); + CacheValueDetails GetValueDetails(const std::string_view NamespaceFilter, + const std::string_view BucketFilter, + const std::string_view ValueFilter) const; + GcStorageSize StorageSize() const; // const Configuration& GetConfiguration() const { return m_Configuration; } -- cgit v1.2.3