diff options
| author | Dan Engelbrecht <[email protected]> | 2023-10-30 18:29:09 +0100 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-10-30 18:29:09 +0100 |
| commit | cbdda104ada38108700f9da5b192867d83074119 (patch) | |
| tree | 98c04b344e041c156fdc1a5c393672bef743be34 /src/zenserver | |
| parent | fix changelog (diff) | |
| download | zen-cbdda104ada38108700f9da5b192867d83074119.tar.xz zen-cbdda104ada38108700f9da5b192867d83074119.zip | |
individual gc stats (#506)
- Feature: New parameter for endpoint `admin/gc` (GET) `details=true` which gives details stats on GC operation when using GC V2
- Feature: New options for zen command `gc-status`
- `--details` that enables the detailed output from the last GC operation when using GC V2
Diffstat (limited to 'src/zenserver')
| -rw-r--r-- | src/zenserver/admin/admin.cpp | 130 | ||||
| -rw-r--r-- | src/zenserver/cache/cachedisklayer.cpp | 65 | ||||
| -rw-r--r-- | src/zenserver/cache/cachedisklayer.h | 3 | ||||
| -rw-r--r-- | src/zenserver/cache/structuredcachestore.cpp | 306 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 91 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.h | 3 |
6 files changed, 402 insertions, 196 deletions
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index fb376f238..83f33d4ea 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -199,6 +199,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, [this](HttpRouterRequest& Req) { const GcSchedulerState State = m_GcScheduler.GetState(); + const HttpServerRequest::QueryParams Params = Req.ServerRequest().GetQueryParams(); + + bool Details = false; + if (auto Param = Params.GetValue("details"); Param == "true") + { + Details = true; + } + auto SecondsToString = [](std::chrono::seconds Secs) { return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count())); }; @@ -226,6 +234,101 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Response << "DiskUsed" << NiceBytes(State.DiskUsed); Response << "DiskFree" << NiceBytes(State.DiskFree); + auto WriteReferencerStats = [&](CbObjectWriter& Response, const GcReferencerStats& Stats) { + if (Stats.Count == 0) + { + return; + } + Response << "Count" << Stats.Count; + Response << "Expired" << Stats.Expired; + Response << "Deleted" << Stats.Deleted; + + Response << "RemovedDisk" << NiceBytes(Stats.RemovedDisk); + Response << "RemovedMemory" << NiceBytes(Stats.RemovedMemory); + + Response << "RemoveExpiredData" << NiceTimeSpanMs(Stats.RemoveExpiredDataMS.count()); + Response << "CreateReferenceCheckers" << NiceTimeSpanMs(Stats.CreateReferenceCheckersMS.count()); + Response << "LockState" << NiceTimeSpanMs(Stats.LockStateMS.count()); + Response << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count()); + }; + + auto WriteReferenceStoreStats = [&](CbObjectWriter& Response, const GcReferenceStoreStats& Stats) { + if (Stats.Count == 0) + { + return; + } + Response << "Count" << Stats.Count; + Response << "Pruned" << Stats.Pruned; + Response << "Compacted" << Stats.Compacted; + + Response << "RemovedDisk" << NiceBytes(Stats.RemovedDisk); + Response << "RemovedMemory" << NiceBytes(Stats.RemovedMemory); + + Response << "CreateReferencePruner" << NiceTimeSpanMs(Stats.CreateReferencePrunerMS.count()); + Response << "RemoveUnreferencedData" << NiceTimeSpanMs(Stats.RemoveUnreferencedDataMS.count()); + Response << "CompactReferenceStore" << NiceTimeSpanMs(Stats.CompactReferenceStoreMS.count()); + Response << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count()); + }; + + auto WriteGCResult = [&](CbObjectWriter& Response, const GcResult& Result) { + Response << "RemovedDisk" << NiceBytes(Result.RemovedDisk); + Response << "RemovedMemory" << NiceBytes(Result.RemovedMemory); + Response << "WriteBlock" << NiceTimeSpanMs(Result.WriteBlockMS.count()); + Response << "Elapsed" << NiceTimeSpanMs(Result.ElapsedMS.count()); + + if (!Details) + { + return; + } + + Response << "RemoveExpiredData" << NiceTimeSpanMs(Result.RemoveExpiredDataMS.count()); + Response << "CreateReferenceCheckers" << NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count()); + Response << "LockState" << NiceTimeSpanMs(Result.LockStateMS.count()); + + Response << "CreateReferencePruner" << NiceTimeSpanMs(Result.CreateReferencePrunerMS.count()); + Response << "RemoveUnreferencedData" << NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count()); + Response << "CompactReferenceStore" << NiceTimeSpanMs(Result.CompactReferenceStoreMS.count()); + + Response.BeginObject("ReferencerStats"); + { + WriteReferencerStats(Response, Result.ReferencerStat); + } + Response.EndObject(); + + Response.BeginObject("ReferenceStoreStats"); + { + WriteReferenceStoreStats(Response, Result.ReferenceStoreStat); + } + Response.EndObject(); + + if (!Result.ReferencerStats.empty()) + { + Response.BeginArray("Referencers"); + { + for (const std::pair<std::string, GcReferencerStats>& It : Result.ReferencerStats) + { + Response.BeginObject(); + Response << "Name" << It.first; + WriteReferencerStats(Response, It.second); + Response.EndObject(); + } + } + Response.EndArray(); + } + if (!Result.ReferenceStoreStats.empty()) + { + Response.BeginArray("ReferenceStores"); + for (const std::pair<std::string, GcReferenceStoreStats>& It : Result.ReferenceStoreStats) + { + Response.BeginObject(); + Response << "Name" << It.first; + WriteReferenceStoreStats(Response, It.second); + Response.EndObject(); + } + Response.EndArray(); + } + }; + Response.BeginObject("FullGC"); { Response << "LastTime" << fmt::format("{}", State.LastFullGcTime); @@ -234,18 +337,33 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, { Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC); } - Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count()); - Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); - Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); + if (State.LastFullGCV2Result) + { + WriteGCResult(Response, State.LastFullGCV2Result.value()); + } + else + { + Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count()); + Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); + Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); + } } Response.EndObject(); Response.BeginObject("LightweightGC"); { Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime); Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc); - Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count()); - Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize); - Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize); + + if (State.LastLightweightGCV2Result) + { + WriteGCResult(Response, State.LastLightweightGCV2Result.value()); + } + else + { + Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count()); + Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize); + Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize); + } } Response.EndObject(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp index 38cbf3a93..fa80ed414 100644 --- a/src/zenserver/cache/cachedisklayer.cpp +++ b/src/zenserver/cache/cachedisklayer.cpp @@ -2213,8 +2213,14 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const }); } +std::string +ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&) +{ + return fmt::format("cachebucket:'{}'", m_BucketDir.string()); +} + void -ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) +ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) { size_t TotalEntries = 0; tsl::robin_set<IoHash, IoHash::Hasher> ExpiredInlineKeys; @@ -2222,11 +2228,18 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) Stopwatch Timer; const auto _ = MakeGuard([&] { - ZEN_DEBUG("gc cache bucket '{}': removed {} expired keys out of {} in {}", - m_BucketDir, - ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size(), - TotalEntries, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}", + m_BucketDir, + Stats.Count, + Stats.Expired, + Stats.Deleted, + NiceBytes(Stats.RemovedDisk), + NiceBytes(Stats.RemovedMemory), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count(); @@ -2278,7 +2291,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) } } - Ctx.ExpiredItems.fetch_add(ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size()); + Stats.Expired += ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size(); // Get all locations we need to keep for affected blocks if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty()) @@ -2312,7 +2325,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[It->second]) { size_t PayloadSize = m_CachedPayloads[It->second].GetSize(); - Ctx.RemovedMemory.fetch_add(PayloadSize); + Stats.RemovedMemory += PayloadSize; RemoveMemCacheUsage(PayloadSize); } m_Index.erase(It); @@ -2321,7 +2334,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed); } } - Ctx.Items.fetch_add(TotalEntries); + Stats.Count += TotalEntries; if (ExpiredEntries.empty()) { @@ -2333,7 +2346,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) return; } - Ctx.DeletedItems.fetch_add(ExpiredEntries.size()); + Stats.Deleted += ExpiredEntries.size(); // Compact standalone items ExtendablePathBuilder<256> Path; @@ -2370,7 +2383,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) Ec.message()); continue; } - Ctx.RemovedDiskSpace.fetch_add(ExpiredKey.second); + Stats.RemovedDisk += ExpiredKey.second; } if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty()) @@ -2405,7 +2418,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx) } } m_SlogFile.Append(MovedEntries); - Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace); + Stats.RemovedDisk += FreedDiskSpace; }, [&]() { return 0; }); } @@ -2436,14 +2449,18 @@ public: } } - virtual void LockState(GcCtx&) override + virtual void LockState(GcCtx& Ctx) override { Stopwatch Timer; const auto _ = MakeGuard([&] { - ZEN_DEBUG("gc cache bucket '{}': found {} references in {}", - m_CacheBucket.m_BucketDir, - m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(), - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}", + m_CacheBucket.m_BucketDir, + m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock); @@ -2503,11 +2520,19 @@ public: }; std::vector<GcReferenceChecker*> -ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx&) +ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx) { Stopwatch Timer; - const auto _ = MakeGuard( - [&] { ZEN_DEBUG("gc cache bucket '{}': refreshed reference cache in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': found {} references in {}", + m_BucketDir, + m_ReferenceCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); std::vector<IoHash> UpdateKeys; std::vector<IoHash> StandaloneKeys; diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h index d8f51c398..ded8f2e70 100644 --- a/src/zenserver/cache/cachedisklayer.h +++ b/src/zenserver/cache/cachedisklayer.h @@ -267,7 +267,8 @@ private: std::atomic_uint64_t m_StandaloneSize{}; std::atomic_uint64_t m_MemCachedSize{}; - virtual void RemoveExpiredData(GcCtx& Ctx) override; + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override; virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const; diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp index 516532528..55ab6bf31 100644 --- a/src/zenserver/cache/structuredcachestore.cpp +++ b/src/zenserver/cache/structuredcachestore.cpp @@ -1982,18 +1982,20 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() - std::chrono::hours(1), .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(1), .CollectSmallObjects = false, - .IsDeleteMode = false}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(0u, Result.ExpiredItems); - CHECK_EQ(0u, Result.DeletedItems); - CHECK_EQ(5u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_EQ(0u, Result.RemovedDiskSpace); + .IsDeleteMode = false, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(0u, Result.ReferencerStat.Expired); + CHECK_EQ(0u, Result.ReferencerStat.Deleted); + CHECK_EQ(5u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_EQ(0u, Result.RemovedDisk); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true)); @@ -2014,18 +2016,20 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = false, - .IsDeleteMode = false}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(1u, Result.ExpiredItems); - CHECK_EQ(0u, Result.DeletedItems); - CHECK_EQ(5u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_EQ(0u, Result.RemovedDiskSpace); + .IsDeleteMode = false, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(1u, Result.ReferencerStat.Expired); + CHECK_EQ(0u, Result.ReferencerStat.Deleted); + CHECK_EQ(5u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_EQ(0u, Result.RemovedDisk); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true)); @@ -2046,18 +2050,20 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = true, - .IsDeleteMode = false}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(7u, Result.ExpiredItems); - CHECK_EQ(0u, Result.DeletedItems); - CHECK_EQ(5u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_EQ(0u, Result.RemovedDiskSpace); + .IsDeleteMode = false, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(7u, Result.ReferencerStat.Expired); + CHECK_EQ(0u, Result.ReferencerStat.Deleted); + CHECK_EQ(5u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_EQ(0u, Result.RemovedDisk); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true)); @@ -2078,19 +2084,21 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = false, .IsDeleteMode = true, - .SkipCidDelete = true}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(1u, Result.ExpiredItems); - CHECK_EQ(1u, Result.DeletedItems); - CHECK_EQ(0u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace); + .SkipCidDelete = true, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(1u, Result.ReferencerStat.Expired); + CHECK_EQ(1u, Result.ReferencerStat.Deleted); + CHECK_EQ(0u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true)); @@ -2111,19 +2119,21 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = true, .IsDeleteMode = true, - .SkipCidDelete = true}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(7u, Result.ExpiredItems); - CHECK_EQ(7u, Result.DeletedItems); - CHECK_EQ(0u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_GE(Result.RemovedDiskSpace, 0); + .SkipCidDelete = true, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(7u, Result.ReferencerStat.Expired); + CHECK_EQ(7u, Result.ReferencerStat.Deleted); + CHECK_EQ(0u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_GE(Result.RemovedDisk, 0); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true)); @@ -2144,21 +2154,24 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = false, .IsDeleteMode = true, - .SkipCidDelete = false}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(1u, Result.ExpiredItems); // Only one cache value is pruned/deleted as that is the only large item in the cache (all other - // large items as in cas) - CHECK_EQ(1u, Result.DeletedItems); - CHECK_EQ(5u, Result.References); + .SkipCidDelete = false, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(1u, Result.ReferencerStat.Expired); // Only one cache value is pruned/deleted as that is the only large item in the cache + // (all other large items as in cas) + CHECK_EQ(1u, Result.ReferencerStat.Deleted); + CHECK_EQ(5u, Result.ReferenceStoreStat.Count); CHECK_EQ(0u, - Result.PrunedReferences); // We won't remove any references since all referencers are small which retains all references - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace); + Result.ReferenceStoreStat + .Pruned); // We won't remove any references since all referencers are small which retains all references + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true)); @@ -2179,18 +2192,21 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), - .CollectSmallObjects = true, - .IsDeleteMode = true, - .SkipCidDelete = false}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(7u, Result.ExpiredItems); - CHECK_EQ(7u, Result.DeletedItems); - CHECK_EQ(5u, Result.References); - CHECK_EQ(5u, Result.PrunedReferences); - CHECK_EQ(5u, Result.CompactedReferences); - CHECK_GT(Result.RemovedDiskSpace, 0); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); + + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), + .CollectSmallObjects = true, + .IsDeleteMode = true, + .SkipCidDelete = false, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(7u, Result.ReferencerStat.Expired); + CHECK_EQ(7u, Result.ReferencerStat.Deleted); + CHECK_EQ(5u, Result.ReferenceStoreStat.Count); + CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted); + CHECK_GT(Result.RemovedDisk, 0); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false)); @@ -2212,20 +2228,23 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); - Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2)); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2)); + + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = true, .IsDeleteMode = true, - .SkipCidDelete = true}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(6u, Result.ExpiredItems); - CHECK_EQ(6u, Result.DeletedItems); - CHECK_EQ(0u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_GT(Result.RemovedDiskSpace, 0); + .SkipCidDelete = true, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(6u, Result.ReferencerStat.Expired); + CHECK_EQ(6u, Result.ReferencerStat.Deleted); + CHECK_EQ(0u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_GT(Result.RemovedDisk, 0); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true)); @@ -2247,21 +2266,24 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); - Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2)); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); + + Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2)); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = true, .IsDeleteMode = true, - .SkipCidDelete = false}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(5u, Result.ExpiredItems); - CHECK_EQ(5u, Result.DeletedItems); - CHECK_EQ(5u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_GT(Result.RemovedDiskSpace, 0); + .SkipCidDelete = false, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(5u, Result.ReferencerStat.Expired); + CHECK_EQ(5u, Result.ReferencerStat.Deleted); + CHECK_EQ(5u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_GT(Result.RemovedDisk, 0); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true)); @@ -2283,22 +2305,25 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2)); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); + + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2)); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = true, .IsDeleteMode = true, - .SkipCidDelete = false}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(4u, Result.ExpiredItems); - CHECK_EQ(4u, Result.DeletedItems); - CHECK_EQ(5u, Result.References); - CHECK_EQ(5u, Result.PrunedReferences); - CHECK_EQ(5u, Result.CompactedReferences); - CHECK_GT(Result.RemovedDiskSpace, 0); + .SkipCidDelete = false, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(4u, Result.ReferencerStat.Expired); + CHECK_EQ(4u, Result.ReferencerStat.Deleted); + CHECK_EQ(5u, Result.ReferenceStoreStat.Count); + CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted); + CHECK_GT(Result.RemovedDisk, 0); CHECK_EQ(0u, Result.RemovedMemory); CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false)); @@ -2320,6 +2345,7 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); // Prime so we can check GC of memory layer ZenCacheValue Dummy; @@ -2331,22 +2357,23 @@ TEST_CASE("z$.newgc.basics") Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[2], Dummy); Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[3], Dummy); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2)); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = true, .IsDeleteMode = true, - .SkipCidDelete = true}); - CHECK_EQ(7u, Result.Items); - CHECK_EQ(4u, Result.ExpiredItems); - CHECK_EQ(4u, Result.DeletedItems); - CHECK_EQ(0u, Result.References); - CHECK_EQ(0u, Result.PrunedReferences); - CHECK_EQ(0u, Result.CompactedReferences); - CHECK_GT(Result.RemovedDiskSpace, 0); + .SkipCidDelete = true, + .Verbose = true}); + CHECK_EQ(7u, Result.ReferencerStat.Count); + CHECK_EQ(4u, Result.ReferencerStat.Expired); + CHECK_EQ(4u, Result.ReferencerStat.Deleted); + CHECK_EQ(0u, Result.ReferenceStoreStat.Count); + CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted); + CHECK_GT(Result.RemovedDisk, 0); uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() + CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize(); CHECK_EQ(MemoryClean, Result.RemovedMemory); @@ -2370,35 +2397,36 @@ TEST_CASE("z$.newgc.basics") *JobQueue, TempDir.Path() / "cache", {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}}); + CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); auto Attachments = CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187}); IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments); - Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::minutes(2)); + Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::hours(2)); - Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::minutes(2)); + Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::hours(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2)); - Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2)); + Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2)); - GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::minutes(1), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1), + GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1), + .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1), .CollectSmallObjects = true, .IsDeleteMode = true, - .SkipCidDelete = false}); - CHECK_EQ(8u, Result.Items); - CHECK_EQ(1u, Result.ExpiredItems); - CHECK_EQ(1u, Result.DeletedItems); - CHECK_EQ(9u, Result.References); - CHECK_EQ(4u, Result.PrunedReferences); - CHECK_EQ(4u, Result.CompactedReferences); - CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(), - Result.RemovedDiskSpace); + .SkipCidDelete = false, + .Verbose = true}); + CHECK_EQ(8u, Result.ReferencerStat.Count); + CHECK_EQ(1u, Result.ReferencerStat.Expired); + CHECK_EQ(1u, Result.ReferencerStat.Deleted); + CHECK_EQ(9u, Result.ReferenceStoreStat.Count); + CHECK_EQ(4u, Result.ReferenceStoreStat.Pruned); + CHECK_EQ(4u, Result.ReferenceStoreStat.Compacted); + CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(), Result.RemovedDisk); uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize(); CHECK_EQ(MemoryClean, Result.RemovedMemory); } diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 274876123..7da2f7695 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -3036,8 +3036,14 @@ ProjectStore::AreDiskWritesAllowed() const return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); } +std::string +ProjectStore::GetGcName(GcCtx&) +{ + return fmt::format("projectstore:'{}'", m_ProjectBasePath.string()); +} + void -ProjectStore::RemoveExpiredData(GcCtx& Ctx) +ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) { size_t ProjectCount = 0; size_t ExpiredProjectCount = 0; @@ -3045,13 +3051,18 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx) size_t ExpiredOplogCount = 0; Stopwatch Timer; const auto _ = MakeGuard([&] { - ZEN_DEBUG("gc project store '{}': removed {} expired projects out of {}, {} expired oplogs out of {} in {}", - m_ProjectBasePath, - ExpiredProjectCount, - ProjectCount, - ExpiredOplogCount, - OplogCount, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}", + m_ProjectBasePath, + Stats.Count, + Stats.Expired, + Stats.Deleted, + NiceBytes(Stats.RemovedDisk), + NiceBytes(Stats.RemovedMemory), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); std::vector<Ref<Project>> ExpiredProjects; @@ -3097,14 +3108,14 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx) OplogId, Project->Identifier); Project->DeleteOplog(OplogId); - Ctx.RemovedDiskSpace.fetch_add(OplogSize); + Stats.RemovedDisk += OplogSize; } - Ctx.DeletedItems.fetch_add(ExpiredOplogs.size()); + Stats.Deleted += ExpiredOplogs.size(); Project->Flush(); } } ProjectCount = Projects.size(); - Ctx.Items.fetch_add(ProjectCount + OplogCount); + Stats.Count += ProjectCount + OplogCount; ExpiredProjectCount = ExpiredProjects.size(); if (ExpiredProjects.empty()) @@ -3150,20 +3161,34 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx) DeleteDirectories(PathToRemove); } - Ctx.DeletedItems.fetch_add(ExpiredProjects.size()); + Stats.Deleted += ExpiredProjects.size(); } - Ctx.ExpiredItems.fetch_add(ExpiredOplogCount + ExpiredProjectCount); + Stats.Expired += ExpiredOplogCount + ExpiredProjectCount; } class ProjectStoreReferenceChecker : public GcReferenceChecker { public: - ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner) + ProjectStoreReferenceChecker(GcCtx& Ctx, ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner) { if (PreCache) { - RwLock::SharedLockScope _(m_Oplog.m_OplogLock); + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': precached {} references in {} from {}/{}", + m_Oplog.m_BasePath, + m_UncachedReferences.size(), + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + m_Oplog.m_OuterProject->Identifier, + m_Oplog.OplogId()); + }); + + RwLock::SharedLockScope __(m_Oplog.m_OplogLock); m_Oplog.IterateOplog([&](CbObjectView Op) { Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); }); }); @@ -3173,16 +3198,20 @@ public: virtual ~ProjectStoreReferenceChecker() {} - virtual void LockState(GcCtx&) override + virtual void LockState(GcCtx& Ctx) override { Stopwatch Timer; const auto _ = MakeGuard([&] { - ZEN_DEBUG("gc project oplog '{}': found {} references in {} from {}/{}", - m_Oplog.m_BasePath, - m_UncachedReferences.size(), - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - m_Oplog.m_OuterProject->Identifier, - m_Oplog.OplogId()); + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}", + m_Oplog.m_BasePath, + m_UncachedReferences.size(), + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + m_Oplog.m_OuterProject->Identifier, + m_Oplog.OplogId()); }); m_OplogLock = std::make_unique<RwLock::SharedLockScope>(m_Oplog.m_OplogLock); @@ -3211,18 +3240,22 @@ public: }; std::vector<GcReferenceChecker*> -ProjectStore::CreateReferenceCheckers(GcCtx&) +ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) { size_t ProjectCount = 0; size_t OplogCount = 0; Stopwatch Timer; const auto _ = MakeGuard([&] { - ZEN_DEBUG("gc project store '{}': opened {} projects and {} oplogs in {}", - m_ProjectBasePath, - ProjectCount, - OplogCount, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: projectstore [CREATE CHECKERS] '{}': opened {} projects and {} oplogs in {}", + m_ProjectBasePath, + ProjectCount, + OplogCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); DiscoverProjects(); @@ -3250,7 +3283,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx&) ProjectStore::Oplog* Oplog = Project->OpenOplog(OpLogId); GcClock::TimePoint Now = GcClock::Now(); bool TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5)); - Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache)); + Checkers.emplace_back(new ProjectStoreReferenceChecker(Ctx, *Oplog, TryPreCache)); } OplogCount += OpLogs.size(); } diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 94e697278..fbff1444d 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -293,7 +293,8 @@ public: virtual void CollectGarbage(GcContext& GcCtx) override; virtual GcStorageSize StorageSize() const override; - virtual void RemoveExpiredData(GcCtx& Ctx) override; + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override; virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; CbArray GetProjectsList(); |