diff options
| author | Dan Engelbrecht <[email protected]> | 2023-11-21 16:58:43 +0100 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-11-21 16:58:43 +0100 |
| commit | 669a8869b5414c0e8708dd90b1f4aa297d091887 (patch) | |
| tree | c81c309cf3013e3129caeefb5c3eaba2aaed6967 /src/zenstore/gc.cpp | |
| parent | fixed paths in ue_build_linux/README.md (diff) | |
| download | zen-669a8869b5414c0e8708dd90b1f4aa297d091887.tar.xz zen-669a8869b5414c0e8708dd90b1f4aa297d091887.zip | |
add command line options for compact block threshold and gc verbose (#557)
- Feature: Added new options to zenserver for GC V2
- `--gc-compactblock-threshold` GCV2 - how much of a compact block should be used to skip compacting the block, default is 90%
- `--gc-verbose` GCV2 - enable more verbose output when running a GC pass
- Feature: Added new options to `zen gc` command for GC V2
- `--compactblockthreshold` GCV2 - how much of a compact block should be used to skip compacting the block, default is 90%
- `--verbose` GCV2 - enable more verbose output when running a GC pass
- Feature: Added new parameters for endpoint `admin/gc` (PUT)
- `compactblockthreshold` GCV2 - how much of a compact block should be used to skip compacting the block, default is 90%
- `verbose` GCV2 - enable more verbose output when running a GC pass
Diffstat (limited to 'src/zenstore/gc.cpp')
| -rw-r--r-- | src/zenstore/gc.cpp | 356 |
1 files changed, 210 insertions, 146 deletions
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index b78b23350..64958cdea 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -620,27 +620,34 @@ GcManager::CollectGarbage(const GcSettings& Settings) if (!m_GcReferencers.empty()) { Latch WorkLeft(1); - // First remove any cache keys that may own references - SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - for (size_t Index = 0; Index < m_GcReferencers.size(); Index++) { - GcReferencer* Owner = m_GcReferencers[Index]; - std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index]; - WorkLeft.AddCount(1); - ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Owner, &Stats, &StoreCompactorsLock, &StoreCompactors]() { - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - Stats.first = Owner->GetGcName(Ctx); - SCOPED_TIMER(Stats.second.RemoveExpiredDataStats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - std::unique_ptr<GcStoreCompactor> StoreCompactor(Owner->RemoveExpiredData(Ctx, Stats.second.RemoveExpiredDataStats)); - if (StoreCompactor) - { - RwLock::ExclusiveLockScope __(StoreCompactorsLock); - StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.second.CompactStoreStats); - } + // First remove any cache keys that may own references + SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) { + ZEN_INFO("GCV2: Removed epxired data for {} referenceners in {}", + m_GcReferencers.size(), + NiceTimeSpanMs(Result.RemoveExpiredDataMS.count())); }); + for (size_t Index = 0; Index < m_GcReferencers.size(); Index++) + { + GcReferencer* Owner = m_GcReferencers[Index]; + std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index]; + WorkLeft.AddCount(1); + ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Owner, &Stats, &StoreCompactorsLock, &StoreCompactors]() { + auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); + Stats.first = Owner->GetGcName(Ctx); + SCOPED_TIMER(Stats.second.RemoveExpiredDataStats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + std::unique_ptr<GcStoreCompactor> StoreCompactor( + Owner->RemoveExpiredData(Ctx, Stats.second.RemoveExpiredDataStats)); + if (StoreCompactor) + { + RwLock::ExclusiveLockScope __(StoreCompactorsLock); + StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.second.CompactStoreStats); + } + }); + } + WorkLeft.CountDown(); + WorkLeft.Wait(); } - WorkLeft.CountDown(); - WorkLeft.Wait(); } if (!Ctx.Settings.SkipCidDelete) @@ -654,31 +661,42 @@ GcManager::CollectGarbage(const GcSettings& Settings) ReferencePruners.reserve(m_GcReferenceStores.size()); Latch WorkLeft(1); RwLock ReferencePrunersLock; - // CreateReferencePruner is usually not very heavy but big data sets change that - SCOPED_TIMER(Result.CreateReferencePrunersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++) { - GcReferenceStore* ReferenceStore = m_GcReferenceStores[Index]; - std::pair<std::string, GcReferenceStoreStats>& Stats = Result.ReferenceStoreStats[Index]; - WorkLeft.AddCount(1); - ThreadPool.ScheduleWork([&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() { - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - Stats.first = ReferenceStore->GetGcName(Ctx); - std::unique_ptr<GcReferencePruner> ReferencePruner; - { - SCOPED_TIMER(Stats.second.CreateReferencePrunersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - // The ReferenceStore will pick a list of CId entries to check, returning a collector - ReferencePruner = std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second)); - } - if (ReferencePruner) - { - RwLock::ExclusiveLockScope __(ReferencePrunersLock); - ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner)); - } - }); + // CreateReferencePruner is usually not very heavy but big data sets change that + SCOPED_TIMER(Result.CreateReferencePrunersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); + if (Ctx.Settings.Verbose) { + ZEN_INFO("GCV2: Created {} reference pruners using {} referencer stores in {}", + ReferencePruners.size(), + m_GcReferenceStores.size(), + NiceTimeSpanMs(Result.CreateReferencePrunersMS.count())); + }); + for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++) + { + GcReferenceStore* ReferenceStore = m_GcReferenceStores[Index]; + std::pair<std::string, GcReferenceStoreStats>& Stats = Result.ReferenceStoreStats[Index]; + WorkLeft.AddCount(1); + ThreadPool.ScheduleWork( + [&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() { + auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); + Stats.first = ReferenceStore->GetGcName(Ctx); + std::unique_ptr<GcReferencePruner> ReferencePruner; + { + SCOPED_TIMER(Stats.second.CreateReferencePrunersMS = + std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + // The ReferenceStore will pick a list of CId entries to check, returning a collector + ReferencePruner = + std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second)); + } + if (ReferencePruner) + { + RwLock::ExclusiveLockScope __(ReferencePrunersLock); + ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner)); + } + }); + } + WorkLeft.CountDown(); + WorkLeft.Wait(); } - WorkLeft.CountDown(); - WorkLeft.Wait(); } if (!ReferencePruners.empty()) @@ -690,47 +708,57 @@ GcManager::CollectGarbage(const GcSettings& Settings) ReferenceCheckers.reserve(m_GcReferencers.size()); Latch WorkLeft(1); RwLock ReferenceCheckersLock; - SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - // Lock all reference owners from changing the reference data and get access to check for referenced data - for (size_t Index = 0; Index < m_GcReferencers.size(); Index++) { - GcReferencer* Referencer = m_GcReferencers[Index]; - std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index]; - WorkLeft.AddCount(1); - ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() { - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - // The Referencer will create a reference checker that guarrantees that the references do not change as long as - // it lives - std::vector<GcReferenceChecker*> Checkers; - { - SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - Checkers = Referencer->CreateReferenceCheckers(Ctx); - } - try - { - if (!Checkers.empty()) - { - RwLock::ExclusiveLockScope __(ReferenceCheckersLock); - for (auto& Checker : Checkers) + SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); + if (Ctx.Settings.Verbose) { + ZEN_INFO("GCV2: Created {} reference checkers using {} referencers in {}", + ReferenceCheckers.size(), + m_GcReferencers.size(), + NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count())); + }); + // Lock all reference owners from changing the reference data and get access to check for referenced data + for (size_t Index = 0; Index < m_GcReferencers.size(); Index++) + { + GcReferencer* Referencer = m_GcReferencers[Index]; + std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index]; + WorkLeft.AddCount(1); + ThreadPool.ScheduleWork( + [&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() { + auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); + // The Referencer will create a reference checker that guarrantees that the references do not change as + // long as it lives + std::vector<GcReferenceChecker*> Checkers; + { + SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS = + std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + Checkers = Referencer->CreateReferenceCheckers(Ctx); + } + try { - ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index); - Checker = nullptr; + if (!Checkers.empty()) + { + RwLock::ExclusiveLockScope __(ReferenceCheckersLock); + for (auto& Checker : Checkers) + { + ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index); + Checker = nullptr; + } + } } - } - } - catch (std::exception&) - { - while (!Checkers.empty()) - { - delete Checkers.back(); - Checkers.pop_back(); - } - throw; - } - }); + catch (std::exception&) + { + while (!Checkers.empty()) + { + delete Checkers.back(); + Checkers.pop_back(); + } + throw; + } + }); + } + WorkLeft.CountDown(); + WorkLeft.Wait(); } - WorkLeft.CountDown(); - WorkLeft.Wait(); } ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size()); @@ -744,21 +772,28 @@ GcManager::CollectGarbage(const GcSettings& Settings) // we delete the ReferenceCheckers Latch WorkLeft(1); - SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - for (auto& It : ReferenceCheckers) { - GcReferenceChecker* Checker = It.first.get(); - size_t Index = It.second; - std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index]; - WorkLeft.AddCount(1); - ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() { - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - Checker->LockState(Ctx); - }); + SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); + if (Ctx.Settings.Verbose) { + ZEN_INFO("GCV2: Locked state using {} reference checkers in {}", + ReferenceCheckers.size(), + NiceTimeSpanMs(Result.LockStateMS.count())); + }); + for (auto& It : ReferenceCheckers) + { + GcReferenceChecker* Checker = It.first.get(); + size_t Index = It.second; + std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index]; + WorkLeft.AddCount(1); + ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() { + auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); + SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + Checker->LockState(Ctx); + }); + } + WorkLeft.CountDown(); + WorkLeft.Wait(); } - WorkLeft.CountDown(); - WorkLeft.Wait(); } ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size()); @@ -783,34 +818,43 @@ GcManager::CollectGarbage(const GcSettings& Settings) Latch WorkLeft(1); - SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - for (auto& It : ReferencePruners) { - GcReferencePruner* Pruner = It.second.get(); - size_t Index = It.first; - GcReferenceStoreStats& Stats = Result.ReferenceStoreStats[Index].second; - WorkLeft.AddCount(1); - ThreadPool.ScheduleWork( - [&Ctx, Pruner, &Stats, &WorkLeft, &GetUnusedReferences, &StoreCompactorsLock, &StoreCompactors]() { - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced - // or not. - std::unique_ptr<GcStoreCompactor> StoreCompactor; - { - SCOPED_TIMER(Stats.RemoveUnreferencedDataStats.ElapsedMS = - std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - StoreCompactor = std::unique_ptr<GcStoreCompactor>( - Pruner->RemoveUnreferencedData(Ctx, Stats.RemoveUnreferencedDataStats, GetUnusedReferences)); - } - if (StoreCompactor) - { - RwLock::ExclusiveLockScope __(StoreCompactorsLock); - StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.CompactStoreStats); - } - }); + SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); + if (Ctx.Settings.Verbose) { + ZEN_INFO("GCV2: Removed unused data using {} pruners in {}", + ReferencePruners.size(), + NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count())); + }); + for (auto& It : ReferencePruners) + { + GcReferencePruner* Pruner = It.second.get(); + size_t Index = It.first; + GcReferenceStoreStats& Stats = Result.ReferenceStoreStats[Index].second; + WorkLeft.AddCount(1); + ThreadPool.ScheduleWork( + [&Ctx, Pruner, &Stats, &WorkLeft, &GetUnusedReferences, &StoreCompactorsLock, &StoreCompactors]() { + auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); + // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are + // referenced or not. + std::unique_ptr<GcStoreCompactor> StoreCompactor; + { + SCOPED_TIMER(Stats.RemoveUnreferencedDataStats.ElapsedMS = + std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + StoreCompactor = std::unique_ptr<GcStoreCompactor>( + Pruner->RemoveUnreferencedData(Ctx, + Stats.RemoveUnreferencedDataStats, + GetUnusedReferences)); + } + if (StoreCompactor) + { + RwLock::ExclusiveLockScope __(StoreCompactorsLock); + StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.CompactStoreStats); + } + }); + } + WorkLeft.CountDown(); + WorkLeft.Wait(); } - WorkLeft.CountDown(); - WorkLeft.Wait(); } // Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed ReferenceCheckers.clear(); @@ -819,7 +863,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) } } - ZEN_INFO("GCV2: Compacting reference stores for {} store compactors", StoreCompactors.size()); + ZEN_INFO("GCV2: Compacting using {} store compactors", StoreCompactors.size()); if (!StoreCompactors.empty()) { auto ClaimDiskReserve = [&]() -> uint64_t { @@ -836,15 +880,19 @@ GcManager::CollectGarbage(const GcSettings& Settings) }; // Remove the stuff we deemed unreferenced from disk - may be heavy operation // Don't do in parallel, we don't want to steal CPU/Disk from regular operation - SCOPED_TIMER(Result.CompactStoresMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - for (auto& It : StoreCompactors) { - GcStoreCompactor* Compactor = It.first.get(); - GcCompactStoreStats& Stats = *It.second; + SCOPED_TIMER(Result.CompactStoresMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) { + ZEN_INFO("GCV2: Compacted {} stores in {}", StoreCompactors.size(), NiceTimeSpanMs(Result.CompactStoresMS.count())); + }); + for (auto& It : StoreCompactors) { - // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not. - SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve); + GcStoreCompactor* Compactor = It.first.get(); + GcCompactStoreStats& Stats = *It.second; + { + // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not. + SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve); + } } } StoreCompactors.clear(); @@ -1445,18 +1493,20 @@ GcScheduler::SchedulerThread() try { - bool DoGc = m_Config.Enabled; - bool DoScrubbing = false; - std::chrono::seconds ScrubTimeslice = std::chrono::seconds::max(); - bool DoDelete = true; - bool CollectSmallObjects = m_Config.CollectSmallObjects; - std::chrono::seconds GcInterval = m_Config.Interval; - std::chrono::seconds LightweightGcInterval = m_Config.LightweightInterval; - std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration; - std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration; - uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit; - bool SkipCid = false; - GcVersion UseGCVersion = m_Config.UseGCVersion; + bool DoGc = m_Config.Enabled; + bool DoScrubbing = false; + std::chrono::seconds ScrubTimeslice = std::chrono::seconds::max(); + bool DoDelete = true; + bool CollectSmallObjects = m_Config.CollectSmallObjects; + std::chrono::seconds GcInterval = m_Config.Interval; + std::chrono::seconds LightweightGcInterval = m_Config.LightweightInterval; + std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration; + std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration; + uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit; + bool SkipCid = false; + GcVersion UseGCVersion = m_Config.UseGCVersion; + uint32_t CompactBlockUsageThresholdPercent = m_Config.CompactBlockUsageThresholdPercent; + bool Verbose = m_Config.Verbose; bool DiskSpaceGCTriggered = false; bool TimeBasedGCTriggered = false; @@ -1491,7 +1541,10 @@ GcScheduler::SchedulerThread() DoDelete = false; } UseGCVersion = TriggerParams.ForceGCVersion.value_or(UseGCVersion); - DoGc = true; + CompactBlockUsageThresholdPercent = + TriggerParams.CompactBlockUsageThresholdPercent.value_or(CompactBlockUsageThresholdPercent); + Verbose = TriggerParams.Verbose.value_or(Verbose); + DoGc = true; } if (m_TriggerScrubParams) @@ -1696,7 +1749,14 @@ GcScheduler::SchedulerThread() } } - CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, DoDelete, CollectSmallObjects, SkipCid, UseGCVersion); + CollectGarbage(CacheExpireTime, + ProjectStoreExpireTime, + DoDelete, + CollectSmallObjects, + SkipCid, + UseGCVersion, + CompactBlockUsageThresholdPercent, + Verbose); uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning); if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle))) @@ -1778,7 +1838,9 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, bool Delete, bool CollectSmallObjects, bool SkipCid, - GcVersion UseGCVersion) + GcVersion UseGCVersion, + uint32_t CompactBlockUsageThresholdPercent, + bool Verbose) { ZEN_TRACE_CPU("GcScheduler::CollectGarbage"); @@ -1841,12 +1903,14 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, break; case GcVersion::kV2: { - const GcSettings Settings = {.CacheExpireTime = CacheExpireTime, - .ProjectStoreExpireTime = ProjectStoreExpireTime, - .CollectSmallObjects = CollectSmallObjects, - .IsDeleteMode = Delete, - .SkipCidDelete = SkipCid, - .DiskReservePath = m_Config.RootDirectory / "reserve.gc"}; + const GcSettings Settings = {.CacheExpireTime = CacheExpireTime, + .ProjectStoreExpireTime = ProjectStoreExpireTime, + .CollectSmallObjects = CollectSmallObjects, + .IsDeleteMode = Delete, + .SkipCidDelete = SkipCid, + .Verbose = Verbose, + .CompactBlockUsageThresholdPercent = CompactBlockUsageThresholdPercent, + .DiskReservePath = m_Config.RootDirectory / "reserve.gc"}; GcClock::TimePoint GcStartTime = GcClock::Now(); GcResult Result = m_GcManager.CollectGarbage(Settings); |