diff options
| author | Dan Engelbrecht <[email protected]> | 2022-12-02 16:56:51 +0100 |
|---|---|---|
| committer | GitHub <[email protected]> | 2022-12-02 07:56:51 -0800 |
| commit | 921078b38bfa91424c27ff707d950e26c18d3cd3 (patch) | |
| tree | e71446e3ebc29f7ff8ad47c0c23d8b2e49211dbe | |
| parent | reduce gc log spam (INFO -> DEBUG) (#199) (diff) | |
| download | zen-921078b38bfa91424c27ff707d950e26c18d3cd3.tar.xz zen-921078b38bfa91424c27ff707d950e26c18d3cd3.zip | |
Size based gc trigger (#197)
- Feature: Disk size triggered GC, a soft disk usage limit for cache data.
- Feature: New option `--gc-disk-size-soft-limit` (command line), `gc.cache.disksizesoftlimit` (lua config) controlling limit for soft disk usage limit. Defaults to zero which disables soft disk usage limit.
- Improvement: Disk write pressure in GC log and cleaned up clutter in GC logging.
| -rw-r--r-- | CHANGELOG.md | 4 | ||||
| -rw-r--r-- | zenserver/admin/admin.cpp | 8 | ||||
| -rw-r--r-- | zenserver/cache/structuredcachestore.cpp | 28 | ||||
| -rw-r--r-- | zenserver/config.cpp | 9 | ||||
| -rw-r--r-- | zenserver/config.h | 1 | ||||
| -rw-r--r-- | zenserver/projectstore.cpp | 6 | ||||
| -rw-r--r-- | zenserver/zenserver.cpp | 17 | ||||
| -rw-r--r-- | zenstore/compactcas.cpp | 22 | ||||
| -rw-r--r-- | zenstore/filecas.cpp | 6 | ||||
| -rw-r--r-- | zenstore/gc.cpp | 555 | ||||
| -rw-r--r-- | zenstore/include/zenstore/gc.h | 39 |
11 files changed, 554 insertions, 141 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 792e3a4c9..2e21096ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## +- Feature: Disk size triggered GC, a soft disk usage limit for cache data. +- Feature: New option `--gc-disk-size-soft-limit` (command line), `gc.cache.disksizesoftlimit` (lua config) controlling limit for soft disk usage limit. Defaults to zero which disables soft disk usage limit. +- Improvement: Disk write pressure in GC log and cleaned up clutter in GC logging. - Bugfix: Always store records or oplog entries before storing attachments to avoid GC finding unreferenced chunks i CidStore +- Changed: Reduced GC `INFO` spam by converting to `DEBUG` log messages ## 0.1.9 - Feature: Adds two command to Zen command tool to export/import project store oplogs with attachments diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp index 676e9a830..7aa1b48d1 100644 --- a/zenserver/admin/admin.cpp +++ b/zenserver/admin/admin.cpp @@ -54,6 +54,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler) : m_GcScheduler(Sched } } + if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + GcParams.DiskSizeSoftLimit = Value.value(); + } + } + const bool Started = m_GcScheduler.Trigger(GcParams); CbObjectWriter Response; diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 91ae452a8..c20e40655 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1213,8 +1213,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx) NiceLatencyNs(ReadBlockLongestTimeUs)); }); - const GcClock::TimePoint ExpireTime = - GcCtx.MaxCacheDuration() == GcClock::Duration::max() ? GcClock::TimePoint::min() : GcCtx.Time() - GcCtx.MaxCacheDuration(); + const GcClock::TimePoint ExpireTime = GcCtx.ExpireTime(); const GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count(); @@ -1310,7 +1309,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) uint64_t MovedCount = 0; const auto _ = MakeGuard([&] { - ZEN_INFO( + ZEN_DEBUG( "garbage collect from '{}' DONE after {}, write lock: {} ({}), read lock: {} ({}), collected {} bytes, deleted #{} and moved " "#{} " "of #{} " @@ -2379,8 +2378,7 @@ TEST_CASE("z$.gc") GcClock::Duration MaxDuration, std::span<const IoHash> Cids, std::vector<IoHash>& OutKeep) { - GcContext GcCtx(Time); - GcCtx.MaxCacheDuration(MaxDuration); + GcContext GcCtx(Time - MaxDuration); Gc.CollectGarbage(GcCtx); OutKeep.clear(); GcCtx.FilterCids(Cids, [&OutKeep](const IoHash& Hash) { OutKeep.push_back(Hash); }); @@ -2459,8 +2457,7 @@ TEST_CASE("z$.gc") } { - GcContext GcCtx; - GcCtx.MaxCacheDuration(std::chrono::hours(46)); + GcContext GcCtx(CurrentTime - std::chrono::hours(46)); Gc.CollectGarbage(GcCtx); @@ -2474,9 +2471,7 @@ TEST_CASE("z$.gc") // Move forward in time and collect again { - GcContext GcCtx(CurrentTime + std::chrono::hours(46)); - GcCtx.MaxCacheDuration(std::chrono::minutes(2)); - + GcContext GcCtx(CurrentTime + std::chrono::minutes(2)); Gc.CollectGarbage(GcCtx); for (const auto& Key : Keys) @@ -2495,8 +2490,7 @@ TEST_CASE("z$.gc") ScopedTemporaryDirectory TempDir; GcManager Gc; ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache"); - const auto Bucket = "rightintwo"sv; - const GcClock::TimePoint CurrentTime = GcClock::Now(); + const auto Bucket = "rightintwo"sv; std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)}; @@ -2507,8 +2501,7 @@ TEST_CASE("z$.gc") } { - GcContext GcCtx; - GcCtx.MaxCacheDuration(std::chrono::hours(2)); + GcContext GcCtx(GcClock::Now() - std::chrono::hours(2)); GcCtx.CollectSmallObjects(true); Gc.CollectGarbage(GcCtx); @@ -2523,8 +2516,7 @@ TEST_CASE("z$.gc") // Move forward in time and collect again { - GcContext GcCtx(CurrentTime + std::chrono::hours(2)); - GcCtx.MaxCacheDuration(std::chrono::minutes(2)); + GcContext GcCtx(GcClock::Now() + std::chrono::minutes(2)); GcCtx.CollectSmallObjects(true); Zcs.Flush(); @@ -2713,7 +2705,7 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true)) C++; } - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); GcCtx.AddRetainedCids(KeepHashes); Zcs.CollectGarbage(GcCtx); @@ -2761,7 +2753,7 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true)) C++; } - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); GcCtx.AddRetainedCids(KeepHashes); Zcs.CollectGarbage(GcCtx); diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 9531a5251..057b26c5b 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -493,6 +493,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "Garbage collection monitoring interval in seconds.", cxxopts::value<int32_t>(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), ""); + + options.add_option("gc", + "", + "gc-disksize-softlimit", + "Garbage collection disk usage soft limit. Default set to 0 (Off).", + cxxopts::value<uint64_t>(ServerOptions.GcConfig.Cache.DiskSizeSoftLimit)->default_value("0"), + ""); + try { auto result = options.parse(argc, argv); @@ -825,6 +833,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio ServerOptions.GcConfig.Cache.MaxDurationSeconds = CacheGcConfig.value().get_or("maxdurationseconds", int32_t(0)); ServerOptions.GcConfig.Cache.DiskSizeLimit = CacheGcConfig.value().get_or("disksizelimit", ~uint64_t(0)); ServerOptions.GcConfig.Cache.MemorySizeLimit = CacheGcConfig.value().get_or("memorysizelimit", ~uint64_t(0)); + ServerOptions.GcConfig.Cache.DiskSizeSoftLimit = CacheGcConfig.value().get_or("disksizesoftlimit", 0); } if (sol::optional<sol::table> CasGcConfig = GcConfig.value()["cas"]) diff --git a/zenserver/config.h b/zenserver/config.h index 5dbca4c41..4cdef0318 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -76,6 +76,7 @@ struct ZenCacheEvictionPolicy uint64_t DiskSizeLimit = ~uint64_t(0); uint64_t MemorySizeLimit = 1024 * 1024 * 1024; int32_t MaxDurationSeconds = 24 * 60 * 60; + uint64_t DiskSizeSoftLimit = 0; bool Enabled = true; }; diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index c3f10c9cd..3a65feb0f 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -1156,7 +1156,7 @@ ProjectStore::CollectGarbage(GcContext& GcCtx) if (!GcCtx.IsDeletionMode()) { - ZEN_INFO("garbage collect DISABLED, for '{}' ", m_ProjectBasePath.string()); + ZEN_DEBUG("garbage collect DISABLED, for '{}' ", m_ProjectBasePath.string()); return; } @@ -2756,7 +2756,7 @@ TEST_CASE("project.store.gc") } { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); ProjectStore.GatherReferences(GcCtx); size_t RefCount = 0; GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); @@ -2769,7 +2769,7 @@ TEST_CASE("project.store.gc") std::filesystem::remove(Project1FilePath); { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); ProjectStore.GatherReferences(GcCtx); size_t RefCount = 0; GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index cabff7389..cf3d90324 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -357,15 +357,14 @@ public: } ZEN_INFO("initializing GC, enabled '{}', interval {}s", ServerOptions.GcConfig.Enabled, ServerOptions.GcConfig.IntervalSeconds); - zen::GcSchedulerConfig GcConfig{ - .RootDirectory = m_DataRoot / "gc", - .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds), - .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), - .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), - .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, - .Enabled = ServerOptions.GcConfig.Enabled, - .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, - }; + zen::GcSchedulerConfig GcConfig{.RootDirectory = m_DataRoot / "gc", + .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds), + .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), + .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), + .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, + .Enabled = ServerOptions.GcConfig.Enabled, + .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, + .DiskSizeSoftLimit = ServerOptions.GcConfig.Cache.DiskSizeSoftLimit}; m_GcScheduler.Initialize(GcConfig); return EffectiveBasePort; diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index e52f5b7cb..7507a82f6 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -903,7 +903,7 @@ TEST_CASE("compactcas.gc.basic") CHECK(InsertResult.New); Cas.Flush(); - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); Cas.CollectGarbage(GcCtx); @@ -933,7 +933,7 @@ TEST_CASE("compactcas.gc.removefile") CasContainerStrategy Cas(Gc); Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, false); - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); Cas.CollectGarbage(GcCtx); @@ -988,7 +988,7 @@ TEST_CASE("compactcas.gc.compact") // Keep first and last { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; @@ -1023,7 +1023,7 @@ TEST_CASE("compactcas.gc.compact") // Keep last { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; KeepChunks.push_back(ChunkHashes[8]); @@ -1055,7 +1055,7 @@ TEST_CASE("compactcas.gc.compact") // Keep mixed { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; KeepChunks.push_back(ChunkHashes[1]); @@ -1090,7 +1090,7 @@ TEST_CASE("compactcas.gc.compact") // Keep multiple at end { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; KeepChunks.push_back(ChunkHashes[6]); @@ -1125,7 +1125,7 @@ TEST_CASE("compactcas.gc.compact") // Keep every other { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; KeepChunks.push_back(ChunkHashes[0]); @@ -1204,7 +1204,7 @@ TEST_CASE("compactcas.gc.deleteblockonopen") // GC every other block { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; for (size_t i = 0; i < 20; i += 2) @@ -1271,7 +1271,7 @@ TEST_CASE("compactcas.gc.handleopeniobuffer") Cas.Flush(); // GC everything - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); Cas.CollectGarbage(GcCtx); @@ -1427,7 +1427,7 @@ TEST_CASE("compactcas.threadedinsert") C++; } - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); GcCtx.AddRetainedCids(KeepHashes); Cas.CollectGarbage(GcCtx); @@ -1468,7 +1468,7 @@ TEST_CASE("compactcas.threadedinsert") C++; } - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); GcCtx.AddRetainedCids(KeepHashes); Cas.CollectGarbage(GcCtx); diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index 1d3542f62..38d7898cf 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -912,7 +912,7 @@ FileCasStrategy::CollectGarbage(GcContext& GcCtx) if (GcCtx.IsDeletionMode() == false) { - ZEN_INFO("NOTE: not actually deleting anything since deletion is disabled"); + ZEN_DEBUG("NOTE: not actually deleting anything since deletion is disabled"); return; } @@ -1050,7 +1050,7 @@ TEST_CASE("cas.file.gc") { InsertChunks(); - GcContext Ctx; + GcContext Ctx(GcClock::Now() - std::chrono::hours(24)); FileCas.CollectGarbage(Ctx); for (const IoHash& Key : Keys) @@ -1066,7 +1066,7 @@ TEST_CASE("cas.file.gc") { InsertChunks(); - GcContext Ctx; + GcContext Ctx(GcClock::Now() - std::chrono::hours(24)); for (const IoHash& Key : Keys) { diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 8aac65bb4..4094716ae 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -36,6 +36,19 @@ # include <random> #endif +template<> +struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx) + { + std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint); + zen::ExtendableStringBuilder<32> String; + String << std::ctime(&Time); + return formatter<string_view>::format(String.ToView(), ctx); + } +}; + namespace zen { using namespace std::literals; @@ -176,17 +189,16 @@ struct GcContext::GcState CacheKeyContexts m_ExpiredCacheKeys; HashKeySet m_RetainedCids; HashKeySet m_DeletedCids; - GcClock::TimePoint m_GcTime; - GcClock::Duration m_MaxCacheDuration = std::chrono::hours(24); + GcClock::TimePoint m_ExpireTime; bool m_DeletionMode = true; bool m_CollectSmallObjects = false; std::filesystem::path DiskReservePath; }; -GcContext::GcContext(GcClock::TimePoint Time) : m_State(std::make_unique<GcState>()) +GcContext::GcContext(const GcClock::TimePoint& ExpireTime) : m_State(std::make_unique<GcState>()) { - m_State->m_GcTime = Time; + m_State->m_ExpireTime = ExpireTime; } GcContext::~GcContext() @@ -266,21 +278,9 @@ GcContext::CollectSmallObjects(bool NewState) } GcClock::TimePoint -GcContext::Time() const -{ - return m_State->m_GcTime; -} - -GcClock::Duration -GcContext::MaxCacheDuration() const -{ - return m_State->m_MaxCacheDuration; -} - -void -GcContext::MaxCacheDuration(GcClock::Duration Duration) +GcContext::ExpireTime() const { - m_State->m_MaxCacheDuration = Duration; + return m_State->m_ExpireTime; } void @@ -330,7 +330,7 @@ GcStorage::~GcStorage() ////////////////////////////////////////////////////////////////////////// -GcManager::GcManager() +GcManager::GcManager() : m_Log(logging::Get("gc")) { } @@ -383,13 +383,22 @@ GcManager::CollectGarbage(GcContext& GcCtx) } // Then trim storage - { - Stopwatch Timer; - const auto Guard = MakeGuard([&] { ZEN_INFO("collected garbage in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + GcStorageSize GCTotalSizeDiff; + Stopwatch Timer; + const auto Guard = MakeGuard([&] { + ZEN_INFO("collected garbage in {}. Removed {} disk space, {} memory", + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + NiceBytes(GCTotalSizeDiff.DiskSize), + NiceBytes(GCTotalSizeDiff.MemorySize)); + }); for (GcStorage* Storage : m_GcStorage) { + const auto PreSize = Storage->StorageSize(); Storage->CollectGarbage(GcCtx); + const auto PostSize = Storage->StorageSize(); + GCTotalSizeDiff.DiskSize += PreSize.DiskSize > PostSize.DiskSize ? PreSize.DiskSize - PostSize.DiskSize : 0; + GCTotalSizeDiff.MemorySize += PreSize.MemorySize > PostSize.MemorySize ? PreSize.MemorySize - PostSize.MemorySize : 0; } } } @@ -432,6 +441,121 @@ GcManager::OnDroppedCidReferences(std::span<IoHash> Hashes) #endif ////////////////////////////////////////////////////////////////////////// +void +DiskUsageWindow::KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick) +{ + auto It = m_LogWindow.begin(); + if (It == m_LogWindow.end()) + { + return; + } + while (It->SampleTime < StartTick) + { + ++It; + if (It == m_LogWindow.end()) + { + m_LogWindow.clear(); + return; + } + } + m_LogWindow.erase(m_LogWindow.begin(), It); + + It = m_LogWindow.begin(); + while (It != m_LogWindow.end()) + { + if (It->SampleTime >= EndTick) + { + m_LogWindow.erase(It, m_LogWindow.end()); + return; + } + It++; + } +} + +std::vector<uint64_t> +DiskUsageWindow::GetDiskDeltas(GcClock::Tick StartTick, GcClock::Tick EndTick, GcClock::Tick DeltaWidth, uint64_t& OutMaxDelta) const +{ + ZEN_ASSERT(StartTick != -1); + ZEN_ASSERT(DeltaWidth > 0); + + std::vector<uint64_t> Result; + Result.reserve((EndTick - StartTick + DeltaWidth - 1) / DeltaWidth); + + size_t WindowSize = m_LogWindow.size(); + GcClock::Tick FirstWindowTick = WindowSize < 2 ? EndTick : m_LogWindow[1].SampleTime; + + GcClock::Tick RangeStart = StartTick; + while (FirstWindowTick >= RangeStart + DeltaWidth && RangeStart < EndTick) + { + Result.push_back(0); + RangeStart += DeltaWidth; + } + + uint64_t DeltaSum = 0; + size_t WindowIndex = 1; + while (WindowIndex < WindowSize && RangeStart < EndTick) + { + const DiskUsageEntry& Entry = m_LogWindow[WindowIndex]; + if (Entry.SampleTime < RangeStart) + { + ++WindowIndex; + continue; + } + GcClock::Tick RangeEnd = Min(EndTick, RangeStart + DeltaWidth); + ZEN_ASSERT(Entry.SampleTime >= RangeStart); + if (Entry.SampleTime >= RangeEnd) + { + Result.push_back(DeltaSum); + OutMaxDelta = Max(DeltaSum, OutMaxDelta); + DeltaSum = 0; + RangeStart = RangeEnd; + continue; + } + const DiskUsageEntry& PrevEntry = m_LogWindow[WindowIndex - 1]; + if (Entry.DiskUsage > PrevEntry.DiskUsage) + { + uint64_t Delta = Entry.DiskUsage - PrevEntry.DiskUsage; + DeltaSum += Delta; + } + WindowIndex++; + } + + while (RangeStart < EndTick) + { + Result.push_back(DeltaSum); + OutMaxDelta = Max(DeltaSum, OutMaxDelta); + DeltaSum = 0; + RangeStart += DeltaWidth; + } + return Result; +} + +GcClock::Tick +DiskUsageWindow::FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const +{ + ZEN_ASSERT(Amount > 0); + uint64_t RemainingToFind = Amount; + size_t Offset = 1; + while (Offset < m_LogWindow.size()) + { + const DiskUsageEntry& Entry = m_LogWindow[Offset]; + if (Entry.SampleTime >= EndTick) + { + return EndTick; + } + const DiskUsageEntry& PreviousEntry = m_LogWindow[Offset - 1]; + uint64_t Delta = Entry.DiskUsage > PreviousEntry.DiskUsage ? Entry.DiskUsage - PreviousEntry.DiskUsage : 0; + if (Delta >= RemainingToFind) + { + return m_LogWindow[Offset].SampleTime + 1; + } + RemainingToFind -= Delta; + Offset++; + } + return EndTick; +} + +////////////////////////////////////////////////////////////////////////// GcScheduler::GcScheduler(GcManager& GcManager) : m_Log(logging::Get("gc")), m_GcManager(GcManager) { @@ -465,12 +589,14 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) Ec.message()); } - m_LastGcTime = GcClock::Now(); + m_LastGcTime = GcClock::Now(); + m_LastGcExpireTime = GcClock::TimePoint::min(); if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state")) { m_LastGcTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcTime"sv].AsInt64())); - + m_LastGcExpireTime = + GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcExpireTime"].AsInt64(GcClock::Duration::min().count()))); if (m_LastGcTime + m_Config.Interval < GcClock::Now()) { // TODO: Trigger GC? @@ -478,6 +604,18 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) } } + m_DiskUsageLog.Open(m_Config.RootDirectory / "gc.dlog", CasLogFile::Mode::kWrite); + m_DiskUsageLog.Initialize(); + const GcClock::Tick LastGCTick = m_LastGcTime.time_since_epoch().count(); + m_DiskUsageLog.Replay( + [this, LastGCTick](const DiskUsageWindow::DiskUsageEntry& Entry) { + if (Entry.SampleTime >= m_LastGcExpireTime.time_since_epoch().count()) + { + m_DiskUsageWindow.Append(Entry); + } + }, + 0); + m_NextGcTime = NextGcTime(m_LastGcTime); m_GcThread = std::thread(&GcScheduler::SchedulerThread, this); } @@ -500,6 +638,8 @@ GcScheduler::Shutdown() m_GcThread.join(); } } + m_DiskUsageLog.Flush(); + m_DiskUsageLog.Close(); } bool @@ -547,34 +687,107 @@ GcScheduler::SchedulerThread() continue; } + bool Delete = true; + bool CollectSmallObjects = m_Config.CollectSmallObjects; + std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration; + uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit; + GcClock::TimePoint Now = GcClock::Now(); + if (m_TriggerParams) + { + const auto TriggerParams = m_TriggerParams.value(); + m_TriggerParams.reset(); + + CollectSmallObjects = TriggerParams.CollectSmallObjects; + if (TriggerParams.MaxCacheDuration != std::chrono::seconds::max()) + { + MaxCacheDuration = TriggerParams.MaxCacheDuration; + } + if (TriggerParams.DiskSizeSoftLimit != 0) + { + DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit; + } + } + + GcClock::TimePoint ExpireTime = MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration; + + std::error_code Ec; + const GcStorageSize TotalSize = m_GcManager.TotalStorageSize(); + if (Timeout && Status() == GcSchedulerStatus::kIdle) { - std::error_code Ec; - DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Ec); - GcStorageSize TotalSize = m_GcManager.TotalStorageSize(); + DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Ec); + if (Ec) + { + ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message()); + } + std::chrono::seconds RemaingTime = std::chrono::duration_cast<std::chrono::seconds>(m_NextGcTime - GcClock::Now()); + const int64_t PressureGraphLength = 30; + const std::chrono::duration LoadGraphTime = PressureGraphLength * m_Config.MonitorInterval; + std::vector<uint64_t> DiskDeltas; + uint64_t MaxLoad = 0; + { + const GcClock::Tick EpochTickCount = GcClock::Now().time_since_epoch().count(); + std::unique_lock Lock(m_GcMutex); + m_DiskUsageWindow.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize}); + m_DiskUsageLog.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize}); + const GcClock::TimePoint LoadGraphStartTime = Now - LoadGraphTime; + GcClock::Tick Start = LoadGraphStartTime.time_since_epoch().count(); + GcClock::Tick End = Now.time_since_epoch().count(); + DiskDeltas = m_DiskUsageWindow.GetDiskDeltas(Start, + End, + Max(1, (End - Start + PressureGraphLength - 1) / PressureGraphLength), + MaxLoad); + } + if (RemaingTime < std::chrono::seconds::zero()) { RemaingTime = std::chrono::seconds::zero(); } - if (Ec) + std::string LoadGraph; + LoadGraph.resize(DiskDeltas.size(), '0'); + if (DiskDeltas.size() > 0 && MaxLoad > 0) { - ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message()); + char LoadIndicator[11] = "0123456789"; + for (size_t Index = 0; Index < DiskDeltas.size(); ++Index) + { + size_t LoadIndex = (9 * DiskDeltas[Index] + MaxLoad - 1) / MaxLoad; + LoadGraph[Index] = LoadIndicator[LoadIndex]; + } + } + + uint64_t GcDiskSpaceGoal = 0; + if (DiskSizeSoftLimit != 0 && TotalSize.DiskSize > DiskSizeSoftLimit) + { + GcDiskSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit; + std::unique_lock Lock(m_GcMutex); + GcClock::Tick AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count()); + GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick); + if (SizeBasedExpireTime > ExpireTime) + { + ExpireTime = SizeBasedExpireTime; + } } - ZEN_INFO("{} in use, {} of total {} free disk space, {}", + bool DiskSpaceGCTriggered = GcDiskSpaceGoal > 0; + bool TimeBasedGCTriggered = !DiskSpaceGCTriggered && RemaingTime.count() == 0; + ZEN_INFO("{} in use,{} {} of total {} free disk space, disk writes last {} per {} [{}], peak {}/s. {}", NiceBytes(TotalSize.DiskSize), + DiskSizeSoftLimit == 0 ? "" : fmt::format(" {} soft limit,", NiceBytes(DiskSizeSoftLimit)), NiceBytes(Space.Free), NiceBytes(Space.Total), - m_Config.Interval.count() - ? fmt::format("{} until next GC", NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count()))) - : std::string("no GC scheduled")); - - // TODO: Trigger GC if max disk usage water mark is reached - - if (RemaingTime.count() > 0) + NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count())), + NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count() / PressureGraphLength)), + LoadGraph, + NiceBytes(MaxLoad * uint64_t(std::chrono::seconds(1).count()) / uint64_t(std::chrono::seconds(LoadGraphTime).count())), + DiskSpaceGCTriggered ? fmt::format("Disk use threshold triggered, trying to reclaim {}. ", NiceBytes(GcDiskSpaceGoal)) + : TimeBasedGCTriggered ? "GC schedule triggered." + : fmt::format("{} until next scheduled GC.", + NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count())))); + + if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered) { WaitTime = m_Config.MonitorInterval < RemaingTime ? m_Config.MonitorInterval : RemaingTime; continue; @@ -588,54 +801,7 @@ GcScheduler::SchedulerThread() } } - GcContext GcCtx; - GcCtx.SetDeletionMode(true); - GcCtx.CollectSmallObjects(m_Config.CollectSmallObjects); - GcCtx.MaxCacheDuration(m_Config.MaxCacheDuration); - GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc"); - - if (m_TriggerParams) - { - const auto TriggerParams = m_TriggerParams.value(); - m_TriggerParams.reset(); - - GcCtx.CollectSmallObjects(TriggerParams.CollectSmallObjects); - if (TriggerParams.MaxCacheDuration != std::chrono::seconds::max()) - { - GcCtx.MaxCacheDuration(TriggerParams.MaxCacheDuration); - } - } - - ZEN_INFO("garbage collection STARTING, small objects gc {}, max cache duration {}", - GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv, - NiceTimeSpanMs(uint64_t(std::chrono::duration_cast<std::chrono::milliseconds>(GcCtx.MaxCacheDuration()).count()))); - { - Stopwatch Timer; - const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - - m_GcManager.CollectGarbage(GcCtx); - - m_LastGcTime = GcClock::Now(); - m_NextGcTime = NextGcTime(m_LastGcTime); - WaitTime = m_Config.MonitorInterval; - - { - const fs::path Path = m_Config.RootDirectory / "gc_state"; - ZEN_DEBUG("saving scheduler state to '{}'", Path); - CbObjectWriter SchedulderState; - SchedulderState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count()); - SaveCompactBinaryObject(Path, SchedulderState.Save()); - } - - std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); - if (Ec) - { - ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'", - m_Config.RootDirectory / "reserve.gc", - NiceBytes(m_Config.DiskReserveSize), - Ec.message()); - } - } + CollectGarbage(ExpireTime, Delete, CollectSmallObjects); uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning); if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle))) @@ -643,6 +809,8 @@ GcScheduler::SchedulerThread() ZEN_ASSERT(m_Status == static_cast<uint32_t>(GcSchedulerStatus::kStopped)); break; } + + WaitTime = m_Config.MonitorInterval; } } @@ -659,6 +827,54 @@ GcScheduler::NextGcTime(GcClock::TimePoint CurrentTime) } } +void +GcScheduler::CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, bool CollectSmallObjects) +{ + GcContext GcCtx(ExpireTime); + GcCtx.SetDeletionMode(Delete); + GcCtx.CollectSmallObjects(CollectSmallObjects); + // GcCtx.MaxCacheDuration(MaxCacheDuration); + GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc"); + + ZEN_INFO("garbage collection STARTING, small objects gc {}, cutoff time {}", + GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv, + ExpireTime); + { + Stopwatch Timer; + const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + + m_GcManager.CollectGarbage(GcCtx); + + if (Delete) + { + m_LastGcExpireTime = ExpireTime; + std::unique_lock Lock(m_GcMutex); + m_DiskUsageWindow.KeepRange(ExpireTime.time_since_epoch().count(), GcClock::Duration::max().count()); + } + + m_LastGcTime = GcClock::Now(); + m_NextGcTime = NextGcTime(m_LastGcTime); + + { + const fs::path Path = m_Config.RootDirectory / "gc_state"; + ZEN_DEBUG("saving scheduler state to '{}'", Path); + CbObjectWriter SchedulerState; + SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count()); + SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count()); + SaveCompactBinaryObject(Path, SchedulerState.Save()); + } + + std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); + if (Ec) + { + ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'", + m_Config.RootDirectory / "reserve.gc", + NiceBytes(m_Config.DiskReserveSize), + Ec.message()); + } + } +} + ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS @@ -704,7 +920,7 @@ TEST_CASE("gc.basic") const auto InsertResult = CidStore.AddChunk(CompressedChunk); CHECK(InsertResult.New); - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); CidStore.Flush(); @@ -761,7 +977,7 @@ TEST_CASE("gc.full") // Keep first and last { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; @@ -796,7 +1012,7 @@ TEST_CASE("gc.full") // Keep last { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; KeepChunks.push_back(ChunkHashes[8]); @@ -828,7 +1044,7 @@ TEST_CASE("gc.full") // Keep mixed { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; KeepChunks.push_back(ChunkHashes[1]); @@ -863,7 +1079,7 @@ TEST_CASE("gc.full") // Keep multiple at end { - GcContext GcCtx; + GcContext GcCtx(GcClock::Now() - std::chrono::hours(24)); GcCtx.CollectSmallObjects(true); std::vector<IoHash> KeepChunks; KeepChunks.push_back(ChunkHashes[6]); @@ -912,6 +1128,165 @@ TEST_CASE("gc.full") CHECK_LE(InitialSize.TinySize, FinalSize.TinySize); CHECK_GE(InitialSize.TinySize + (1u << 28), FinalSize.TinySize); } + +TEST_CASE("gc.diskusagewindow") +{ + DiskUsageWindow Stats; + Stats.Append({.SampleTime = 0, .DiskUsage = 0}); // 0 0 + Stats.Append({.SampleTime = 10, .DiskUsage = 10}); // 1 10 + Stats.Append({.SampleTime = 20, .DiskUsage = 20}); // 2 10 + Stats.Append({.SampleTime = 30, .DiskUsage = 20}); // 3 0 + Stats.Append({.SampleTime = 40, .DiskUsage = 15}); // 4 0 + Stats.Append({.SampleTime = 50, .DiskUsage = 25}); // 5 10 + Stats.Append({.SampleTime = 60, .DiskUsage = 30}); // 6 5 + Stats.Append({.SampleTime = 70, .DiskUsage = 45}); // 7 15 + + SUBCASE("Truncate start") + { + Stats.KeepRange(-15, 31); + CHECK(Stats.m_LogWindow.size() == 4); + CHECK(Stats.m_LogWindow[0].SampleTime == 0); + CHECK(Stats.m_LogWindow[3].SampleTime == 30); + } + + SUBCASE("Truncate end") + { + Stats.KeepRange(70, 71); + CHECK(Stats.m_LogWindow.size() == 1); + CHECK(Stats.m_LogWindow[0].SampleTime == 70); + } + + SUBCASE("Truncate middle") + { + Stats.KeepRange(29, 69); + CHECK(Stats.m_LogWindow.size() == 4); + CHECK(Stats.m_LogWindow[0].SampleTime == 30); + CHECK(Stats.m_LogWindow[3].SampleTime == 60); + } + + SUBCASE("Full range") + { + uint64_t MaxDelta = 0; + // 0-10, 10-20, 20-30, 30-40, 40-50, 50-60, 60-70, 70-80 + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 10, MaxDelta); + CHECK(DiskDeltas.size() == 8); + CHECK(MaxDelta == 15); + CHECK(DiskDeltas[0] == 0); + CHECK(DiskDeltas[1] == 10); + CHECK(DiskDeltas[2] == 10); + CHECK(DiskDeltas[3] == 0); + CHECK(DiskDeltas[4] == 0); + CHECK(DiskDeltas[5] == 10); + CHECK(DiskDeltas[6] == 5); + CHECK(DiskDeltas[7] == 15); + } + + SUBCASE("Sub range") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(20, 40, 10, MaxDelta); + CHECK(DiskDeltas.size() == 2); + CHECK(MaxDelta == 10); + CHECK(DiskDeltas[0] == 10); // [20:30] + CHECK(DiskDeltas[1] == 0); // [30:40] + } + SUBCASE("Unaligned sub range 1") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(21, 51, 10, MaxDelta); + CHECK(DiskDeltas.size() == 3); + CHECK(MaxDelta == 10); + CHECK(DiskDeltas[0] == 0); // [21:31] + CHECK(DiskDeltas[1] == 0); // [31:41] + CHECK(DiskDeltas[2] == 10); // [41:51] + } + SUBCASE("Unaligned end range") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(29, 79, 10, MaxDelta); + CHECK(DiskDeltas.size() == 5); + CHECK(MaxDelta == 15); + CHECK(DiskDeltas[0] == 0); // [29:39] + CHECK(DiskDeltas[1] == 0); // [39:49] + CHECK(DiskDeltas[2] == 10); // [49:59] + CHECK(DiskDeltas[3] == 5); // [59:69] + CHECK(DiskDeltas[4] == 15); // [69:79] + } + SUBCASE("Ahead of window") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-40, 0, 10, MaxDelta); + CHECK(DiskDeltas.size() == 4); + CHECK(MaxDelta == 0); + CHECK(DiskDeltas[0] == 0); // [-40:-30] + CHECK(DiskDeltas[1] == 0); // [-30:-20] + CHECK(DiskDeltas[2] == 0); // [-20:-10] + CHECK(DiskDeltas[3] == 0); // [-10:0] + } + SUBCASE("After of window") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(90, 120, 10, MaxDelta); + CHECK(DiskDeltas.size() == 3); + CHECK(MaxDelta == 0); + CHECK(DiskDeltas[0] == 0); // [90:100] + CHECK(DiskDeltas[1] == 0); // [100:110] + CHECK(DiskDeltas[2] == 0); // [110:120] + } + SUBCASE("Encapsulating window") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-20, 100, 10, MaxDelta); + CHECK(DiskDeltas.size() == 12); + CHECK(MaxDelta == 15); + CHECK(DiskDeltas[0] == 0); // [-20:-10] + CHECK(DiskDeltas[1] == 0); // [ -10:0] + CHECK(DiskDeltas[2] == 0); // [0:10] + CHECK(DiskDeltas[3] == 10); // [10:20] + CHECK(DiskDeltas[4] == 10); // [20:30] + CHECK(DiskDeltas[5] == 0); // [30:40] + CHECK(DiskDeltas[6] == 0); // [40:50] + CHECK(DiskDeltas[7] == 10); // [50:60] + CHECK(DiskDeltas[8] == 5); // [60:70] + CHECK(DiskDeltas[9] == 15); // [70:80] + CHECK(DiskDeltas[10] == 0); // [80:90] + CHECK(DiskDeltas[11] == 0); // [90:100] + } + + SUBCASE("Full range half stride") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 20, MaxDelta); + CHECK(DiskDeltas.size() == 4); + CHECK(MaxDelta == 20); + CHECK(DiskDeltas[0] == 10); // [0:20] + CHECK(DiskDeltas[1] == 10); // [20:40] + CHECK(DiskDeltas[2] == 10); // [40:60] + CHECK(DiskDeltas[3] == 20); // [60:80] + } + + SUBCASE("Partial odd stride") + { + uint64_t MaxDelta = 0; + std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(13, 67, 18, MaxDelta); + CHECK(DiskDeltas.size() == 3); + CHECK(MaxDelta == 15); + CHECK(DiskDeltas[0] == 10); // [13:31] + CHECK(DiskDeltas[1] == 0); // [31:49] + CHECK(DiskDeltas[2] == 15); // [49:67] + } + + SUBCASE("Find size window") + { + DiskUsageWindow Empty; + CHECK(Empty.FindTimepointThatRemoves(15u, 10000) == 10000); + + CHECK(Stats.FindTimepointThatRemoves(15u, 40) == 21); + CHECK(Stats.FindTimepointThatRemoves(15u, 20) == 20); + CHECK(Stats.FindTimepointThatRemoves(100000u, 50) == 50); + CHECK(Stats.FindTimepointThatRemoves(100000u, 1000)); + } +} #endif void diff --git a/zenstore/include/zenstore/gc.h b/zenstore/include/zenstore/gc.h index 656e594af..e0354b331 100644 --- a/zenstore/include/zenstore/gc.h +++ b/zenstore/include/zenstore/gc.h @@ -4,6 +4,7 @@ #include <zencore/iohash.h> #include <zencore/thread.h> +#include <zenstore/caslog.h> #include <atomic> #include <chrono> @@ -47,7 +48,7 @@ public: class GcContext { public: - GcContext(GcClock::TimePoint Time = GcClock::Now()); + GcContext(const GcClock::TimePoint& ExpireTime); ~GcContext(); void AddRetainedCids(std::span<const IoHash> Cid); @@ -69,16 +70,11 @@ public: bool CollectSmallObjects() const; void CollectSmallObjects(bool NewState); - GcClock::TimePoint Time() const; - - GcClock::Duration MaxCacheDuration() const; - void MaxCacheDuration(GcClock::Duration Duration); + GcClock::TimePoint ExpireTime() const; void DiskReservePath(const std::filesystem::path& Path); uint64_t ClaimGCReserve(); - inline bool Expired(GcClock::Tick TickCount) { return Time() - GcClock::TimePointFromTick(TickCount) > MaxCacheDuration(); } - private: struct GcState; @@ -150,6 +146,8 @@ public: #endif private: + spdlog::logger& Log() { return m_Log; } + spdlog::logger& m_Log; mutable RwLock m_Lock; std::vector<GcContributor*> m_GcContribs; std::vector<GcStorage*> m_GcStorage; @@ -172,6 +170,27 @@ struct GcSchedulerConfig bool CollectSmallObjects = true; bool Enabled = true; uint64_t DiskReserveSize = 1ul << 28; + uint64_t DiskSizeSoftLimit = 0; +}; + +class DiskUsageWindow +{ +public: + struct DiskUsageEntry + { + GcClock::Tick SampleTime; + uint64_t DiskUsage; + }; + + std::vector<DiskUsageEntry> m_LogWindow; + inline void Append(const DiskUsageEntry& Entry) { m_LogWindow.push_back(Entry); } + inline void Append(DiskUsageEntry&& Entry) { m_LogWindow.emplace_back(std::move(Entry)); } + void KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick); + std::vector<uint64_t> GetDiskDeltas(GcClock::Tick StartTick, + GcClock::Tick EndTick, + GcClock::Tick DeltaWidth, + uint64_t& OutMaxDelta) const; + GcClock::Tick FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const; }; /** @@ -191,12 +210,14 @@ public: { bool CollectSmallObjects = false; std::chrono::seconds MaxCacheDuration = std::chrono::seconds::max(); + uint64_t DiskSizeSoftLimit = 0; }; bool Trigger(const TriggerParams& Params); private: void SchedulerThread(); + void CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, bool CollectSmallObjects); GcClock::TimePoint NextGcTime(GcClock::TimePoint CurrentTime); spdlog::logger& Log() { return m_Log; } @@ -204,12 +225,16 @@ private: GcManager& m_GcManager; GcSchedulerConfig m_Config; GcClock::TimePoint m_LastGcTime{}; + GcClock::TimePoint m_LastGcExpireTime{}; GcClock::TimePoint m_NextGcTime{}; std::atomic_uint32_t m_Status{}; std::thread m_GcThread; std::mutex m_GcMutex; std::condition_variable m_GcSignal; std::optional<TriggerParams> m_TriggerParams; + + TCasLogFile<DiskUsageWindow::DiskUsageEntry> m_DiskUsageLog; + DiskUsageWindow m_DiskUsageWindow; }; void gc_forcelink(); |