aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-12-02 16:56:51 +0100
committerGitHub <[email protected]>2022-12-02 07:56:51 -0800
commit921078b38bfa91424c27ff707d950e26c18d3cd3 (patch)
treee71446e3ebc29f7ff8ad47c0c23d8b2e49211dbe
parentreduce gc log spam (INFO -> DEBUG) (#199) (diff)
downloadzen-921078b38bfa91424c27ff707d950e26c18d3cd3.tar.xz
zen-921078b38bfa91424c27ff707d950e26c18d3cd3.zip
Size based gc trigger (#197)
- Feature: Disk size triggered GC, a soft disk usage limit for cache data. - Feature: New option `--gc-disk-size-soft-limit` (command line), `gc.cache.disksizesoftlimit` (lua config) controlling limit for soft disk usage limit. Defaults to zero which disables soft disk usage limit. - Improvement: Disk write pressure in GC log and cleaned up clutter in GC logging.
-rw-r--r--CHANGELOG.md4
-rw-r--r--zenserver/admin/admin.cpp8
-rw-r--r--zenserver/cache/structuredcachestore.cpp28
-rw-r--r--zenserver/config.cpp9
-rw-r--r--zenserver/config.h1
-rw-r--r--zenserver/projectstore.cpp6
-rw-r--r--zenserver/zenserver.cpp17
-rw-r--r--zenstore/compactcas.cpp22
-rw-r--r--zenstore/filecas.cpp6
-rw-r--r--zenstore/gc.cpp555
-rw-r--r--zenstore/include/zenstore/gc.h39
11 files changed, 554 insertions, 141 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 792e3a4c9..2e21096ed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
##
+- Feature: Disk size triggered GC, a soft disk usage limit for cache data.
+- Feature: New option `--gc-disk-size-soft-limit` (command line), `gc.cache.disksizesoftlimit` (lua config) controlling limit for soft disk usage limit. Defaults to zero which disables soft disk usage limit.
+- Improvement: Disk write pressure in GC log and cleaned up clutter in GC logging.
- Bugfix: Always store records or oplog entries before storing attachments to avoid GC finding unreferenced chunks i CidStore
+- Changed: Reduced GC `INFO` spam by converting to `DEBUG` log messages
## 0.1.9
- Feature: Adds two command to Zen command tool to export/import project store oplogs with attachments
diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp
index 676e9a830..7aa1b48d1 100644
--- a/zenserver/admin/admin.cpp
+++ b/zenserver/admin/admin.cpp
@@ -54,6 +54,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler) : m_GcScheduler(Sched
}
}
+ if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false)
+ {
+ if (auto Value = ParseInt<uint64_t>(Param))
+ {
+ GcParams.DiskSizeSoftLimit = Value.value();
+ }
+ }
+
const bool Started = m_GcScheduler.Trigger(GcParams);
CbObjectWriter Response;
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 91ae452a8..c20e40655 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -1213,8 +1213,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
NiceLatencyNs(ReadBlockLongestTimeUs));
});
- const GcClock::TimePoint ExpireTime =
- GcCtx.MaxCacheDuration() == GcClock::Duration::max() ? GcClock::TimePoint::min() : GcCtx.Time() - GcCtx.MaxCacheDuration();
+ const GcClock::TimePoint ExpireTime = GcCtx.ExpireTime();
const GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
@@ -1310,7 +1309,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
uint64_t MovedCount = 0;
const auto _ = MakeGuard([&] {
- ZEN_INFO(
+ ZEN_DEBUG(
"garbage collect from '{}' DONE after {}, write lock: {} ({}), read lock: {} ({}), collected {} bytes, deleted #{} and moved "
"#{} "
"of #{} "
@@ -2379,8 +2378,7 @@ TEST_CASE("z$.gc")
GcClock::Duration MaxDuration,
std::span<const IoHash> Cids,
std::vector<IoHash>& OutKeep) {
- GcContext GcCtx(Time);
- GcCtx.MaxCacheDuration(MaxDuration);
+ GcContext GcCtx(Time - MaxDuration);
Gc.CollectGarbage(GcCtx);
OutKeep.clear();
GcCtx.FilterCids(Cids, [&OutKeep](const IoHash& Hash) { OutKeep.push_back(Hash); });
@@ -2459,8 +2457,7 @@ TEST_CASE("z$.gc")
}
{
- GcContext GcCtx;
- GcCtx.MaxCacheDuration(std::chrono::hours(46));
+ GcContext GcCtx(CurrentTime - std::chrono::hours(46));
Gc.CollectGarbage(GcCtx);
@@ -2474,9 +2471,7 @@ TEST_CASE("z$.gc")
// Move forward in time and collect again
{
- GcContext GcCtx(CurrentTime + std::chrono::hours(46));
- GcCtx.MaxCacheDuration(std::chrono::minutes(2));
-
+ GcContext GcCtx(CurrentTime + std::chrono::minutes(2));
Gc.CollectGarbage(GcCtx);
for (const auto& Key : Keys)
@@ -2495,8 +2490,7 @@ TEST_CASE("z$.gc")
ScopedTemporaryDirectory TempDir;
GcManager Gc;
ZenCacheNamespace Zcs(Gc, TempDir.Path() / "cache");
- const auto Bucket = "rightintwo"sv;
- const GcClock::TimePoint CurrentTime = GcClock::Now();
+ const auto Bucket = "rightintwo"sv;
std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
@@ -2507,8 +2501,7 @@ TEST_CASE("z$.gc")
}
{
- GcContext GcCtx;
- GcCtx.MaxCacheDuration(std::chrono::hours(2));
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(2));
GcCtx.CollectSmallObjects(true);
Gc.CollectGarbage(GcCtx);
@@ -2523,8 +2516,7 @@ TEST_CASE("z$.gc")
// Move forward in time and collect again
{
- GcContext GcCtx(CurrentTime + std::chrono::hours(2));
- GcCtx.MaxCacheDuration(std::chrono::minutes(2));
+ GcContext GcCtx(GcClock::Now() + std::chrono::minutes(2));
GcCtx.CollectSmallObjects(true);
Zcs.Flush();
@@ -2713,7 +2705,7 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true))
C++;
}
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
GcCtx.AddRetainedCids(KeepHashes);
Zcs.CollectGarbage(GcCtx);
@@ -2761,7 +2753,7 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true))
C++;
}
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
GcCtx.AddRetainedCids(KeepHashes);
Zcs.CollectGarbage(GcCtx);
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index 9531a5251..057b26c5b 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -493,6 +493,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
"Garbage collection monitoring interval in seconds.",
cxxopts::value<int32_t>(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"),
"");
+
+ options.add_option("gc",
+ "",
+ "gc-disksize-softlimit",
+ "Garbage collection disk usage soft limit. Default set to 0 (Off).",
+ cxxopts::value<uint64_t>(ServerOptions.GcConfig.Cache.DiskSizeSoftLimit)->default_value("0"),
+ "");
+
try
{
auto result = options.parse(argc, argv);
@@ -825,6 +833,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio
ServerOptions.GcConfig.Cache.MaxDurationSeconds = CacheGcConfig.value().get_or("maxdurationseconds", int32_t(0));
ServerOptions.GcConfig.Cache.DiskSizeLimit = CacheGcConfig.value().get_or("disksizelimit", ~uint64_t(0));
ServerOptions.GcConfig.Cache.MemorySizeLimit = CacheGcConfig.value().get_or("memorysizelimit", ~uint64_t(0));
+ ServerOptions.GcConfig.Cache.DiskSizeSoftLimit = CacheGcConfig.value().get_or("disksizesoftlimit", 0);
}
if (sol::optional<sol::table> CasGcConfig = GcConfig.value()["cas"])
diff --git a/zenserver/config.h b/zenserver/config.h
index 5dbca4c41..4cdef0318 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -76,6 +76,7 @@ struct ZenCacheEvictionPolicy
uint64_t DiskSizeLimit = ~uint64_t(0);
uint64_t MemorySizeLimit = 1024 * 1024 * 1024;
int32_t MaxDurationSeconds = 24 * 60 * 60;
+ uint64_t DiskSizeSoftLimit = 0;
bool Enabled = true;
};
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index c3f10c9cd..3a65feb0f 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -1156,7 +1156,7 @@ ProjectStore::CollectGarbage(GcContext& GcCtx)
if (!GcCtx.IsDeletionMode())
{
- ZEN_INFO("garbage collect DISABLED, for '{}' ", m_ProjectBasePath.string());
+ ZEN_DEBUG("garbage collect DISABLED, for '{}' ", m_ProjectBasePath.string());
return;
}
@@ -2756,7 +2756,7 @@ TEST_CASE("project.store.gc")
}
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
ProjectStore.GatherReferences(GcCtx);
size_t RefCount = 0;
GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
@@ -2769,7 +2769,7 @@ TEST_CASE("project.store.gc")
std::filesystem::remove(Project1FilePath);
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
ProjectStore.GatherReferences(GcCtx);
size_t RefCount = 0;
GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index cabff7389..cf3d90324 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -357,15 +357,14 @@ public:
}
ZEN_INFO("initializing GC, enabled '{}', interval {}s", ServerOptions.GcConfig.Enabled, ServerOptions.GcConfig.IntervalSeconds);
- zen::GcSchedulerConfig GcConfig{
- .RootDirectory = m_DataRoot / "gc",
- .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds),
- .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds),
- .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds),
- .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects,
- .Enabled = ServerOptions.GcConfig.Enabled,
- .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize,
- };
+ zen::GcSchedulerConfig GcConfig{.RootDirectory = m_DataRoot / "gc",
+ .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds),
+ .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds),
+ .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds),
+ .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects,
+ .Enabled = ServerOptions.GcConfig.Enabled,
+ .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize,
+ .DiskSizeSoftLimit = ServerOptions.GcConfig.Cache.DiskSizeSoftLimit};
m_GcScheduler.Initialize(GcConfig);
return EffectiveBasePort;
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index e52f5b7cb..7507a82f6 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -903,7 +903,7 @@ TEST_CASE("compactcas.gc.basic")
CHECK(InsertResult.New);
Cas.Flush();
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
Cas.CollectGarbage(GcCtx);
@@ -933,7 +933,7 @@ TEST_CASE("compactcas.gc.removefile")
CasContainerStrategy Cas(Gc);
Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, false);
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
Cas.CollectGarbage(GcCtx);
@@ -988,7 +988,7 @@ TEST_CASE("compactcas.gc.compact")
// Keep first and last
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
@@ -1023,7 +1023,7 @@ TEST_CASE("compactcas.gc.compact")
// Keep last
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
KeepChunks.push_back(ChunkHashes[8]);
@@ -1055,7 +1055,7 @@ TEST_CASE("compactcas.gc.compact")
// Keep mixed
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
KeepChunks.push_back(ChunkHashes[1]);
@@ -1090,7 +1090,7 @@ TEST_CASE("compactcas.gc.compact")
// Keep multiple at end
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
KeepChunks.push_back(ChunkHashes[6]);
@@ -1125,7 +1125,7 @@ TEST_CASE("compactcas.gc.compact")
// Keep every other
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
KeepChunks.push_back(ChunkHashes[0]);
@@ -1204,7 +1204,7 @@ TEST_CASE("compactcas.gc.deleteblockonopen")
// GC every other block
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
for (size_t i = 0; i < 20; i += 2)
@@ -1271,7 +1271,7 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
Cas.Flush();
// GC everything
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
Cas.CollectGarbage(GcCtx);
@@ -1427,7 +1427,7 @@ TEST_CASE("compactcas.threadedinsert")
C++;
}
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
GcCtx.AddRetainedCids(KeepHashes);
Cas.CollectGarbage(GcCtx);
@@ -1468,7 +1468,7 @@ TEST_CASE("compactcas.threadedinsert")
C++;
}
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
GcCtx.AddRetainedCids(KeepHashes);
Cas.CollectGarbage(GcCtx);
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 1d3542f62..38d7898cf 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -912,7 +912,7 @@ FileCasStrategy::CollectGarbage(GcContext& GcCtx)
if (GcCtx.IsDeletionMode() == false)
{
- ZEN_INFO("NOTE: not actually deleting anything since deletion is disabled");
+ ZEN_DEBUG("NOTE: not actually deleting anything since deletion is disabled");
return;
}
@@ -1050,7 +1050,7 @@ TEST_CASE("cas.file.gc")
{
InsertChunks();
- GcContext Ctx;
+ GcContext Ctx(GcClock::Now() - std::chrono::hours(24));
FileCas.CollectGarbage(Ctx);
for (const IoHash& Key : Keys)
@@ -1066,7 +1066,7 @@ TEST_CASE("cas.file.gc")
{
InsertChunks();
- GcContext Ctx;
+ GcContext Ctx(GcClock::Now() - std::chrono::hours(24));
for (const IoHash& Key : Keys)
{
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index 8aac65bb4..4094716ae 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -36,6 +36,19 @@
# include <random>
#endif
+template<>
+struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx)
+ {
+ std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
+ zen::ExtendableStringBuilder<32> String;
+ String << std::ctime(&Time);
+ return formatter<string_view>::format(String.ToView(), ctx);
+ }
+};
+
namespace zen {
using namespace std::literals;
@@ -176,17 +189,16 @@ struct GcContext::GcState
CacheKeyContexts m_ExpiredCacheKeys;
HashKeySet m_RetainedCids;
HashKeySet m_DeletedCids;
- GcClock::TimePoint m_GcTime;
- GcClock::Duration m_MaxCacheDuration = std::chrono::hours(24);
+ GcClock::TimePoint m_ExpireTime;
bool m_DeletionMode = true;
bool m_CollectSmallObjects = false;
std::filesystem::path DiskReservePath;
};
-GcContext::GcContext(GcClock::TimePoint Time) : m_State(std::make_unique<GcState>())
+GcContext::GcContext(const GcClock::TimePoint& ExpireTime) : m_State(std::make_unique<GcState>())
{
- m_State->m_GcTime = Time;
+ m_State->m_ExpireTime = ExpireTime;
}
GcContext::~GcContext()
@@ -266,21 +278,9 @@ GcContext::CollectSmallObjects(bool NewState)
}
GcClock::TimePoint
-GcContext::Time() const
-{
- return m_State->m_GcTime;
-}
-
-GcClock::Duration
-GcContext::MaxCacheDuration() const
-{
- return m_State->m_MaxCacheDuration;
-}
-
-void
-GcContext::MaxCacheDuration(GcClock::Duration Duration)
+GcContext::ExpireTime() const
{
- m_State->m_MaxCacheDuration = Duration;
+ return m_State->m_ExpireTime;
}
void
@@ -330,7 +330,7 @@ GcStorage::~GcStorage()
//////////////////////////////////////////////////////////////////////////
-GcManager::GcManager()
+GcManager::GcManager() : m_Log(logging::Get("gc"))
{
}
@@ -383,13 +383,22 @@ GcManager::CollectGarbage(GcContext& GcCtx)
}
// Then trim storage
-
{
- Stopwatch Timer;
- const auto Guard = MakeGuard([&] { ZEN_INFO("collected garbage in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+ GcStorageSize GCTotalSizeDiff;
+ Stopwatch Timer;
+ const auto Guard = MakeGuard([&] {
+ ZEN_INFO("collected garbage in {}. Removed {} disk space, {} memory",
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ NiceBytes(GCTotalSizeDiff.DiskSize),
+ NiceBytes(GCTotalSizeDiff.MemorySize));
+ });
for (GcStorage* Storage : m_GcStorage)
{
+ const auto PreSize = Storage->StorageSize();
Storage->CollectGarbage(GcCtx);
+ const auto PostSize = Storage->StorageSize();
+ GCTotalSizeDiff.DiskSize += PreSize.DiskSize > PostSize.DiskSize ? PreSize.DiskSize - PostSize.DiskSize : 0;
+ GCTotalSizeDiff.MemorySize += PreSize.MemorySize > PostSize.MemorySize ? PreSize.MemorySize - PostSize.MemorySize : 0;
}
}
}
@@ -432,6 +441,121 @@ GcManager::OnDroppedCidReferences(std::span<IoHash> Hashes)
#endif
//////////////////////////////////////////////////////////////////////////
+void
+DiskUsageWindow::KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick)
+{
+ auto It = m_LogWindow.begin();
+ if (It == m_LogWindow.end())
+ {
+ return;
+ }
+ while (It->SampleTime < StartTick)
+ {
+ ++It;
+ if (It == m_LogWindow.end())
+ {
+ m_LogWindow.clear();
+ return;
+ }
+ }
+ m_LogWindow.erase(m_LogWindow.begin(), It);
+
+ It = m_LogWindow.begin();
+ while (It != m_LogWindow.end())
+ {
+ if (It->SampleTime >= EndTick)
+ {
+ m_LogWindow.erase(It, m_LogWindow.end());
+ return;
+ }
+ It++;
+ }
+}
+
+std::vector<uint64_t>
+DiskUsageWindow::GetDiskDeltas(GcClock::Tick StartTick, GcClock::Tick EndTick, GcClock::Tick DeltaWidth, uint64_t& OutMaxDelta) const
+{
+ ZEN_ASSERT(StartTick != -1);
+ ZEN_ASSERT(DeltaWidth > 0);
+
+ std::vector<uint64_t> Result;
+ Result.reserve((EndTick - StartTick + DeltaWidth - 1) / DeltaWidth);
+
+ size_t WindowSize = m_LogWindow.size();
+ GcClock::Tick FirstWindowTick = WindowSize < 2 ? EndTick : m_LogWindow[1].SampleTime;
+
+ GcClock::Tick RangeStart = StartTick;
+ while (FirstWindowTick >= RangeStart + DeltaWidth && RangeStart < EndTick)
+ {
+ Result.push_back(0);
+ RangeStart += DeltaWidth;
+ }
+
+ uint64_t DeltaSum = 0;
+ size_t WindowIndex = 1;
+ while (WindowIndex < WindowSize && RangeStart < EndTick)
+ {
+ const DiskUsageEntry& Entry = m_LogWindow[WindowIndex];
+ if (Entry.SampleTime < RangeStart)
+ {
+ ++WindowIndex;
+ continue;
+ }
+ GcClock::Tick RangeEnd = Min(EndTick, RangeStart + DeltaWidth);
+ ZEN_ASSERT(Entry.SampleTime >= RangeStart);
+ if (Entry.SampleTime >= RangeEnd)
+ {
+ Result.push_back(DeltaSum);
+ OutMaxDelta = Max(DeltaSum, OutMaxDelta);
+ DeltaSum = 0;
+ RangeStart = RangeEnd;
+ continue;
+ }
+ const DiskUsageEntry& PrevEntry = m_LogWindow[WindowIndex - 1];
+ if (Entry.DiskUsage > PrevEntry.DiskUsage)
+ {
+ uint64_t Delta = Entry.DiskUsage - PrevEntry.DiskUsage;
+ DeltaSum += Delta;
+ }
+ WindowIndex++;
+ }
+
+ while (RangeStart < EndTick)
+ {
+ Result.push_back(DeltaSum);
+ OutMaxDelta = Max(DeltaSum, OutMaxDelta);
+ DeltaSum = 0;
+ RangeStart += DeltaWidth;
+ }
+ return Result;
+}
+
+GcClock::Tick
+DiskUsageWindow::FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const
+{
+ ZEN_ASSERT(Amount > 0);
+ uint64_t RemainingToFind = Amount;
+ size_t Offset = 1;
+ while (Offset < m_LogWindow.size())
+ {
+ const DiskUsageEntry& Entry = m_LogWindow[Offset];
+ if (Entry.SampleTime >= EndTick)
+ {
+ return EndTick;
+ }
+ const DiskUsageEntry& PreviousEntry = m_LogWindow[Offset - 1];
+ uint64_t Delta = Entry.DiskUsage > PreviousEntry.DiskUsage ? Entry.DiskUsage - PreviousEntry.DiskUsage : 0;
+ if (Delta >= RemainingToFind)
+ {
+ return m_LogWindow[Offset].SampleTime + 1;
+ }
+ RemainingToFind -= Delta;
+ Offset++;
+ }
+ return EndTick;
+}
+
+//////////////////////////////////////////////////////////////////////////
GcScheduler::GcScheduler(GcManager& GcManager) : m_Log(logging::Get("gc")), m_GcManager(GcManager)
{
@@ -465,12 +589,14 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
Ec.message());
}
- m_LastGcTime = GcClock::Now();
+ m_LastGcTime = GcClock::Now();
+ m_LastGcExpireTime = GcClock::TimePoint::min();
if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state"))
{
m_LastGcTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcTime"sv].AsInt64()));
-
+ m_LastGcExpireTime =
+ GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcExpireTime"].AsInt64(GcClock::Duration::min().count())));
if (m_LastGcTime + m_Config.Interval < GcClock::Now())
{
// TODO: Trigger GC?
@@ -478,6 +604,18 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
}
}
+ m_DiskUsageLog.Open(m_Config.RootDirectory / "gc.dlog", CasLogFile::Mode::kWrite);
+ m_DiskUsageLog.Initialize();
+ const GcClock::Tick LastGCTick = m_LastGcTime.time_since_epoch().count();
+ m_DiskUsageLog.Replay(
+ [this, LastGCTick](const DiskUsageWindow::DiskUsageEntry& Entry) {
+ if (Entry.SampleTime >= m_LastGcExpireTime.time_since_epoch().count())
+ {
+ m_DiskUsageWindow.Append(Entry);
+ }
+ },
+ 0);
+
m_NextGcTime = NextGcTime(m_LastGcTime);
m_GcThread = std::thread(&GcScheduler::SchedulerThread, this);
}
@@ -500,6 +638,8 @@ GcScheduler::Shutdown()
m_GcThread.join();
}
}
+ m_DiskUsageLog.Flush();
+ m_DiskUsageLog.Close();
}
bool
@@ -547,34 +687,107 @@ GcScheduler::SchedulerThread()
continue;
}
+ bool Delete = true;
+ bool CollectSmallObjects = m_Config.CollectSmallObjects;
+ std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration;
+ uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit;
+ GcClock::TimePoint Now = GcClock::Now();
+ if (m_TriggerParams)
+ {
+ const auto TriggerParams = m_TriggerParams.value();
+ m_TriggerParams.reset();
+
+ CollectSmallObjects = TriggerParams.CollectSmallObjects;
+ if (TriggerParams.MaxCacheDuration != std::chrono::seconds::max())
+ {
+ MaxCacheDuration = TriggerParams.MaxCacheDuration;
+ }
+ if (TriggerParams.DiskSizeSoftLimit != 0)
+ {
+ DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit;
+ }
+ }
+
+ GcClock::TimePoint ExpireTime = MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
+
+ std::error_code Ec;
+ const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
+
if (Timeout && Status() == GcSchedulerStatus::kIdle)
{
- std::error_code Ec;
- DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
- GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
+ DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
+ if (Ec)
+ {
+ ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message());
+ }
+
std::chrono::seconds RemaingTime = std::chrono::duration_cast<std::chrono::seconds>(m_NextGcTime - GcClock::Now());
+ const int64_t PressureGraphLength = 30;
+ const std::chrono::duration LoadGraphTime = PressureGraphLength * m_Config.MonitorInterval;
+ std::vector<uint64_t> DiskDeltas;
+ uint64_t MaxLoad = 0;
+ {
+ const GcClock::Tick EpochTickCount = GcClock::Now().time_since_epoch().count();
+ std::unique_lock Lock(m_GcMutex);
+ m_DiskUsageWindow.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize});
+ m_DiskUsageLog.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize});
+ const GcClock::TimePoint LoadGraphStartTime = Now - LoadGraphTime;
+ GcClock::Tick Start = LoadGraphStartTime.time_since_epoch().count();
+ GcClock::Tick End = Now.time_since_epoch().count();
+ DiskDeltas = m_DiskUsageWindow.GetDiskDeltas(Start,
+ End,
+ Max(1, (End - Start + PressureGraphLength - 1) / PressureGraphLength),
+ MaxLoad);
+ }
+
if (RemaingTime < std::chrono::seconds::zero())
{
RemaingTime = std::chrono::seconds::zero();
}
- if (Ec)
+ std::string LoadGraph;
+ LoadGraph.resize(DiskDeltas.size(), '0');
+ if (DiskDeltas.size() > 0 && MaxLoad > 0)
{
- ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message());
+ char LoadIndicator[11] = "0123456789";
+ for (size_t Index = 0; Index < DiskDeltas.size(); ++Index)
+ {
+ size_t LoadIndex = (9 * DiskDeltas[Index] + MaxLoad - 1) / MaxLoad;
+ LoadGraph[Index] = LoadIndicator[LoadIndex];
+ }
+ }
+
+ uint64_t GcDiskSpaceGoal = 0;
+ if (DiskSizeSoftLimit != 0 && TotalSize.DiskSize > DiskSizeSoftLimit)
+ {
+ GcDiskSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit;
+ std::unique_lock Lock(m_GcMutex);
+ GcClock::Tick AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count());
+ GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick);
+ if (SizeBasedExpireTime > ExpireTime)
+ {
+ ExpireTime = SizeBasedExpireTime;
+ }
}
- ZEN_INFO("{} in use, {} of total {} free disk space, {}",
+ bool DiskSpaceGCTriggered = GcDiskSpaceGoal > 0;
+ bool TimeBasedGCTriggered = !DiskSpaceGCTriggered && RemaingTime.count() == 0;
+ ZEN_INFO("{} in use,{} {} of total {} free disk space, disk writes last {} per {} [{}], peak {}/s. {}",
NiceBytes(TotalSize.DiskSize),
+ DiskSizeSoftLimit == 0 ? "" : fmt::format(" {} soft limit,", NiceBytes(DiskSizeSoftLimit)),
NiceBytes(Space.Free),
NiceBytes(Space.Total),
- m_Config.Interval.count()
- ? fmt::format("{} until next GC", NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count())))
- : std::string("no GC scheduled"));
-
- // TODO: Trigger GC if max disk usage water mark is reached
-
- if (RemaingTime.count() > 0)
+ NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count())),
+ NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count() / PressureGraphLength)),
+ LoadGraph,
+ NiceBytes(MaxLoad * uint64_t(std::chrono::seconds(1).count()) / uint64_t(std::chrono::seconds(LoadGraphTime).count())),
+ DiskSpaceGCTriggered ? fmt::format("Disk use threshold triggered, trying to reclaim {}. ", NiceBytes(GcDiskSpaceGoal))
+ : TimeBasedGCTriggered ? "GC schedule triggered."
+ : fmt::format("{} until next scheduled GC.",
+ NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count()))));
+
+ if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered)
{
WaitTime = m_Config.MonitorInterval < RemaingTime ? m_Config.MonitorInterval : RemaingTime;
continue;
@@ -588,54 +801,7 @@ GcScheduler::SchedulerThread()
}
}
- GcContext GcCtx;
- GcCtx.SetDeletionMode(true);
- GcCtx.CollectSmallObjects(m_Config.CollectSmallObjects);
- GcCtx.MaxCacheDuration(m_Config.MaxCacheDuration);
- GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc");
-
- if (m_TriggerParams)
- {
- const auto TriggerParams = m_TriggerParams.value();
- m_TriggerParams.reset();
-
- GcCtx.CollectSmallObjects(TriggerParams.CollectSmallObjects);
- if (TriggerParams.MaxCacheDuration != std::chrono::seconds::max())
- {
- GcCtx.MaxCacheDuration(TriggerParams.MaxCacheDuration);
- }
- }
-
- ZEN_INFO("garbage collection STARTING, small objects gc {}, max cache duration {}",
- GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
- NiceTimeSpanMs(uint64_t(std::chrono::duration_cast<std::chrono::milliseconds>(GcCtx.MaxCacheDuration()).count())));
- {
- Stopwatch Timer;
- const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
-
- m_GcManager.CollectGarbage(GcCtx);
-
- m_LastGcTime = GcClock::Now();
- m_NextGcTime = NextGcTime(m_LastGcTime);
- WaitTime = m_Config.MonitorInterval;
-
- {
- const fs::path Path = m_Config.RootDirectory / "gc_state";
- ZEN_DEBUG("saving scheduler state to '{}'", Path);
- CbObjectWriter SchedulderState;
- SchedulderState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
- SaveCompactBinaryObject(Path, SchedulderState.Save());
- }
-
- std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
- if (Ec)
- {
- ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'",
- m_Config.RootDirectory / "reserve.gc",
- NiceBytes(m_Config.DiskReserveSize),
- Ec.message());
- }
- }
+ CollectGarbage(ExpireTime, Delete, CollectSmallObjects);
uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
@@ -643,6 +809,8 @@ GcScheduler::SchedulerThread()
ZEN_ASSERT(m_Status == static_cast<uint32_t>(GcSchedulerStatus::kStopped));
break;
}
+
+ WaitTime = m_Config.MonitorInterval;
}
}
@@ -659,6 +827,54 @@ GcScheduler::NextGcTime(GcClock::TimePoint CurrentTime)
}
}
+void
+GcScheduler::CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, bool CollectSmallObjects)
+{
+ GcContext GcCtx(ExpireTime);
+ GcCtx.SetDeletionMode(Delete);
+ GcCtx.CollectSmallObjects(CollectSmallObjects);
+ // GcCtx.MaxCacheDuration(MaxCacheDuration);
+ GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc");
+
+ ZEN_INFO("garbage collection STARTING, small objects gc {}, cutoff time {}",
+ GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
+ ExpireTime);
+ {
+ Stopwatch Timer;
+ const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+
+ m_GcManager.CollectGarbage(GcCtx);
+
+ if (Delete)
+ {
+ m_LastGcExpireTime = ExpireTime;
+ std::unique_lock Lock(m_GcMutex);
+ m_DiskUsageWindow.KeepRange(ExpireTime.time_since_epoch().count(), GcClock::Duration::max().count());
+ }
+
+ m_LastGcTime = GcClock::Now();
+ m_NextGcTime = NextGcTime(m_LastGcTime);
+
+ {
+ const fs::path Path = m_Config.RootDirectory / "gc_state";
+ ZEN_DEBUG("saving scheduler state to '{}'", Path);
+ CbObjectWriter SchedulerState;
+ SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
+ SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
+ SaveCompactBinaryObject(Path, SchedulerState.Save());
+ }
+
+ std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
+ if (Ec)
+ {
+ ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'",
+ m_Config.RootDirectory / "reserve.gc",
+ NiceBytes(m_Config.DiskReserveSize),
+ Ec.message());
+ }
+ }
+}
+
//////////////////////////////////////////////////////////////////////////
#if ZEN_WITH_TESTS
@@ -704,7 +920,7 @@ TEST_CASE("gc.basic")
const auto InsertResult = CidStore.AddChunk(CompressedChunk);
CHECK(InsertResult.New);
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
CidStore.Flush();
@@ -761,7 +977,7 @@ TEST_CASE("gc.full")
// Keep first and last
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
@@ -796,7 +1012,7 @@ TEST_CASE("gc.full")
// Keep last
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
KeepChunks.push_back(ChunkHashes[8]);
@@ -828,7 +1044,7 @@ TEST_CASE("gc.full")
// Keep mixed
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
KeepChunks.push_back(ChunkHashes[1]);
@@ -863,7 +1079,7 @@ TEST_CASE("gc.full")
// Keep multiple at end
{
- GcContext GcCtx;
+ GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
GcCtx.CollectSmallObjects(true);
std::vector<IoHash> KeepChunks;
KeepChunks.push_back(ChunkHashes[6]);
@@ -912,6 +1128,165 @@ TEST_CASE("gc.full")
CHECK_LE(InitialSize.TinySize, FinalSize.TinySize);
CHECK_GE(InitialSize.TinySize + (1u << 28), FinalSize.TinySize);
}
+
+TEST_CASE("gc.diskusagewindow")
+{
+ DiskUsageWindow Stats;
+ Stats.Append({.SampleTime = 0, .DiskUsage = 0}); // 0 0
+ Stats.Append({.SampleTime = 10, .DiskUsage = 10}); // 1 10
+ Stats.Append({.SampleTime = 20, .DiskUsage = 20}); // 2 10
+ Stats.Append({.SampleTime = 30, .DiskUsage = 20}); // 3 0
+ Stats.Append({.SampleTime = 40, .DiskUsage = 15}); // 4 0
+ Stats.Append({.SampleTime = 50, .DiskUsage = 25}); // 5 10
+ Stats.Append({.SampleTime = 60, .DiskUsage = 30}); // 6 5
+ Stats.Append({.SampleTime = 70, .DiskUsage = 45}); // 7 15
+
+ SUBCASE("Truncate start")
+ {
+ Stats.KeepRange(-15, 31);
+ CHECK(Stats.m_LogWindow.size() == 4);
+ CHECK(Stats.m_LogWindow[0].SampleTime == 0);
+ CHECK(Stats.m_LogWindow[3].SampleTime == 30);
+ }
+
+ SUBCASE("Truncate end")
+ {
+ Stats.KeepRange(70, 71);
+ CHECK(Stats.m_LogWindow.size() == 1);
+ CHECK(Stats.m_LogWindow[0].SampleTime == 70);
+ }
+
+ SUBCASE("Truncate middle")
+ {
+ Stats.KeepRange(29, 69);
+ CHECK(Stats.m_LogWindow.size() == 4);
+ CHECK(Stats.m_LogWindow[0].SampleTime == 30);
+ CHECK(Stats.m_LogWindow[3].SampleTime == 60);
+ }
+
+ SUBCASE("Full range")
+ {
+ uint64_t MaxDelta = 0;
+ // 0-10, 10-20, 20-30, 30-40, 40-50, 50-60, 60-70, 70-80
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 10, MaxDelta);
+ CHECK(DiskDeltas.size() == 8);
+ CHECK(MaxDelta == 15);
+ CHECK(DiskDeltas[0] == 0);
+ CHECK(DiskDeltas[1] == 10);
+ CHECK(DiskDeltas[2] == 10);
+ CHECK(DiskDeltas[3] == 0);
+ CHECK(DiskDeltas[4] == 0);
+ CHECK(DiskDeltas[5] == 10);
+ CHECK(DiskDeltas[6] == 5);
+ CHECK(DiskDeltas[7] == 15);
+ }
+
+ SUBCASE("Sub range")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(20, 40, 10, MaxDelta);
+ CHECK(DiskDeltas.size() == 2);
+ CHECK(MaxDelta == 10);
+ CHECK(DiskDeltas[0] == 10); // [20:30]
+ CHECK(DiskDeltas[1] == 0); // [30:40]
+ }
+ SUBCASE("Unaligned sub range 1")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(21, 51, 10, MaxDelta);
+ CHECK(DiskDeltas.size() == 3);
+ CHECK(MaxDelta == 10);
+ CHECK(DiskDeltas[0] == 0); // [21:31]
+ CHECK(DiskDeltas[1] == 0); // [31:41]
+ CHECK(DiskDeltas[2] == 10); // [41:51]
+ }
+ SUBCASE("Unaligned end range")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(29, 79, 10, MaxDelta);
+ CHECK(DiskDeltas.size() == 5);
+ CHECK(MaxDelta == 15);
+ CHECK(DiskDeltas[0] == 0); // [29:39]
+ CHECK(DiskDeltas[1] == 0); // [39:49]
+ CHECK(DiskDeltas[2] == 10); // [49:59]
+ CHECK(DiskDeltas[3] == 5); // [59:69]
+ CHECK(DiskDeltas[4] == 15); // [69:79]
+ }
+ SUBCASE("Ahead of window")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-40, 0, 10, MaxDelta);
+ CHECK(DiskDeltas.size() == 4);
+ CHECK(MaxDelta == 0);
+ CHECK(DiskDeltas[0] == 0); // [-40:-30]
+ CHECK(DiskDeltas[1] == 0); // [-30:-20]
+ CHECK(DiskDeltas[2] == 0); // [-20:-10]
+ CHECK(DiskDeltas[3] == 0); // [-10:0]
+ }
+ SUBCASE("After of window")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(90, 120, 10, MaxDelta);
+ CHECK(DiskDeltas.size() == 3);
+ CHECK(MaxDelta == 0);
+ CHECK(DiskDeltas[0] == 0); // [90:100]
+ CHECK(DiskDeltas[1] == 0); // [100:110]
+ CHECK(DiskDeltas[2] == 0); // [110:120]
+ }
+ SUBCASE("Encapsulating window")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-20, 100, 10, MaxDelta);
+ CHECK(DiskDeltas.size() == 12);
+ CHECK(MaxDelta == 15);
+ CHECK(DiskDeltas[0] == 0); // [-20:-10]
+ CHECK(DiskDeltas[1] == 0); // [ -10:0]
+ CHECK(DiskDeltas[2] == 0); // [0:10]
+ CHECK(DiskDeltas[3] == 10); // [10:20]
+ CHECK(DiskDeltas[4] == 10); // [20:30]
+ CHECK(DiskDeltas[5] == 0); // [30:40]
+ CHECK(DiskDeltas[6] == 0); // [40:50]
+ CHECK(DiskDeltas[7] == 10); // [50:60]
+ CHECK(DiskDeltas[8] == 5); // [60:70]
+ CHECK(DiskDeltas[9] == 15); // [70:80]
+ CHECK(DiskDeltas[10] == 0); // [80:90]
+ CHECK(DiskDeltas[11] == 0); // [90:100]
+ }
+
+ SUBCASE("Full range half stride")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 20, MaxDelta);
+ CHECK(DiskDeltas.size() == 4);
+ CHECK(MaxDelta == 20);
+ CHECK(DiskDeltas[0] == 10); // [0:20]
+ CHECK(DiskDeltas[1] == 10); // [20:40]
+ CHECK(DiskDeltas[2] == 10); // [40:60]
+ CHECK(DiskDeltas[3] == 20); // [60:80]
+ }
+
+ SUBCASE("Partial odd stride")
+ {
+ uint64_t MaxDelta = 0;
+ std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(13, 67, 18, MaxDelta);
+ CHECK(DiskDeltas.size() == 3);
+ CHECK(MaxDelta == 15);
+ CHECK(DiskDeltas[0] == 10); // [13:31]
+ CHECK(DiskDeltas[1] == 0); // [31:49]
+ CHECK(DiskDeltas[2] == 15); // [49:67]
+ }
+
+ SUBCASE("Find size window")
+ {
+ DiskUsageWindow Empty;
+ CHECK(Empty.FindTimepointThatRemoves(15u, 10000) == 10000);
+
+ CHECK(Stats.FindTimepointThatRemoves(15u, 40) == 21);
+ CHECK(Stats.FindTimepointThatRemoves(15u, 20) == 20);
+ CHECK(Stats.FindTimepointThatRemoves(100000u, 50) == 50);
+ CHECK(Stats.FindTimepointThatRemoves(100000u, 1000));
+ }
+}
#endif
void
diff --git a/zenstore/include/zenstore/gc.h b/zenstore/include/zenstore/gc.h
index 656e594af..e0354b331 100644
--- a/zenstore/include/zenstore/gc.h
+++ b/zenstore/include/zenstore/gc.h
@@ -4,6 +4,7 @@
#include <zencore/iohash.h>
#include <zencore/thread.h>
+#include <zenstore/caslog.h>
#include <atomic>
#include <chrono>
@@ -47,7 +48,7 @@ public:
class GcContext
{
public:
- GcContext(GcClock::TimePoint Time = GcClock::Now());
+ GcContext(const GcClock::TimePoint& ExpireTime);
~GcContext();
void AddRetainedCids(std::span<const IoHash> Cid);
@@ -69,16 +70,11 @@ public:
bool CollectSmallObjects() const;
void CollectSmallObjects(bool NewState);
- GcClock::TimePoint Time() const;
-
- GcClock::Duration MaxCacheDuration() const;
- void MaxCacheDuration(GcClock::Duration Duration);
+ GcClock::TimePoint ExpireTime() const;
void DiskReservePath(const std::filesystem::path& Path);
uint64_t ClaimGCReserve();
- inline bool Expired(GcClock::Tick TickCount) { return Time() - GcClock::TimePointFromTick(TickCount) > MaxCacheDuration(); }
-
private:
struct GcState;
@@ -150,6 +146,8 @@ public:
#endif
private:
+ spdlog::logger& Log() { return m_Log; }
+ spdlog::logger& m_Log;
mutable RwLock m_Lock;
std::vector<GcContributor*> m_GcContribs;
std::vector<GcStorage*> m_GcStorage;
@@ -172,6 +170,27 @@ struct GcSchedulerConfig
bool CollectSmallObjects = true;
bool Enabled = true;
uint64_t DiskReserveSize = 1ul << 28;
+ uint64_t DiskSizeSoftLimit = 0;
+};
+
+class DiskUsageWindow
+{
+public:
+ struct DiskUsageEntry
+ {
+ GcClock::Tick SampleTime;
+ uint64_t DiskUsage;
+ };
+
+ std::vector<DiskUsageEntry> m_LogWindow;
+ inline void Append(const DiskUsageEntry& Entry) { m_LogWindow.push_back(Entry); }
+ inline void Append(DiskUsageEntry&& Entry) { m_LogWindow.emplace_back(std::move(Entry)); }
+ void KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick);
+ std::vector<uint64_t> GetDiskDeltas(GcClock::Tick StartTick,
+ GcClock::Tick EndTick,
+ GcClock::Tick DeltaWidth,
+ uint64_t& OutMaxDelta) const;
+ GcClock::Tick FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const;
};
/**
@@ -191,12 +210,14 @@ public:
{
bool CollectSmallObjects = false;
std::chrono::seconds MaxCacheDuration = std::chrono::seconds::max();
+ uint64_t DiskSizeSoftLimit = 0;
};
bool Trigger(const TriggerParams& Params);
private:
void SchedulerThread();
+ void CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, bool CollectSmallObjects);
GcClock::TimePoint NextGcTime(GcClock::TimePoint CurrentTime);
spdlog::logger& Log() { return m_Log; }
@@ -204,12 +225,16 @@ private:
GcManager& m_GcManager;
GcSchedulerConfig m_Config;
GcClock::TimePoint m_LastGcTime{};
+ GcClock::TimePoint m_LastGcExpireTime{};
GcClock::TimePoint m_NextGcTime{};
std::atomic_uint32_t m_Status{};
std::thread m_GcThread;
std::mutex m_GcMutex;
std::condition_variable m_GcSignal;
std::optional<TriggerParams> m_TriggerParams;
+
+ TCasLogFile<DiskUsageWindow::DiskUsageEntry> m_DiskUsageLog;
+ DiskUsageWindow m_DiskUsageWindow;
};
void gc_forcelink();