aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenstore')
-rw-r--r--src/zenstore/compactcas.cpp89
-rw-r--r--src/zenstore/compactcas.h3
-rw-r--r--src/zenstore/filecas.cpp99
-rw-r--r--src/zenstore/filecas.h3
-rw-r--r--src/zenstore/gc.cpp631
-rw-r--r--src/zenstore/include/zenstore/gc.h90
6 files changed, 558 insertions, 357 deletions
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index f93dafa21..655c0558d 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -567,15 +567,22 @@ public:
{
}
- virtual void CompactReferenceStore(GcCtx& Ctx)
+ virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- size_t CompactedCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc block store '{}': compacted {} cids in {}",
- m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
- CompactedCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: compactcas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
if (Ctx.Settings.IsDeleteMode && Ctx.Settings.CollectSmallObjects)
@@ -609,13 +616,12 @@ public:
}
}
m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
- Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+ Stats.RemovedDisk += FreedDiskSpace;
},
[&]() { return 0; });
- CompactedCount = m_PrunedKeys.size();
- Ctx.CompactedReferences.fetch_add(
- CompactedCount); // Slightly missleading, it might not be compacted if the block is the currently writing block
+ Stats.Compacted +=
+ m_PrunedKeys.size(); // Slightly missleading, it might not be compacted if the block is the currently writing block
}
}
@@ -634,21 +640,27 @@ public:
{
}
- virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences)
+ virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx,
+ GcReferenceStoreStats& Stats,
+ const GetUnusedReferencesFunc& GetUnusedReferences)
{
- size_t TotalCount = m_Cids.size();
- size_t PruneCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc block store '{}': removed {} unused cid out of {} in {}",
- m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
- PruneCount,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
- m_Cids.clear();
if (UnusedCids.empty())
{
@@ -725,8 +737,7 @@ public:
}
}
- PruneCount = UnusedKeys.size();
- Ctx.PrunedReferences.fetch_add(PruneCount);
+ Stats.Pruned += UnusedKeys.size();
return new CasContainerStoreCompactor(m_CasContainerStrategy,
std::move(CompactState),
std::move(CompactStateKeys),
@@ -738,33 +749,45 @@ private:
std::vector<IoHash> m_Cids;
};
+std::string
+CasContainerStrategy::GetGcName(GcCtx&)
+{
+ return fmt::format("compactcas:'{}'", (m_RootDirectory / m_ContainerBaseName).string());
+}
+
GcReferencePruner*
-CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx)
+CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- size_t TotalCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc block store '{}': found {} cid keys to check in {}",
- m_RootDirectory / m_ContainerBaseName,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: compactcas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_RootDirectory / m_ContainerBaseName,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> CidsToCheck;
{
RwLock::SharedLockScope __(m_LocationMapLock);
+ if (m_LocationMap.empty())
+ {
+ return {};
+ }
CidsToCheck.reserve(m_LocationMap.size());
for (const auto& It : m_LocationMap)
{
CidsToCheck.push_back(It.first);
}
}
- TotalCount = CidsToCheck.size();
- if (TotalCount == 0)
- {
- return {};
- }
- Ctx.References.fetch_add(TotalCount);
+ Stats.Count += CidsToCheck.size();
return new CasContainerReferencePruner(*this, std::move(CidsToCheck));
}
diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h
index 9ff4ae4fc..9f3aa6b07 100644
--- a/src/zenstore/compactcas.h
+++ b/src/zenstore/compactcas.h
@@ -71,7 +71,8 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override;
- virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override;
private:
CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index e28e0dea4..c021e0e21 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1340,15 +1340,22 @@ public:
{
}
- virtual void CompactReferenceStore(GcCtx& Ctx)
+ virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- size_t CompactedCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc file store '{}': removed data for {} unused cids in {}",
- m_FileCasStrategy.m_RootDirectory,
- CompactedCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: filecas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_FileCasStrategy.m_RootDirectory,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> ReferencedCleaned;
ReferencedCleaned.reserve(m_ReferencesToClean.size());
@@ -1382,7 +1389,7 @@ public:
{
continue;
}
- Ctx.RemovedDiskSpace.fetch_add(SizeOnDisk);
+ Stats.RemovedDisk += SizeOnDisk;
}
else
{
@@ -1401,8 +1408,7 @@ public:
ReferencedCleaned.push_back(ChunkHash);
}
}
- CompactedCount = ReferencedCleaned.size();
- Ctx.CompactedReferences.fetch_add(ReferencedCleaned.size());
+ Stats.Compacted += ReferencedCleaned.size();
}
private:
@@ -1415,27 +1421,38 @@ class FileCasReferencePruner : public GcReferencePruner
public:
FileCasReferencePruner(FileCasStrategy& Owner, std::vector<IoHash>&& Cids) : m_FileCasStrategy(Owner), m_Cids(std::move(Cids)) {}
- virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences)
+ virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx,
+ GcReferenceStoreStats& Stats,
+ const GetUnusedReferencesFunc& GetUnusedReferences)
{
- size_t TotalCount = m_Cids.size();
- size_t PruneCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc file store '{}': removed {} unused cid out of {} in {}",
- m_FileCasStrategy.m_RootDirectory,
- PruneCount,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: filecas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_FileCasStrategy.m_RootDirectory,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
- std::vector<IoHash> UnusedReferences = GetUnusedReferences(m_Cids);
- m_Cids.clear();
+ std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
+ if (UnusedCids.empty())
+ {
+ // Nothing to collect
+ return nullptr;
+ }
std::vector<IoHash> PrunedReferences;
- PrunedReferences.reserve(UnusedReferences.size());
+ PrunedReferences.reserve(UnusedCids.size());
{
RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock);
- for (const IoHash& ChunkHash : UnusedReferences)
+ for (const IoHash& ChunkHash : UnusedCids)
{
auto It = m_FileCasStrategy.m_Index.find(ChunkHash);
if (It == m_FileCasStrategy.m_Index.end())
@@ -1454,8 +1471,7 @@ public:
}
}
- PruneCount = PrunedReferences.size();
- Ctx.PrunedReferences.fetch_add(PruneCount);
+ Stats.Pruned += PrunedReferences.size();
return new FileCasStoreCompactor(m_FileCasStrategy, std::move(PrunedReferences));
}
@@ -1464,33 +1480,44 @@ private:
std::vector<IoHash> m_Cids;
};
+std::string
+FileCasStrategy::GetGcName(GcCtx&)
+{
+ return fmt::format("filecas:'{}'", m_RootDirectory.string());
+}
+
GcReferencePruner*
-FileCasStrategy::CreateReferencePruner(GcCtx& Ctx)
+FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
{
- // TODO
- std::size_t TotalCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
- ZEN_DEBUG("gc file store '{}': found {} cid keys to check in {}",
- m_RootDirectory,
- TotalCount,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: filecas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ m_RootDirectory,
+ Stats.Count,
+ Stats.Pruned,
+ Stats.Compacted,
+ NiceBytes(Stats.RemovedDisk),
+ NiceBytes(Stats.RemovedMemory),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
std::vector<IoHash> CidsToCheck;
{
RwLock::SharedLockScope __(m_Lock);
+ if (m_Index.empty())
+ {
+ return {};
+ }
CidsToCheck.reserve(m_Index.size());
for (const auto& It : m_Index)
{
CidsToCheck.push_back(It.first);
}
}
- TotalCount = CidsToCheck.size();
- if (TotalCount == 0)
- {
- return {};
- }
- Ctx.References.fetch_add(TotalCount);
+ Stats.Count += CidsToCheck.size();
return new FileCasReferencePruner(*this, std::move(CidsToCheck));
}
diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h
index 2e9a1d5dc..c39a39bb7 100644
--- a/src/zenstore/filecas.h
+++ b/src/zenstore/filecas.h
@@ -44,7 +44,8 @@ struct FileCasStrategy final : public GcStorage, public GcReferenceStore
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override;
- virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx) override;
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override;
private:
void MakeIndexSnapshot();
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index e09f46063..4d146c16c 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -330,6 +330,47 @@ GcManager::~GcManager()
//////// Begin New GC WIP
void
+GcResult::Sum()
+{
+ for (std::pair<std::string, GcReferencerStats>& Referencer : ReferencerStats)
+ {
+ GcReferencerStats& SubStat = Referencer.second;
+ ReferencerStat.Count += SubStat.Count;
+ ReferencerStat.Expired += SubStat.Expired;
+ ReferencerStat.Deleted += SubStat.Deleted;
+ ReferencerStat.RemovedDisk += SubStat.RemovedDisk;
+ ReferencerStat.RemovedMemory += SubStat.RemovedMemory;
+ SubStat.ElapsedMS = SubStat.RemoveExpiredDataMS + SubStat.CreateReferenceCheckersMS + SubStat.LockStateMS;
+
+ ReferencerStat.RemoveExpiredDataMS += SubStat.RemoveExpiredDataMS;
+ ReferencerStat.CreateReferenceCheckersMS += SubStat.CreateReferenceCheckersMS;
+ ReferencerStat.LockStateMS += SubStat.LockStateMS;
+ ReferencerStat.ElapsedMS += SubStat.ElapsedMS;
+
+ RemovedDisk += SubStat.RemovedDisk;
+ RemovedMemory += SubStat.RemovedMemory;
+ }
+ for (std::pair<std::string, GcReferenceStoreStats>& ReferenceStore : ReferenceStoreStats)
+ {
+ GcReferenceStoreStats& SubStat = ReferenceStore.second;
+ ReferenceStoreStat.Count += SubStat.Count;
+ ReferenceStoreStat.Pruned += SubStat.Pruned;
+ ReferenceStoreStat.Compacted += SubStat.Compacted;
+ ReferenceStoreStat.RemovedDisk += SubStat.RemovedDisk;
+ ReferenceStoreStat.RemovedMemory += SubStat.RemovedMemory;
+ SubStat.ElapsedMS = SubStat.CreateReferencePrunerMS + SubStat.RemoveUnreferencedDataMS + SubStat.CompactReferenceStoreMS;
+
+ ReferenceStoreStat.CreateReferencePrunerMS += SubStat.CreateReferencePrunerMS;
+ ReferenceStoreStat.RemoveUnreferencedDataMS += SubStat.RemoveUnreferencedDataMS;
+ ReferenceStoreStat.CompactReferenceStoreMS += SubStat.CompactReferenceStoreMS;
+ ReferenceStoreStat.ElapsedMS += SubStat.ElapsedMS;
+
+ RemovedDisk += SubStat.RemovedDisk;
+ RemovedMemory += SubStat.RemovedMemory;
+ }
+}
+
+void
GcManager::AddGcReferencer(GcReferencer& Referencer)
{
RwLock::ExclusiveLockScope _(m_Lock);
@@ -358,245 +399,270 @@ GcManager::RemoveGcReferenceStore(GcReferenceStore& ReferenceStore)
GcResult
GcManager::CollectGarbage(const GcSettings& Settings)
{
- GcCtx Ctx{.Settings = Settings};
-
- Stopwatch TotalTimer;
- auto __ = MakeGuard([&]() {
- ZEN_INFO(
- "GC: Removed {} items out of {}, deleted {} out of {}. Pruned {} Cid entries out of {}, compacted {} Cid entries out of {}, "
- "freed "
- "{} on disk and {} of memory in {}",
- Ctx.ExpiredItems.load(),
- Ctx.Items.load(),
- Ctx.DeletedItems.load(),
- Ctx.ExpiredItems.load(),
- Ctx.PrunedReferences.load(),
- Ctx.References.load(),
- Ctx.CompactedReferences.load(),
- Ctx.PrunedReferences.load(),
- NiceBytes(Ctx.RemovedDiskSpace.load()),
- NiceBytes(Ctx.RemovedMemory.load()),
- NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
- });
-
- RwLock::SharedLockScope GcLock(m_Lock);
-
- static const bool SingleThread =
-#if ZEN_BUILD_DEBUG
- true
-#else
- false
-#endif
- ;
- WorkerThreadPool ThreadPool(SingleThread ? 0 : 8);
-
- if (!m_GcReferencers.empty())
- {
- Latch WorkLeft(1);
- // First remove any cache keys that may own references
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Removed expired data in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (GcReferencer* Owner : m_GcReferencers)
- {
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, Owner, &WorkLeft]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- Owner->RemoveExpiredData(Ctx);
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
+ GcCtx Ctx{.Settings = Settings};
+ GcResult Result;
- if (Ctx.Settings.SkipCidDelete)
{
- return GcResult{.Items = Ctx.Items.load(),
- .ExpiredItems = Ctx.ExpiredItems.load(),
- .DeletedItems = Ctx.DeletedItems.load(),
- .References = Ctx.References.load(),
- .PrunedReferences = Ctx.PrunedReferences.load(),
- .CompactedReferences = Ctx.CompactedReferences.load(),
- .RemovedDiskSpace = Ctx.RemovedDiskSpace.load(),
- .RemovedMemory = Ctx.RemovedMemory.load()};
- }
+ Stopwatch TotalTimer;
+ auto __ = MakeGuard([&]() { Result.ElapsedMS = std::chrono::milliseconds(TotalTimer.GetElapsedTimeMs()); });
- std::vector<std::unique_ptr<GcReferencePruner>> ReferencePruners;
- if (!m_GcReferenceStores.empty())
- {
- ReferencePruners.reserve(m_GcReferenceStores.size());
- Latch WorkLeft(1);
- RwLock ReferencePrunersLock;
- // Easy to go wide, CreateReferencePruner is usually not very heavy but big data sets change that
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Created Cid pruners in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (GcReferenceStore* CidStore : m_GcReferenceStores)
- {
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, CidStore, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // The CidStore will pick a list of CId entries to check, returning a collector
- std::unique_ptr<GcReferencePruner> ReferencePruner(CidStore->CreateReferencePruner(Ctx));
- if (ReferencePruner)
- {
- RwLock::ExclusiveLockScope __(ReferencePrunersLock);
- ReferencePruners.emplace_back(std::move(ReferencePruner));
- }
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
+ RwLock::SharedLockScope GcLock(m_Lock);
- std::vector<std::unique_ptr<GcReferenceChecker>> ReferenceCheckers;
- if (!m_GcReferencers.empty())
- {
- ReferenceCheckers.reserve(m_GcReferencers.size());
- Latch WorkLeft(1);
- RwLock ReferenceCheckersLock;
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Created Cid checkers in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- // Easy to go wide, CreateReferenceCheckers is potentially heavy
- // Lock all reference owners from changing the reference data and get access to check for referenced data
- for (GcReferencer* Referencer : m_GcReferencers)
- {
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Referencer, &ReferenceCheckersLock, &ReferenceCheckers]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // The Referencer will create a reference checker that guarrantees that the references do not change as long as it lives
- std::vector<GcReferenceChecker*> Checkers = Referencer->CreateReferenceCheckers(Ctx);
- try
- {
- if (!Checkers.empty())
- {
- RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
- for (auto& Checker : Checkers)
- {
- ReferenceCheckers.emplace_back(std::unique_ptr<GcReferenceChecker>(Checker));
- Checker = nullptr;
- }
- }
- }
- catch (std::exception&)
- {
- while (!Checkers.empty())
- {
- delete Checkers.back();
- Checkers.pop_back();
- }
- throw;
- }
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
-
- Stopwatch LockStateTimer;
- if (!ReferenceCheckers.empty())
- {
- // Easy to go wide, locking all references checkers so we hafve a stead state of which references are used
- // From this point we have block all writes to all References (DiskBucket/ProjectStore) until we do delete the ReferenceCheckers
- Latch WorkLeft(1);
-
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Locked Cid checkers in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (std::unique_ptr<GcReferenceChecker>& ReferenceChecker : ReferenceCheckers)
- {
- GcReferenceChecker* Checker = ReferenceChecker.get();
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, Checker, &WorkLeft, &ReferenceCheckers]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- Checker->LockState(Ctx);
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
-
- std::vector<std::unique_ptr<GcReferenceStoreCompactor>> ReferenceStoreCompactors;
- ReferenceStoreCompactors.reserve(ReferencePruners.size());
- if (!ReferencePruners.empty())
- {
- const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
- HashSet UnusedCids(References.begin(), References.end());
- for (const std::unique_ptr<GcReferenceChecker>& ReferenceChecker : ReferenceCheckers)
- {
- ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
- if (UnusedCids.empty())
- {
- return {};
- }
- }
- return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
- };
-
- // Easy to go wide, checking all Cids agains references in cache
- // Ask stores to remove data that the ReferenceCheckers says are not references - this should be a lightweight operation that
- // only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
-
- Latch WorkLeft(1);
- RwLock ReferenceStoreCompactorsLock;
-
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Pruned unreferenced Cid data in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (std::unique_ptr<GcReferencePruner>& ReferencePruner : ReferencePruners)
- {
- GcReferencePruner* Pruner = ReferencePruner.get();
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork(
- [&Ctx, Pruner, &WorkLeft, &GetUnusedReferences, &ReferenceStoreCompactorsLock, &ReferenceStoreCompactors]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
- std::unique_ptr<GcReferenceStoreCompactor> ReferenceCompactor(Pruner->RemoveUnreferencedData(Ctx, GetUnusedReferences));
- if (ReferenceCompactor)
- {
- RwLock::ExclusiveLockScope __(ReferenceStoreCompactorsLock);
- ReferenceStoreCompactors.emplace_back(std::move(ReferenceCompactor));
- }
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
- // Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
- ReferenceCheckers.clear();
- ZEN_INFO("GC: Writes blocked for {}", NiceTimeSpanMs(LockStateTimer.GetElapsedTimeMs()))
-
- // Let go of the pruners
- ReferencePruners.clear();
-
- if (!ReferenceStoreCompactors.empty())
- {
- Latch WorkLeft(1);
-
- // Easy to go wide
- // Remove the stuff we deemed unreferenced from disk - may be heavy operation
- Stopwatch Timer;
- auto _ = MakeGuard([&]() { ZEN_INFO("GC: Compacted Cid stores in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())) });
- for (std::unique_ptr<GcReferenceStoreCompactor>& StoreCompactor : ReferenceStoreCompactors)
- {
- GcReferenceStoreCompactor* Compactor = StoreCompactor.get();
- WorkLeft.AddCount(1);
- ThreadPool.ScheduleWork([&Ctx, Compactor, &WorkLeft]() {
- auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
- // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
- Compactor->CompactReferenceStore(Ctx);
- });
- }
- WorkLeft.CountDown();
- WorkLeft.Wait();
- }
-
- ReferenceStoreCompactors.clear();
-
- return GcResult{.Items = Ctx.Items.load(),
- .ExpiredItems = Ctx.ExpiredItems.load(),
- .DeletedItems = Ctx.DeletedItems.load(),
- .References = Ctx.References.load(),
- .PrunedReferences = Ctx.PrunedReferences.load(),
- .CompactedReferences = Ctx.CompactedReferences.load(),
- .RemovedDiskSpace = Ctx.RemovedDiskSpace.load(),
- .RemovedMemory = Ctx.RemovedMemory.load()};
+ static const bool SingleThread =
+#if ZEN_BUILD_DEBUG
+ true
+#else
+ false
+#endif
+ ;
+
+#define SCOPED_TIMER(closure) \
+ Stopwatch $Timer##__LINE__; \
+ auto $Guard##__LINE = MakeGuard([&, &Timer = $Timer##__LINE__]() { closure })
+
+ Result.ReferencerStats.resize(m_GcReferencers.size());
+ Result.ReferenceStoreStats.resize(m_GcReferenceStores.size());
+
+ WorkerThreadPool ThreadPool(SingleThread ? 0 : 8);
+
+ ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size());
+ if (!m_GcReferencers.empty())
+ {
+ Latch WorkLeft(1);
+ // First remove any cache keys that may own references
+ SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+ {
+ GcReferencer* Owner = m_GcReferencers[Index];
+ std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, Owner, &Stats, &WorkLeft]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ Stats.first = Owner->GetGcName(Ctx);
+ SCOPED_TIMER(Stats.second.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Owner->RemoveExpiredData(Ctx, Stats.second);
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ if (Ctx.Settings.SkipCidDelete)
+ {
+ Result.Sum();
+ return Result;
+ }
+
+ ZEN_INFO("GCV2: Creating reference pruners from {} reference stores", m_GcReferenceStores.size());
+ std::unordered_map<size_t, std::unique_ptr<GcReferencePruner>> ReferencePruners;
+ if (!m_GcReferenceStores.empty())
+ {
+ ReferencePruners.reserve(m_GcReferenceStores.size());
+ Latch WorkLeft(1);
+ RwLock ReferencePrunersLock;
+ // CreateReferencePruner is usually not very heavy but big data sets change that
+ SCOPED_TIMER(Result.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++)
+ {
+ GcReferenceStore* ReferenceStore = m_GcReferenceStores[Index];
+ std::pair<std::string, GcReferenceStoreStats>& Stats = Result.ReferenceStoreStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ Stats.first = ReferenceStore->GetGcName(Ctx);
+ std::unique_ptr<GcReferencePruner> ReferencePruner;
+ {
+ SCOPED_TIMER(Stats.second.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ // The ReferenceStore will pick a list of CId entries to check, returning a collector
+ ReferencePruner = std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second));
+ }
+ if (ReferencePruner)
+ {
+ RwLock::ExclusiveLockScope __(ReferencePrunersLock);
+ ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
+ }
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ ZEN_INFO("GCV2: Creating reference checkers from {} referencers", m_GcReferencers.size());
+ std::unordered_map<std::unique_ptr<GcReferenceChecker>, size_t> ReferenceCheckers;
+ if (!m_GcReferencers.empty())
+ {
+ ReferenceCheckers.reserve(m_GcReferencers.size());
+ Latch WorkLeft(1);
+ RwLock ReferenceCheckersLock;
+ SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ // Lock all reference owners from changing the reference data and get access to check for referenced data
+ for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+ {
+ GcReferencer* Referencer = m_GcReferencers[Index];
+ std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ // The Referencer will create a reference checker that guarrantees that the references do not change as long as it lives
+ std::vector<GcReferenceChecker*> Checkers;
+ {
+ SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Checkers = Referencer->CreateReferenceCheckers(Ctx);
+ }
+ try
+ {
+ if (!Checkers.empty())
+ {
+ RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
+ for (auto& Checker : Checkers)
+ {
+ ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
+ Checker = nullptr;
+ }
+ }
+ }
+ catch (std::exception&)
+ {
+ while (!Checkers.empty())
+ {
+ delete Checkers.back();
+ Checkers.pop_back();
+ }
+ throw;
+ }
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ std::unordered_map<std::unique_ptr<GcReferenceStoreCompactor>, size_t> ReferenceStoreCompactors;
+ ReferenceStoreCompactors.reserve(ReferencePruners.size());
+
+ ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
+ {
+ SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
+ ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
+ if (!ReferenceCheckers.empty())
+ {
+ // Locking all references checkers so we have a steady state of which references are used
+ // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+ // we delete the ReferenceCheckers
+ Latch WorkLeft(1);
+
+ SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (auto& It : ReferenceCheckers)
+ {
+ GcReferenceChecker* Checker = It.first.get();
+ size_t Index = It.second;
+ std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Checker->LockState(Ctx);
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
+ if (!ReferencePruners.empty())
+ {
+ const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
+ HashSet UnusedCids(References.begin(), References.end());
+ for (const auto& It : ReferenceCheckers)
+ {
+ GcReferenceChecker* ReferenceChecker = It.first.get();
+ ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
+ if (UnusedCids.empty())
+ {
+ return {};
+ }
+ }
+ return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
+ };
+
+ // checking all Cids agains references in cache
+ // Ask stores to remove data that the ReferenceCheckers says are not referenced - this should be a lightweight operation
+ // that only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
+
+ Latch WorkLeft(1);
+ RwLock ReferenceStoreCompactorsLock;
+
+ SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (auto& It : ReferencePruners)
+ {
+ GcReferencePruner* Pruner = It.second.get();
+ size_t Index = It.first;
+ GcReferenceStoreStats& Stats = Result.ReferenceStoreStats[Index].second;
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx,
+ Pruner,
+ &Stats,
+ &WorkLeft,
+ Index,
+ &GetUnusedReferences,
+ &ReferenceStoreCompactorsLock,
+ &ReferenceStoreCompactors]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
+ std::unique_ptr<GcReferenceStoreCompactor> ReferenceCompactor;
+ {
+ SCOPED_TIMER(Stats.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ ReferenceCompactor =
+ std::unique_ptr<GcReferenceStoreCompactor>(Pruner->RemoveUnreferencedData(Ctx, Stats, GetUnusedReferences));
+ }
+ if (ReferenceCompactor)
+ {
+ RwLock::ExclusiveLockScope __(ReferenceStoreCompactorsLock);
+ ReferenceStoreCompactors.insert_or_assign(std::move(ReferenceCompactor), Index);
+ }
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+ // Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
+ ReferenceCheckers.clear();
+ }
+
+ // Let go of the pruners
+ ReferencePruners.clear();
+
+ ZEN_INFO("GCV2: Compacting reference stores for {} reference store compactors", ReferenceStoreCompactors.size());
+ if (!ReferenceStoreCompactors.empty())
+ {
+ Latch WorkLeft(1);
+
+ // Remove the stuff we deemed unreferenced from disk - may be heavy operation
+ SCOPED_TIMER(Result.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ for (auto& It : ReferenceStoreCompactors)
+ {
+ GcReferenceStoreCompactor* Compactor = It.first.get();
+ size_t Index = It.second;
+ GcReferenceStoreStats& Stats = Result.ReferenceStoreStats[Index].second;
+ WorkLeft.AddCount(1);
+ ThreadPool.ScheduleWork([&Ctx, Compactor, &Stats, &WorkLeft]() {
+ auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+ // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
+ SCOPED_TIMER(Stats.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+ Compactor->CompactReferenceStore(Ctx, Stats);
+ });
+ }
+ WorkLeft.CountDown();
+ WorkLeft.Wait();
+ }
+
+ ReferenceStoreCompactors.clear();
+
+ ZEN_INFO("GCV2: Completed in {}", NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
+ }
+
+ Result.Sum();
+ return Result;
+#undef SCOPED_TIMER
}
//////// End New GC WIP
@@ -998,15 +1064,6 @@ GcScheduler::GetState() const
GcSchedulerState Result{.Status = Status(), .Config = m_Config, .AreDiskWritesBlocked = m_AreDiskWritesBlocked.load()};
- {
- std::unique_lock Lock(m_GcMutex);
- Result.LastFullGcTime = m_LastGcTime;
- Result.LastFullGCDiff = m_LastFullGCDiff;
- Result.LastFullGcDuration = m_LastFullGcDuration;
- Result.LastLightweightGcTime = m_LastLightweightGcTime;
- Result.LastLightweightGCDiff = m_LastLightweightGCDiff;
- Result.LastLightweightGcDuration = m_LastLightweightGcDuration;
- }
std::error_code Ec;
DiskSpace Space = DiskSpaceInfo(Result.Config.RootDirectory, Ec);
if (!Ec)
@@ -1026,30 +1083,40 @@ GcScheduler::GetState() const
Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
}
- GcClock::TimePoint CacheExpireTime =
- Result.Config.MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - Result.Config.MaxCacheDuration;
- GcClock::TimePoint ProjectStoreExpireTime = Result.Config.MaxProjectStoreDuration == GcClock::Duration::max()
- ? GcClock::TimePoint::min()
- : Now - Result.Config.MaxProjectStoreDuration;
+ if (Result.Status != GcSchedulerStatus::kRunning)
+ {
+ {
+ std::unique_lock Lock(m_GcMutex);
+ Result.LastFullGcTime = m_LastGcTime;
+ Result.LastFullGCDiff = m_LastFullGCDiff;
+ Result.LastFullGcDuration = m_LastFullGcDuration;
+ Result.LastLightweightGcTime = m_LastLightweightGcTime;
+ Result.LastLightweightGCDiff = m_LastLightweightGCDiff;
+ Result.LastLightweightGcDuration = m_LastLightweightGcDuration;
+
+ Result.LastLightweightGCV2Result = m_LastLightweightGCV2Result;
+ Result.LastFullGCV2Result = m_LastFullGCV2Result;
+ }
- Result.RemainingTimeUntilFullGc =
- Result.Config.Interval.count() == 0
- ? std::chrono::seconds::max()
- : std::chrono::duration_cast<std::chrono::seconds>(Result.LastFullGcTime + Result.Config.Interval - Now);
+ Result.RemainingTimeUntilFullGc =
+ Result.Config.Interval.count() == 0
+ ? std::chrono::seconds::max()
+ : std::chrono::duration_cast<std::chrono::seconds>(Result.LastFullGcTime + Result.Config.Interval - Now);
- if (Result.RemainingTimeUntilFullGc < std::chrono::seconds::zero())
- {
- Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero();
- }
+ if (Result.RemainingTimeUntilFullGc < std::chrono::seconds::zero())
+ {
+ Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero();
+ }
- Result.RemainingTimeUntilLightweightGc =
- Result.Config.LightweightInterval.count() == 0
- ? std::chrono::seconds::max()
- : std::chrono::duration_cast<std::chrono::seconds>(Result.LastLightweightGcTime + Result.Config.LightweightInterval - Now);
+ Result.RemainingTimeUntilLightweightGc =
+ Result.Config.LightweightInterval.count() == 0
+ ? std::chrono::seconds::max()
+ : std::chrono::duration_cast<std::chrono::seconds>(Result.LastLightweightGcTime + Result.Config.LightweightInterval - Now);
- if (Result.RemainingTimeUntilLightweightGc < std::chrono::seconds::zero())
- {
- Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
+ if (Result.RemainingTimeUntilLightweightGc < std::chrono::seconds::zero())
+ {
+ Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
+ }
}
return Result;
@@ -1474,6 +1541,14 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
{
case GcVersion::kV1:
Diff = m_GcManager.CollectGarbage(GcCtx);
+ if (SkipCid)
+ {
+ m_LastLightweightGCV2Result.reset();
+ }
+ else
+ {
+ m_LastFullGCV2Result.reset();
+ }
break;
case GcVersion::kV2:
{
@@ -1482,7 +1557,33 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
.CollectSmallObjects = CollectSmallObjects,
.IsDeleteMode = Delete,
.SkipCidDelete = SkipCid});
- Diff.DiskSize = Result.RemovedDiskSpace;
+
+ ZEN_INFO(
+ "GCV2: Removed {} items out of {}, deleted {} out of {}. Pruned {} Cid entries out of {}, compacted {} Cid entries "
+ "out of {}, "
+ "freed "
+ "{} on disk and {} of memory in {}",
+ Result.ReferencerStat.Expired,
+ Result.ReferencerStat.Count,
+ Result.ReferencerStat.Deleted,
+ Result.ReferencerStat.Expired,
+ Result.ReferenceStoreStat.Pruned,
+ Result.ReferenceStoreStat.Count,
+ Result.ReferenceStoreStat.Compacted,
+ Result.ReferenceStoreStat.Pruned,
+ NiceBytes(Result.RemovedDisk),
+ NiceBytes(Result.RemovedMemory),
+ NiceTimeSpanMs(Result.ElapsedMS.count()));
+
+ if (SkipCid)
+ {
+ m_LastLightweightGCV2Result = Result;
+ }
+ else
+ {
+ m_LastFullGCV2Result = Result;
+ }
+ Diff.DiskSize = Result.RemovedDisk;
Diff.MemorySize = Result.RemovedMemory;
}
break;
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index fa7dce331..e2e99d5a4 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -61,31 +61,67 @@ struct GcSettings
bool CollectSmallObjects = false;
bool IsDeleteMode = false;
bool SkipCidDelete = false;
+ bool Verbose = false;
+};
+
+struct GcReferencerStats
+{
+ std::uint64_t Count = 0;
+ std::uint64_t Expired = 0;
+ std::uint64_t Deleted = 0;
+ std::uint64_t RemovedDisk = 0;
+ std::uint64_t RemovedMemory = 0;
+
+ std::chrono::milliseconds RemoveExpiredDataMS = {};
+ std::chrono::milliseconds CreateReferenceCheckersMS = {};
+ std::chrono::milliseconds LockStateMS = {};
+ std::chrono::milliseconds ElapsedMS = {};
+};
+
+struct GcReferenceStoreStats
+{
+ std::uint64_t Count = 0;
+ std::uint64_t Pruned = 0;
+ std::uint64_t Compacted = 0;
+ std::uint64_t RemovedDisk = 0;
+ std::uint64_t RemovedMemory = 0;
+
+ std::chrono::milliseconds CreateReferencePrunerMS = {};
+ std::chrono::milliseconds RemoveUnreferencedDataMS = {};
+ std::chrono::milliseconds CompactReferenceStoreMS = {};
+ std::chrono::milliseconds ElapsedMS = {};
};
struct GcResult
{
- uint64_t Items = 0;
- uint64_t ExpiredItems = 0;
- uint64_t DeletedItems = 0;
- uint64_t References = 0;
- uint64_t PrunedReferences = 0;
- uint64_t CompactedReferences = 0;
- uint64_t RemovedDiskSpace = 0;
- uint64_t RemovedMemory = 0;
+ GcReferencerStats ReferencerStat;
+ GcReferenceStoreStats ReferenceStoreStat;
+
+ std::uint64_t RemovedDisk = 0;
+ std::uint64_t RemovedMemory = 0;
+
+ std::vector<std::pair<std::string, GcReferencerStats>> ReferencerStats;
+ std::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats;
+
+ // Wall times, not sum of each
+ std::chrono::milliseconds RemoveExpiredDataMS = {};
+ std::chrono::milliseconds CreateReferenceCheckersMS = {};
+ std::chrono::milliseconds LockStateMS = {};
+
+ std::chrono::milliseconds CreateReferencePrunerMS = {};
+ std::chrono::milliseconds RemoveUnreferencedDataMS = {};
+ std::chrono::milliseconds CompactReferenceStoreMS = {};
+
+ std::chrono::milliseconds WriteBlockMS = {};
+
+ std::chrono::milliseconds ElapsedMS = {};
+
+ void Sum();
};
struct GcCtx
{
- const GcSettings Settings;
- std::atomic_uint64_t Items = 0;
- std::atomic_uint64_t ExpiredItems = 0;
- std::atomic_uint64_t DeletedItems = 0;
- std::atomic_uint64_t References = 0;
- std::atomic_uint64_t PrunedReferences = 0;
- std::atomic_uint64_t CompactedReferences = 0;
- std::atomic_uint64_t RemovedDiskSpace = 0;
- std::atomic_uint64_t RemovedMemory = 0;
+ const GcSettings Settings;
};
typedef tsl::robin_set<IoHash> HashSet;
@@ -106,7 +142,7 @@ public:
virtual ~GcReferenceStoreCompactor() = default;
// Remove data on disk based on results from GcReferencePruner::RemoveUnreferencedData
- virtual void CompactReferenceStore(GcCtx& Ctx) = 0;
+ virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats) = 0;
};
/**
@@ -149,11 +185,13 @@ protected:
virtual ~GcReferencer() = default;
public:
+ virtual std::string GetGcName(GcCtx& Ctx) = 0;
+
// Remove expired data based on either GcCtx::Settings CacheExpireTime/ProjectExpireTime
// TODO: For disk layer we need to first update it with access times from the memory layer
// The implementer of GcReferencer (in our case a disk bucket) does not know about any
// potential memory cache layer :(
- virtual void RemoveExpiredData(GcCtx& Ctx) = 0;
+ virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) = 0;
// Create 0-n GcReferenceChecker for this GcReferencer. Caller will manage lifetime of
// returned instances
@@ -178,7 +216,9 @@ public:
// Caller will manage lifetime of returned instance
// This function should execute as fast as possible, so try to prepare a list of references to check ahead of
// call to this function and make sure the removal of unreferences items is as lightweight as possible.
- virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
+ virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx,
+ GcReferenceStoreStats& Stats,
+ const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
};
/**
@@ -190,9 +230,11 @@ protected:
virtual ~GcReferenceStore() = default;
public:
+ virtual std::string GetGcName(GcCtx& Ctx) = 0;
+
// Create a GcReferencePruner which can check a set of references (decided by implementor) if they are no longer in use
// Caller will manage lifetime of returned instance
- virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx) = 0;
+ virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) = 0;
};
//////// End New GC WIP
@@ -379,6 +421,9 @@ struct GcSchedulerState
GcStorageSize LastFullGCDiff;
std::chrono::milliseconds LastLightweightGcDuration{};
GcStorageSize LastLightweightGCDiff;
+
+ std::optional<GcResult> LastLightweightGCV2Result;
+ std::optional<GcResult> LastFullGCV2Result;
};
class DiskUsageWindow
@@ -461,6 +506,9 @@ private:
std::chrono::milliseconds m_LastLightweightGcDuration{};
GcStorageSize m_LastLightweightGCDiff;
+ std::optional<GcResult> m_LastLightweightGCV2Result;
+ std::optional<GcResult> m_LastFullGCV2Result;
+
std::atomic_uint32_t m_Status{};
std::thread m_GcThread;
mutable std::mutex m_GcMutex;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-24 04:51:48 +0000