aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2023-11-21 15:06:25 +0100
committerGitHub <[email protected]>2023-11-21 15:06:25 +0100
commit05178f7c18a48b21b9e260de282a86b91df26955 (patch)
tree25f77af287730c6dbe8d655e0cb503f2652cbd36 /src/zenserver
parentzen run command (#552) (diff)
downloadzen-05178f7c18a48b21b9e260de282a86b91df26955.tar.xz
zen-05178f7c18a48b21b9e260de282a86b91df26955.zip
compact separate for gc referencer (#533)
- Refactor GCV2 so GcReferencer::RemoveExpiredData returns a store compactor, moving out the actual disk work from deleting items in the index. - Refactor GCV2 GcResult to reuse GcCompactStoreStats and GcStats - Make Compacting of stores non-parallell to not eat all the disk I/O when running GC
Diffstat (limited to 'src/zenserver')
-rw-r--r--src/zenserver/admin/admin.cpp26
-rw-r--r--src/zenserver/cache/cachedisklayer.cpp380
-rw-r--r--src/zenserver/cache/cachedisklayer.h4
-rw-r--r--src/zenserver/cache/structuredcachestore.cpp228
-rw-r--r--src/zenserver/projectstore/projectstore.cpp283
-rw-r--r--src/zenserver/projectstore/projectstore.h7
6 files changed, 568 insertions, 360 deletions
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index d4c69f41b..0b302c36e 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -204,25 +204,21 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
Details = true;
}
- auto SecondsToString = [](std::chrono::seconds Secs) {
- return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count()));
- };
-
CbObjectWriter Response;
Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv);
Response.BeginObject("Config");
{
Response << "RootDirectory" << State.Config.RootDirectory.string();
- Response << "MonitorInterval" << SecondsToString(State.Config.MonitorInterval);
- Response << "Interval" << SecondsToString(State.Config.Interval);
- Response << "MaxCacheDuration" << SecondsToString(State.Config.MaxCacheDuration);
- Response << "MaxProjectStoreDuration" << SecondsToString(State.Config.MaxProjectStoreDuration);
+ Response << "MonitorInterval" << ToTimeSpan(State.Config.MonitorInterval);
+ Response << "Interval" << ToTimeSpan(State.Config.Interval);
+ Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration);
+ Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration);
Response << "CollectSmallObjects" << State.Config.CollectSmallObjects;
Response << "Enabled" << State.Config.Enabled;
Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize);
Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit);
Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites);
- Response << "LightweightInterval" << SecondsToString(State.Config.LightweightInterval);
+ Response << "LightweightInterval" << ToTimeSpan(State.Config.LightweightInterval);
}
Response.EndObject();
Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked;
@@ -233,8 +229,8 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
Response.BeginObject("FullGC");
{
- Response << "LastTime" << fmt::format("{}", State.LastFullGcTime);
- Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilFullGc);
+ Response << "LastTime" << ToDateTime(State.LastFullGcTime);
+ Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilFullGc);
if (State.Config.DiskSizeSoftLimit != 0)
{
Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC);
@@ -246,7 +242,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
}
else
{
- Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count());
+ Response << "LastDuration" << ToTimeSpan(State.LastFullGcDuration);
Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
}
@@ -254,8 +250,8 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
Response.EndObject();
Response.BeginObject("LightweightGC");
{
- Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime);
- Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc);
+ Response << "LastTime" << ToDateTime(State.LastLightweightGcTime);
+ Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilLightweightGc);
if (State.LastLightweightGCV2Result)
{
@@ -264,7 +260,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
}
else
{
- Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count());
+ Response << "LastDuration" << ToTimeSpan(State.LastLightweightGcDuration);
Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
}
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index afb974d76..32ef420d1 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -2351,12 +2351,212 @@ ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
return fmt::format("cachebucket:'{}'", m_BucketDir.string());
}
-void
-ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
+class DiskBucketStoreCompactor : public GcStoreCompactor
{
- size_t TotalEntries = 0;
- tsl::robin_set<IoHash, IoHash::Hasher> ExpiredInlineKeys;
- std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys;
+public:
+ DiskBucketStoreCompactor(ZenCacheDiskLayer::CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
+ : m_Bucket(Bucket)
+ , m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys))
+ {
+ m_ExpiredStandaloneKeys.shrink_to_fit();
+ }
+
+ virtual ~DiskBucketStoreCompactor() {}
+
+ virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override
+ {
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachebucket [COMPACT] '{}': RemovedDisk: {} in {}",
+ m_Bucket.m_BucketDir,
+ NiceBytes(Stats.RemovedDisk),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
+
+ if (!m_ExpiredStandaloneKeys.empty())
+ {
+ // Compact standalone items
+ size_t Skipped = 0;
+ ExtendablePathBuilder<256> Path;
+ for (const std::pair<IoHash, uint64_t>& ExpiredKey : m_ExpiredStandaloneKeys)
+ {
+ Path.Reset();
+ m_Bucket.BuildPath(Path, ExpiredKey.first);
+ fs::path FilePath = Path.ToPath();
+
+ RwLock::SharedLockScope IndexLock(m_Bucket.m_IndexLock);
+ if (m_Bucket.m_Index.contains(ExpiredKey.first))
+ {
+ // Someone added it back, let the file on disk be
+ ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': skipping z$ delete standalone of file '{}' FAILED, it has been added back",
+ m_Bucket.m_BucketDir,
+ Path.ToUtf8());
+ continue;
+ }
+
+ if (Ctx.Settings.IsDeleteMode)
+ {
+ RwLock::ExclusiveLockScope ValueLock(m_Bucket.LockForHash(ExpiredKey.first));
+ IndexLock.ReleaseNow();
+ ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8());
+
+ std::error_code Ec;
+ if (!fs::remove(FilePath, Ec))
+ {
+ continue;
+ }
+ if (Ec)
+ {
+ ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': delete expired z$ standalone file '{}' FAILED, reason: '{}'",
+ m_Bucket.m_BucketDir,
+ Path.ToUtf8(),
+ Ec.message());
+ continue;
+ }
+ Stats.RemovedDisk += ExpiredKey.second;
+ }
+ else
+ {
+ std::error_code Ec;
+ bool Existed = std::filesystem::is_regular_file(FilePath, Ec);
+ if (Ec)
+ {
+ ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'",
+ m_Bucket.m_BucketDir,
+ FilePath,
+ Ec.message());
+ continue;
+ }
+ if (!Existed)
+ {
+ continue;
+ }
+ Skipped++;
+ }
+ }
+ if (Skipped > 0)
+ {
+ ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': skipped deleting of {} eligible files", m_Bucket.m_BucketDir, Skipped);
+ }
+ }
+
+ if (Ctx.Settings.CollectSmallObjects)
+ {
+ std::unordered_map<uint32_t, uint64_t> BlockUsage;
+ {
+ for (const auto& Entry : m_Bucket.m_Index)
+ {
+ ZenCacheDiskLayer::CacheBucket::PayloadIndex Index = Entry.second;
+ const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+ const DiskLocation& Loc = Payload.Location;
+
+ if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+ {
+ continue;
+ }
+ uint32_t BlockIndex = Loc.Location.BlockLocation.GetBlockIndex();
+ uint64_t ChunkSize = RoundUp(Loc.Size(), m_Bucket.m_Configuration.PayloadAlignment);
+ auto It = BlockUsage.find(BlockIndex);
+ if (It == BlockUsage.end())
+ {
+ BlockUsage.insert_or_assign(BlockIndex, ChunkSize);
+ }
+ else
+ {
+ It->second += ChunkSize;
+ }
+ }
+ }
+
+ {
+ BlockStoreCompactState BlockCompactState;
+ std::vector<IoHash> BlockCompactStateKeys;
+
+ std::vector<uint32_t> BlocksToCompact =
+ m_Bucket.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent);
+ BlockCompactState.IncludeBlocks(BlocksToCompact);
+
+ {
+ RwLock::SharedLockScope __(m_Bucket.m_IndexLock);
+ for (const auto& Entry : m_Bucket.m_Index)
+ {
+ ZenCacheDiskLayer::CacheBucket::PayloadIndex Index = Entry.second;
+ const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+ const DiskLocation& Loc = Payload.Location;
+
+ if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+ {
+ continue;
+ }
+ if (!BlockCompactState.AddKeepLocation(Loc.GetBlockLocation(m_Bucket.m_Configuration.PayloadAlignment)))
+ {
+ continue;
+ }
+ BlockCompactStateKeys.push_back(Entry.first);
+ }
+ }
+
+ if (Ctx.Settings.IsDeleteMode)
+ {
+ ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': compacting {} blocks", m_Bucket.m_BucketDir, BlocksToCompact.size());
+
+ m_Bucket.m_BlockStore.CompactBlocks(
+ BlockCompactState,
+ m_Bucket.m_Configuration.PayloadAlignment,
+ [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
+ std::vector<DiskIndexEntry> MovedEntries;
+ RwLock::ExclusiveLockScope _(m_Bucket.m_IndexLock);
+ for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+ {
+ size_t ChunkIndex = Moved.first;
+ const IoHash& Key = BlockCompactStateKeys[ChunkIndex];
+
+ if (auto It = m_Bucket.m_Index.find(Key); It != m_Bucket.m_Index.end())
+ {
+ ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[It->second];
+ const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex);
+ if (Payload.Location.GetBlockLocation(m_Bucket.m_Configuration.PayloadAlignment) != OldLocation)
+ {
+ // Someone has moved our chunk so lets just skip the new location we were provided, it will be GC:d
+ // at a later time
+ continue;
+ }
+ const BlockStoreLocation& NewLocation = Moved.second;
+
+ Payload.Location =
+ DiskLocation(NewLocation, m_Bucket.m_Configuration.PayloadAlignment, Payload.Location.GetFlags());
+ MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
+ }
+ }
+ m_Bucket.m_SlogFile.Append(MovedEntries);
+ Stats.RemovedDisk += FreedDiskSpace;
+ },
+ ClaimDiskReserveCallback);
+ }
+ else
+ {
+ ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': skipped compacting of {} eligible blocks",
+ m_Bucket.m_BucketDir,
+ BlocksToCompact.size());
+ }
+ }
+ }
+ m_ExpiredStandaloneKeys.clear();
+ }
+
+private:
+ ZenCacheDiskLayer::CacheBucket& m_Bucket;
+ std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys;
+};
+
+GcStoreCompactor*
+ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
+{
+ size_t TotalEntries = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
@@ -2364,37 +2564,30 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats&
{
return;
}
- ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}",
m_BucketDir,
- Stats.Count,
- Stats.Expired,
- Stats.Deleted,
- NiceBytes(Stats.RemovedDisk),
- NiceBytes(Stats.RemovedMemory),
+ Stats.CheckedCount,
+ Stats.FoundCount,
+ Stats.DeletedCount,
+ NiceBytes(Stats.FreedMemory),
NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
- BlockStoreCompactState BlockCompactState;
- BlockStore::ReclaimSnapshotState BlockSnapshotState;
- std::vector<IoHash> BlockCompactStateKeys;
- std::vector<DiskIndexEntry> ExpiredEntries;
- uint64_t RemovedStandaloneSize = 0;
+ std::vector<DiskIndexEntry> ExpiredEntries;
+ std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys;
+ uint64_t RemovedStandaloneSize = 0;
{
RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
- if (Ctx.Settings.CollectSmallObjects)
- {
- BlockSnapshotState = m_BlockStore.GetReclaimSnapshotState();
- }
TotalEntries = m_Index.size();
- // Find out expired keys and affected blocks
+ // Find out expired keys
for (const auto& Entry : m_Index)
{
- const IoHash& Key = Entry.first;
- size_t EntryIndex = Entry.second;
- GcClock::Tick AccessTime = m_AccessTimes[EntryIndex];
+ const IoHash& Key = Entry.first;
+ ZenCacheDiskLayer::CacheBucket::PayloadIndex EntryIndex = Entry.second;
+ GcClock::Tick AccessTime = m_AccessTimes[EntryIndex];
if (AccessTime >= ExpireTicks)
{
continue;
@@ -2412,41 +2605,12 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats&
}
else if (Ctx.Settings.CollectSmallObjects)
{
- ExpiredInlineKeys.insert(Key);
- uint32_t BlockIndex = Payload.Location.Location.BlockLocation.GetBlockIndex();
- bool IsActiveWriteBlock = BlockSnapshotState.m_ActiveWriteBlocks.contains(BlockIndex);
- if (!IsActiveWriteBlock)
- {
- BlockCompactState.IncludeBlock(BlockIndex);
- }
ExpiredEntries.push_back(ExpiredEntry);
}
}
- Stats.Expired += ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size();
-
- // Get all locations we need to keep for affected blocks
- if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
- {
- for (const auto& Entry : m_Index)
- {
- const IoHash& Key = Entry.first;
- if (ExpiredInlineKeys.contains(Key))
- {
- continue;
- }
- size_t EntryIndex = Entry.second;
- const BucketPayload& Payload = m_Payloads[EntryIndex];
- if (Payload.Location.Flags & DiskLocation::kStandaloneFile)
- {
- continue;
- }
- if (BlockCompactState.AddKeepLocation(Payload.Location.GetBlockLocation(m_Configuration.PayloadAlignment)))
- {
- BlockCompactStateKeys.push_back(Key);
- }
- }
- }
+ Stats.CheckedCount += TotalEntries;
+ Stats.FoundCount += ExpiredEntries.size();
if (Ctx.Settings.IsDeleteMode)
{
@@ -2456,112 +2620,30 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats&
ZEN_ASSERT(It != m_Index.end());
BucketPayload& Payload = m_Payloads[It->second];
RemoveMetaData(Payload);
- Stats.RemovedMemory += RemoveMemCachedData(Payload);
+ Stats.FreedMemory += RemoveMemCachedData(Payload);
m_Index.erase(It);
+ Stats.DeletedCount++;
}
m_SlogFile.Append(ExpiredEntries);
m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed);
}
}
- Stats.Count += TotalEntries;
-
- if (ExpiredEntries.empty())
- {
- return;
- }
- if (!Ctx.Settings.IsDeleteMode)
+ if (!ExpiredEntries.empty())
{
- return;
- }
-
- Stats.Deleted += ExpiredEntries.size();
-
- // Compact standalone items
- ExtendablePathBuilder<256> Path;
- for (const std::pair<IoHash, uint64_t>& ExpiredKey : ExpiredStandaloneKeys)
- {
- Path.Reset();
- BuildPath(Path, ExpiredKey.first);
- fs::path FilePath = Path.ToPath();
-
- RwLock::SharedLockScope IndexLock(m_IndexLock);
- if (m_Index.contains(ExpiredKey.first))
- {
- // Someone added it back, let the file on disk be
- ZEN_DEBUG("gc cache bucket '{}': skipping z$ delete standalone of file '{}' FAILED, it has been added back",
- m_BucketDir,
- Path.ToUtf8());
- continue;
- }
-
- RwLock::ExclusiveLockScope ValueLock(LockForHash(ExpiredKey.first));
- IndexLock.ReleaseNow();
- ZEN_DEBUG("gc cache bucket '{}': deleting standalone cache file '{}'", m_BucketDir, Path.ToUtf8());
-
- std::error_code Ec;
- if (!fs::remove(FilePath, Ec))
- {
- continue;
- }
- if (Ec)
+ std::vector<BucketPayload> Payloads;
+ std::vector<AccessTime> AccessTimes;
+ std::vector<BucketMetaData> MetaDatas;
+ std::vector<IoBuffer> MemCachedPayloads;
+ std::vector<ReferenceIndex> FirstReferenceIndex;
+ IndexMap Index;
{
- ZEN_WARN("gc cache bucket '{}': delete expired z$ standalone file '{}' FAILED, reason: '{}'",
- m_BucketDir,
- Path.ToUtf8(),
- Ec.message());
- continue;
+ RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+ CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
}
- Stats.RemovedDisk += ExpiredKey.second;
}
- if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
- {
- // Compact block store
- m_BlockStore.CompactBlocks(
- BlockCompactState,
- m_Configuration.PayloadAlignment,
- [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
- std::vector<DiskIndexEntry> MovedEntries;
- RwLock::ExclusiveLockScope _(m_IndexLock);
- for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
- {
- size_t ChunkIndex = Moved.first;
- const IoHash& Key = BlockCompactStateKeys[ChunkIndex];
-
- if (auto It = m_Index.find(Key); It != m_Index.end())
- {
- BucketPayload& Payload = m_Payloads[It->second];
- const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex);
- if (Payload.Location.GetBlockLocation(m_Configuration.PayloadAlignment) != OldLocation)
- {
- // Someone has moved our chunk so lets just skip the new location we were provided, it will be GC:d at a later
- // time
- continue;
- }
-
- const BlockStoreLocation& NewLocation = Moved.second;
-
- Payload.Location = DiskLocation(NewLocation, m_Configuration.PayloadAlignment, Payload.Location.GetFlags());
- MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
- }
- }
- m_SlogFile.Append(MovedEntries);
- Stats.RemovedDisk += FreedDiskSpace;
- },
- [&]() { return 0; });
- }
-
- std::vector<BucketPayload> Payloads;
- std::vector<AccessTime> AccessTimes;
- std::vector<BucketMetaData> MetaDatas;
- std::vector<IoBuffer> MemCachedPayloads;
- std::vector<ReferenceIndex> FirstReferenceIndex;
- IndexMap Index;
- {
- RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
- CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
- }
+ return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys));
}
class DiskBucketReferenceChecker : public GcReferenceChecker
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index d9884a7bc..2986cedf8 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -334,7 +334,7 @@ private:
std::atomic_uint64_t m_MemCachedSize{};
virtual std::string GetGcName(GcCtx& Ctx) override;
- virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
+ virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
@@ -403,6 +403,7 @@ private:
inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
friend class DiskBucketReferenceChecker;
+ friend class DiskBucketStoreCompactor;
};
inline void TryMemCacheTrim()
@@ -438,6 +439,7 @@ private:
ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
+ friend class DiskBucketStoreCompactor;
friend class DiskBucketReferenceChecker;
};
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index cc6fefc76..25dfd103d 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -1957,14 +1957,14 @@ TEST_CASE("z$.newgc.basics")
.CollectSmallObjects = false,
.IsDeleteMode = false,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(0u, Result.ReferencerStat.Expired);
- CHECK_EQ(0u, Result.ReferencerStat.Deleted);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_EQ(0u, Result.RemovedDisk);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_EQ(0u, Result.CompactStoresStatSum.RemovedDisk);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -1991,14 +1991,14 @@ TEST_CASE("z$.newgc.basics")
.CollectSmallObjects = false,
.IsDeleteMode = false,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(1u, Result.ReferencerStat.Expired);
- CHECK_EQ(0u, Result.ReferencerStat.Deleted);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_EQ(0u, Result.RemovedDisk);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_EQ(0u, Result.CompactStoresStatSum.RemovedDisk);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2025,14 +2025,14 @@ TEST_CASE("z$.newgc.basics")
.CollectSmallObjects = true,
.IsDeleteMode = false,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(7u, Result.ReferencerStat.Expired);
- CHECK_EQ(0u, Result.ReferencerStat.Deleted);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_EQ(0u, Result.RemovedDisk);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_EQ(0u, Result.CompactStoresStatSum.RemovedDisk);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2060,14 +2060,14 @@ TEST_CASE("z$.newgc.basics")
.IsDeleteMode = true,
.SkipCidDelete = true,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(1u, Result.ReferencerStat.Expired);
- CHECK_EQ(1u, Result.ReferencerStat.Deleted);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.CompactStoresStatSum.RemovedDisk);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2095,14 +2095,14 @@ TEST_CASE("z$.newgc.basics")
.IsDeleteMode = true,
.SkipCidDelete = true,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(7u, Result.ReferencerStat.Expired);
- CHECK_EQ(7u, Result.ReferencerStat.Deleted);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_GE(Result.RemovedDisk, 0);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_GE(Result.CompactStoresStatSum.RemovedDisk, 0);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
@@ -2130,17 +2130,20 @@ TEST_CASE("z$.newgc.basics")
.IsDeleteMode = true,
.SkipCidDelete = false,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(1u, Result.ReferencerStat.Expired); // Only one cache value is pruned/deleted as that is the only large item in the cache
- // (all other large items as in cas)
- CHECK_EQ(1u, Result.ReferencerStat.Deleted);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(1u,
+ Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount); // Only one cache value is pruned/deleted as that is the only
+ // large item in the cache (all other large items as in cas)
+ CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
CHECK_EQ(0u,
- Result.ReferenceStoreStat
- .Pruned); // We won't remove any references since all referencers are small which retains all references
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
- CHECK_EQ(0u, Result.RemovedMemory);
+ Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats
+ .FoundCount); // We won't remove any references since all referencers are small which retains all references
+ CHECK_EQ(0u,
+ Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats
+ .DeletedCount); // We won't remove any references since all referencers are small which retains all references
+ CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.CompactStoresStatSum.RemovedDisk);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2168,14 +2171,14 @@ TEST_CASE("z$.newgc.basics")
.IsDeleteMode = true,
.SkipCidDelete = false,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(7u, Result.ReferencerStat.Expired);
- CHECK_EQ(7u, Result.ReferencerStat.Deleted);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
- CHECK_GT(Result.RemovedDisk, 0);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
@@ -2200,20 +2203,22 @@ TEST_CASE("z$.newgc.basics")
Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
- .CollectSmallObjects = true,
- .IsDeleteMode = true,
- .SkipCidDelete = true,
- .Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(6u, Result.ReferencerStat.Expired);
- CHECK_EQ(6u, Result.ReferencerStat.Deleted);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_GT(Result.RemovedDisk, 0);
- CHECK_EQ(0u, Result.RemovedMemory);
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .CollectSmallObjects = true,
+ .IsDeleteMode = true,
+ .SkipCidDelete = true,
+ .Verbose = true,
+ .CompactBlockUsageThresholdPercent = 100});
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(6u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(6u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ uint64_t MinExpectedRemoveSize = CacheEntries[UnstructuredCacheValues[2]].Data.GetSize();
+ CHECK_LT(MinExpectedRemoveSize, Result.CompactStoresStatSum.RemovedDisk);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
@@ -2245,14 +2250,14 @@ TEST_CASE("z$.newgc.basics")
.IsDeleteMode = true,
.SkipCidDelete = false,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(5u, Result.ReferencerStat.Expired);
- CHECK_EQ(5u, Result.ReferencerStat.Deleted);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_GT(Result.RemovedDisk, 0);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2285,14 +2290,14 @@ TEST_CASE("z$.newgc.basics")
.IsDeleteMode = true,
.SkipCidDelete = false,
.Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(4u, Result.ReferencerStat.Expired);
- CHECK_EQ(4u, Result.ReferencerStat.Deleted);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
- CHECK_GT(Result.RemovedDisk, 0);
- CHECK_EQ(0u, Result.RemovedMemory);
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
+ CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
@@ -2329,22 +2334,23 @@ TEST_CASE("z$.newgc.basics")
Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
- GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
- .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
- .CollectSmallObjects = true,
- .IsDeleteMode = true,
- .SkipCidDelete = true,
- .Verbose = true});
- CHECK_EQ(7u, Result.ReferencerStat.Count);
- CHECK_EQ(4u, Result.ReferencerStat.Expired);
- CHECK_EQ(4u, Result.ReferencerStat.Deleted);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
- CHECK_GT(Result.RemovedDisk, 0);
+ GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
+ .CollectSmallObjects = true,
+ .IsDeleteMode = true,
+ .SkipCidDelete = true,
+ .Verbose = true,
+ .CompactBlockUsageThresholdPercent = 100});
+ CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() +
CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize();
- CHECK_EQ(MemoryClean, Result.RemovedMemory);
+ CHECK_EQ(MemoryClean, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
@@ -2393,15 +2399,17 @@ TEST_CASE("z$.newgc.basics")
.IsDeleteMode = true,
.SkipCidDelete = false,
.Verbose = true});
- CHECK_EQ(8u, Result.ReferencerStat.Count);
- CHECK_EQ(1u, Result.ReferencerStat.Expired);
- CHECK_EQ(1u, Result.ReferencerStat.Deleted);
- CHECK_EQ(9u, Result.ReferenceStoreStat.Count);
- CHECK_EQ(4u, Result.ReferenceStoreStat.Pruned);
- CHECK_EQ(4u, Result.ReferenceStoreStat.Compacted);
- CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(), Result.RemovedDisk);
+ // Write block can't be compacted so Compacted will be less than Deleted
+ CHECK_EQ(8u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+ CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+ CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+ CHECK_EQ(9u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+ CHECK_EQ(4u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+ CHECK_EQ(4u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+ CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(),
+ Result.CompactStoresStatSum.RemovedDisk);
uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize();
- CHECK_EQ(MemoryClean, Result.RemovedMemory);
+ CHECK_EQ(MemoryClean, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
}
}
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 9fedd9165..617104ddc 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1435,31 +1435,40 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
return nullptr;
}
-void
-ProjectStore::Project::DeleteOplog(std::string_view OplogId)
+std::filesystem::path
+ProjectStore::Project::RemoveOplog(std::string_view OplogId)
{
+ RwLock::ExclusiveLockScope _(m_ProjectLock);
+
std::filesystem::path DeletePath;
+ if (auto OplogIt = m_Oplogs.find(std::string(OplogId)); OplogIt == m_Oplogs.end())
{
- RwLock::ExclusiveLockScope _(m_ProjectLock);
+ std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
- if (auto OplogIt = m_Oplogs.find(std::string(OplogId)); OplogIt == m_Oplogs.end())
+ if (Oplog::ExistsAt(OplogBasePath))
{
- std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
-
- if (Oplog::ExistsAt(OplogBasePath))
+ std::filesystem::path MovedDir;
+ if (PrepareDirectoryDelete(DeletePath, MovedDir))
{
- DeletePath = OplogBasePath;
+ DeletePath = MovedDir;
}
}
- else
- {
- std::unique_ptr<Oplog>& Oplog = OplogIt->second;
- DeletePath = Oplog->PrepareForDelete(true);
- m_DeletedOplogs.emplace_back(std::move(Oplog));
- m_Oplogs.erase(OplogIt);
- }
- m_LastAccessTimes.erase(std::string(OplogId));
}
+ else
+ {
+ std::unique_ptr<Oplog>& Oplog = OplogIt->second;
+ DeletePath = Oplog->PrepareForDelete(true);
+ m_DeletedOplogs.emplace_back(std::move(Oplog));
+ m_Oplogs.erase(OplogIt);
+ }
+ m_LastAccessTimes.erase(std::string(OplogId));
+ return DeletePath;
+}
+
+void
+ProjectStore::Project::DeleteOplog(std::string_view OplogId)
+{
+ std::filesystem::path DeletePath = RemoveOplog(OplogId);
// Erase content on disk
if (!DeletePath.empty())
@@ -1567,9 +1576,29 @@ ProjectStore::Project::GatherReferences(GcContext& GcCtx)
}
uint64_t
+ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath)
+{
+ using namespace std::literals;
+
+ uint64_t Size = 0;
+ std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv;
+ if (std::filesystem::exists(AccessTimesFilePath))
+ {
+ Size += std::filesystem::file_size(AccessTimesFilePath);
+ }
+ std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv;
+ if (std::filesystem::exists(ProjectFilePath))
+ {
+ Size += std::filesystem::file_size(ProjectFilePath);
+ }
+
+ return Size;
+}
+
+uint64_t
ProjectStore::Project::TotalSize() const
{
- uint64_t Result = 0;
+ uint64_t Result = TotalSize(m_OplogStoragePath);
{
std::vector<std::string> OpLogs = ScanForOplogs();
for (const std::string& OpLogId : OpLogs)
@@ -1954,7 +1983,7 @@ ProjectStore::StorageSize() const
std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv;
if (std::filesystem::exists(ProjectStateFilePath))
{
- Result.DiskSize += std::filesystem::file_size(ProjectStateFilePath);
+ Result.DiskSize += Project::TotalSize(ProjectBasePath);
DirectoryContent DirContent;
GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
for (const std::filesystem::path& OplogBasePath : DirContent.Directories)
@@ -2068,12 +2097,8 @@ ProjectStore::UpdateProject(std::string_view ProjectId,
}
bool
-ProjectStore::DeleteProject(std::string_view ProjectId)
+ProjectStore::RemoveProject(std::string_view ProjectId, std::filesystem::path& OutDeletePath)
{
- ZEN_TRACE_CPU("Store::DeleteProject");
-
- ZEN_INFO("deleting project {}", ProjectId);
-
RwLock::ExclusiveLockScope ProjectsLock(m_ProjectsLock);
auto ProjIt = m_Projects.find(std::string{ProjectId});
@@ -2083,20 +2108,34 @@ ProjectStore::DeleteProject(std::string_view ProjectId)
return true;
}
- std::filesystem::path DeletePath;
- bool Success = ProjIt->second->PrepareForDelete(DeletePath);
+ bool Success = ProjIt->second->PrepareForDelete(OutDeletePath);
if (!Success)
{
return false;
}
m_Projects.erase(ProjIt);
- ProjectsLock.ReleaseNow();
+ return true;
+}
+
+bool
+ProjectStore::DeleteProject(std::string_view ProjectId)
+{
+ ZEN_TRACE_CPU("Store::DeleteProject");
+
+ ZEN_INFO("deleting project {}", ProjectId);
+
+ std::filesystem::path DeletePath;
+ if (!RemoveProject(ProjectId, DeletePath))
+ {
+ return false;
+ }
if (!DeletePath.empty())
{
DeleteDirectories(DeletePath);
}
+
return true;
}
@@ -3042,29 +3081,106 @@ ProjectStore::GetGcName(GcCtx&)
return fmt::format("projectstore:'{}'", m_ProjectBasePath.string());
}
-void
-ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
+class ProjectStoreGcStoreCompactor : public GcStoreCompactor
+{
+public:
+ ProjectStoreGcStoreCompactor(const std::filesystem::path& BasePath,
+ std::vector<std::filesystem::path>&& OplogPathsToRemove,
+ std::vector<std::filesystem::path>&& ProjectPathsToRemove)
+ : m_BasePath(BasePath)
+ , m_OplogPathsToRemove(std::move(OplogPathsToRemove))
+ , m_ProjectPathsToRemove(std::move(ProjectPathsToRemove))
+ {
+ }
+
+ virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&)
+ {
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: projectstore [COMPACT] '{}': RemovedDisk: {} in {}",
+ m_BasePath,
+ NiceBytes(Stats.RemovedDisk),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
+
+ if (Ctx.Settings.IsDeleteMode)
+ {
+ for (const std::filesystem::path& OplogPath : m_OplogPathsToRemove)
+ {
+ uint64_t OplogSize = ProjectStore::Oplog::TotalSize(OplogPath);
+ if (DeleteDirectories(OplogPath))
+ {
+ ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed oplog folder '{}', removed {}",
+ m_BasePath,
+ OplogPath,
+ NiceBytes(OplogSize));
+ Stats.RemovedDisk += OplogSize;
+ }
+ else
+ {
+ ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove oplog folder '{}'", m_BasePath, OplogPath);
+ }
+ }
+
+ for (const std::filesystem::path& ProjectPath : m_ProjectPathsToRemove)
+ {
+ uint64_t ProjectSize = ProjectStore::Project::TotalSize(ProjectPath);
+ if (DeleteDirectories(ProjectPath))
+ {
+ ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed project folder '{}', removed {}",
+ m_BasePath,
+ ProjectPath,
+ NiceBytes(ProjectSize));
+ Stats.RemovedDisk += ProjectSize;
+ }
+ else
+ {
+ ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove project folder '{}'", m_BasePath, ProjectPath);
+ }
+ }
+ }
+ else
+ {
+ ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': Skipped deleting of {} oplogs and {} projects",
+ m_BasePath,
+ m_OplogPathsToRemove.size(),
+ m_ProjectPathsToRemove.size());
+ }
+
+ m_ProjectPathsToRemove.clear();
+ m_OplogPathsToRemove.clear();
+ }
+
+private:
+ std::filesystem::path m_BasePath;
+ std::vector<std::filesystem::path> m_OplogPathsToRemove;
+ std::vector<std::filesystem::path> m_ProjectPathsToRemove;
+};
+
+GcStoreCompactor*
+ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
{
- size_t ProjectCount = 0;
- size_t ExpiredProjectCount = 0;
- size_t OplogCount = 0;
- size_t ExpiredOplogCount = 0;
Stopwatch Timer;
const auto _ = MakeGuard([&] {
if (!Ctx.Settings.Verbose)
{
return;
}
- ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+ ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {} in {}",
m_ProjectBasePath,
- Stats.Count,
- Stats.Expired,
- Stats.Deleted,
- NiceBytes(Stats.RemovedDisk),
- NiceBytes(Stats.RemovedMemory),
+ Stats.CheckedCount,
+ Stats.FoundCount,
+ Stats.DeletedCount,
NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
+ std::vector<std::filesystem::path> OplogPathsToRemove;
+ std::vector<std::filesystem::path> ProjectPathsToRemove;
+
std::vector<Ref<Project>> ExpiredProjects;
std::vector<Ref<Project>> Projects;
@@ -3081,6 +3197,8 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
}
}
+ size_t OplogCount = 0;
+ size_t ExpiredOplogCount = 0;
for (const Ref<Project>& Project : Projects)
{
std::vector<std::string> ExpiredOplogs;
@@ -3101,70 +3219,69 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
{
for (const std::string& OplogId : ExpiredOplogs)
{
- std::filesystem::path OplogBasePath = ProjectPath / OplogId;
- uint64_t OplogSize = Oplog::TotalSize(OplogBasePath);
- ZEN_DEBUG("gc project store '{}': garbage collected oplog '{}' in project '{}'. Removing storage on disk",
- m_ProjectBasePath,
- OplogId,
- Project->Identifier);
- Project->DeleteOplog(OplogId);
- Stats.RemovedDisk += OplogSize;
+ std::filesystem::path RemovePath = Project->RemoveOplog(OplogId);
+ if (!RemovePath.empty())
+ {
+ OplogPathsToRemove.push_back(RemovePath);
+ }
}
- Stats.Deleted += ExpiredOplogs.size();
+ Stats.DeletedCount += ExpiredOplogs.size();
Project->Flush();
}
}
- ProjectCount = Projects.size();
- Stats.Count += ProjectCount + OplogCount;
- ExpiredProjectCount = ExpiredProjects.size();
+ size_t ProjectCount = Projects.size();
+
+ Stats.CheckedCount += ProjectCount + OplogCount;
if (ExpiredProjects.empty())
{
- ZEN_DEBUG("gc project store '{}': no expired projects found", m_ProjectBasePath);
- return;
+ ZEN_DEBUG("GCV2: projectstore [REMOVE EXPIRED] '{}': no expired projects found", m_ProjectBasePath);
+ return nullptr;
}
if (Ctx.Settings.IsDeleteMode)
{
for (const Ref<Project>& Project : ExpiredProjects)
{
- std::filesystem::path PathToRemove;
- std::string ProjectId = Project->Identifier;
+ std::string ProjectId = Project->Identifier;
{
{
RwLock::SharedLockScope Lock(m_ProjectsLock);
if (!Project->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime))
{
- ZEN_DEBUG("gc project store '{}': skipped garbage collect of project '{}'. Project no longer expired.",
- m_ProjectBasePath,
- ProjectId);
+ ZEN_DEBUG(
+ "GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project no longer expired.",
+ m_ProjectBasePath,
+ ProjectId);
continue;
}
}
- RwLock::ExclusiveLockScope __(m_ProjectsLock);
- bool Success = Project->PrepareForDelete(PathToRemove);
+ std::filesystem::path RemovePath;
+ bool Success = RemoveProject(ProjectId, RemovePath);
if (!Success)
{
- ZEN_DEBUG("gc project store '{}': skipped garbage collect of project '{}'. Project folder is locked.",
- m_ProjectBasePath,
- ProjectId);
+ ZEN_DEBUG(
+ "GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project folder is locked.",
+ m_ProjectBasePath,
+ ProjectId);
continue;
}
- m_Projects.erase(ProjectId);
- }
-
- ZEN_DEBUG("gc project store '{}': sgarbage collected project '{}'. Removing storage on disk", m_ProjectBasePath, ProjectId);
- if (PathToRemove.empty())
- {
- continue;
+ if (!RemovePath.empty())
+ {
+ ProjectPathsToRemove.push_back(RemovePath);
+ }
}
-
- DeleteDirectories(PathToRemove);
}
- Stats.Deleted += ExpiredProjects.size();
+ Stats.DeletedCount += ExpiredProjects.size();
}
- Stats.Expired += ExpiredOplogCount + ExpiredProjectCount;
+ size_t ExpiredProjectCount = ExpiredProjects.size();
+ Stats.FoundCount += ExpiredOplogCount + ExpiredProjectCount;
+ if (!OplogPathsToRemove.empty() || !ProjectPathsToRemove.empty())
+ {
+ return new ProjectStoreGcStoreCompactor(m_ProjectBasePath, std::move(OplogPathsToRemove), std::move(ProjectPathsToRemove));
+ }
+ return nullptr;
}
class ProjectStoreReferenceChecker : public GcReferenceChecker
@@ -3182,16 +3299,15 @@ public:
}
ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': precached {} references in {} from {}/{}",
m_Oplog.m_BasePath,
- m_UncachedReferences.size(),
+ m_References.size(),
NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
m_Oplog.m_OuterProject->Identifier,
m_Oplog.OplogId());
});
RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
- m_Oplog.IterateOplog([&](CbObjectView Op) {
- Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
- });
+ m_Oplog.IterateOplog(
+ [&](CbObjectView Op) { Op.IterateAttachments([&](CbFieldView Visitor) { m_References.insert(Visitor.AsAttachment()); }); });
m_PreCachedLsn = m_Oplog.GetMaxOpIndex();
}
}
@@ -3208,7 +3324,7 @@ public:
}
ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
m_Oplog.m_BasePath,
- m_UncachedReferences.size(),
+ m_References.size(),
NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
m_Oplog.m_OuterProject->Identifier,
m_Oplog.OplogId());
@@ -3219,23 +3335,22 @@ public:
{
// TODO: Maybe we could just check the added oplog entries - we might get a few extra references from obsolete entries
// but I don't think that would be critical
- m_UncachedReferences.clear();
- m_Oplog.IterateOplog([&](CbObjectView Op) {
- Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
- });
+ m_References.clear();
+ m_Oplog.IterateOplog(
+ [&](CbObjectView Op) { Op.IterateAttachments([&](CbFieldView Visitor) { m_References.insert(Visitor.AsAttachment()); }); });
}
}
virtual void RemoveUsedReferencesFromSet(GcCtx&, HashSet& IoCids) override
{
- for (const IoHash& ReferenceHash : m_UncachedReferences)
+ for (const IoHash& ReferenceHash : m_References)
{
IoCids.erase(ReferenceHash);
}
}
ProjectStore::Oplog& m_Oplog;
std::unique_ptr<RwLock::SharedLockScope> m_OplogLock;
- HashSet m_UncachedReferences;
+ HashSet m_References;
int m_PreCachedLsn = -1;
};
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index fe1068485..555f8bdf2 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -225,6 +225,7 @@ public:
Oplog* NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath);
Oplog* OpenOplog(std::string_view OplogId);
void DeleteOplog(std::string_view OplogId);
+ std::filesystem::path RemoveOplog(std::string_view OplogId);
void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const;
void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, Oplog&)>&& Fn);
std::vector<std::string> ScanForOplogs() const;
@@ -245,6 +246,7 @@ public:
void ScrubStorage(ScrubContext& Ctx);
LoggerRef Log();
void GatherReferences(GcContext& GcCtx);
+ static uint64_t TotalSize(const std::filesystem::path& BasePath);
uint64_t TotalSize() const;
bool PrepareForDelete(std::filesystem::path& OutDeletePath);
@@ -280,6 +282,7 @@ public:
std::string_view EngineRootDir,
std::string_view ProjectRootDir,
std::string_view ProjectFilePath);
+ bool RemoveProject(std::string_view ProjectId, std::filesystem::path& OutDeletePath);
bool DeleteProject(std::string_view ProjectId);
bool Exists(std::string_view ProjectId);
void Flush();
@@ -295,7 +298,7 @@ public:
virtual GcStorageSize StorageSize() const override;
virtual std::string GetGcName(GcCtx& Ctx) override;
- virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
+ virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
CbArray GetProjectsList();
@@ -379,6 +382,8 @@ private:
const DiskWriteBlocker* m_DiskWriteBlocker = nullptr;
std::filesystem::path BasePathForProject(std::string_view ProjectId);
+
+ friend class ProjectStoreGcStoreCompactor;
};
void prj_forcelink();