aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore/filecas.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2024-10-03 16:42:57 +0200
committerGitHub Enterprise <[email protected]>2024-10-03 16:42:57 +0200
commitb13b5f48bb497aaf9f9f3d74aceb6e474cf12898 (patch)
tree24b1ed63ece11fb773a0ecf41ce6308969468198 /src/zenstore/filecas.cpp
parent5.5.9-pre0 (diff)
downloadzen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.tar.xz
zen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.zip
remove gc v1 (#121)
* kill gc v1 * block use of gc v1 from zen command line * warn and flip to gcv2 if --gc-v2=false is specified for zenserver
Diffstat (limited to 'src/zenstore/filecas.cpp')
-rw-r--r--src/zenstore/filecas.cpp170
1 files changed, 0 insertions, 170 deletions
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 71eead9b2..57b42beb2 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1073,101 +1073,6 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
-void
-FileCasStrategy::CollectGarbage(GcContext& GcCtx)
-{
- ZEN_TRACE_CPU("FileCas::CollectGarbage");
-
- ZEN_ASSERT(m_IsInitialized);
-
- if (GcCtx.SkipCid())
- {
- return;
- }
-
- ZEN_DEBUG("collecting garbage from {}", m_RootDirectory);
-
- std::vector<IoHash> ChunksToDelete;
- std::atomic<uint64_t> ChunksToDeleteBytes{0};
- std::atomic<uint64_t> ChunkCount{0}, ChunkBytes{0};
-
- std::vector<IoHash> CandidateCas;
- CandidateCas.resize(1);
-
- uint64_t DeletedCount = 0;
- uint64_t OldTotalSize = m_TotalSize.load(std::memory_order::relaxed);
-
- Stopwatch TotalTimer;
- const auto _ = MakeGuard([&] {
- ZEN_DEBUG("garbage collect for '{}' DONE after {}, deleted {} out of {} files, removed {} out of {}",
- m_RootDirectory,
- NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()),
- DeletedCount,
- ChunkCount.load(),
- NiceBytes(OldTotalSize - m_TotalSize.load(std::memory_order::relaxed)),
- NiceBytes(OldTotalSize));
- });
-
- {
- ZEN_TRACE_CPU("FileCas::CollectGarbage::Filter");
- IterateChunks([&](const IoHash& Hash, uint64_t Size) {
- bool KeepThis = false;
- CandidateCas[0] = Hash;
- GcCtx.FilterCids(CandidateCas, [&](const IoHash& Hash) {
- ZEN_UNUSED(Hash);
- KeepThis = true;
- });
-
- if (!KeepThis)
- {
- ChunksToDelete.push_back(Hash);
- ChunksToDeleteBytes.fetch_add(Size);
- }
-
- ++ChunkCount;
- ChunkBytes.fetch_add(Size);
- });
- }
-
- // TODO, any entires we did not encounter during our IterateChunks should be removed from the index
-
- if (ChunksToDelete.empty())
- {
- ZEN_DEBUG("gc for '{}' SKIPPED, nothing to delete", m_RootDirectory);
- return;
- }
-
- ZEN_DEBUG("deleting file CAS garbage for '{}': {} out of {} chunks ({})",
- m_RootDirectory,
- ChunksToDelete.size(),
- ChunkCount.load(),
- NiceBytes(ChunksToDeleteBytes));
-
- if (GcCtx.IsDeletionMode() == false)
- {
- ZEN_DEBUG("NOTE: not actually deleting anything since deletion is disabled");
-
- return;
- }
-
- for (const IoHash& Hash : ChunksToDelete)
- {
- ZEN_TRACE("deleting chunk {}", Hash);
-
- std::error_code Ec;
- DeleteChunk(Hash, Ec);
-
- if (Ec)
- {
- ZEN_WARN("gc for '{}' failed to delete file for chunk {}: '{}'", m_RootDirectory, Hash, Ec.message());
- continue;
- }
- DeletedCount++;
- }
-
- GcCtx.AddDeletedCids(ChunksToDelete);
-}
-
GcStorageSize
FileCasStrategy::StorageSize() const
{
@@ -1834,81 +1739,6 @@ TEST_CASE("cas.file.move")
# endif
}
-TEST_CASE("cas.file.gc")
-{
- // specifying an absolute path here can be helpful when using procmon to dig into things
- ScopedTemporaryDirectory TempDir; // {"d:\\filecas_testdir"};
-
- GcManager Gc;
- FileCasStrategy FileCas(Gc);
- FileCas.Initialize(TempDir.Path() / "cas", /* IsNewStore */ true);
-
- const int kIterationCount = 100;
- std::vector<IoHash> Keys{kIterationCount};
-
- auto InsertChunks = [&] {
- for (int i = 0; i < kIterationCount; ++i)
- {
- CbObjectWriter Cbo;
- Cbo << "id" << i;
- CbObject Obj = Cbo.Save();
-
- IoBuffer ObjBuffer = Obj.GetBuffer().AsIoBuffer();
- IoHash Hash = IoHash::HashBuffer(ObjBuffer);
-
- FileCas.InsertChunk(ObjBuffer, Hash);
-
- Keys[i] = Hash;
- }
- };
-
- // Drop everything
-
- {
- InsertChunks();
-
- GcContext Ctx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- FileCas.CollectGarbage(Ctx);
-
- for (const IoHash& Key : Keys)
- {
- IoBuffer Chunk = FileCas.FindChunk(Key);
-
- CHECK(!Chunk);
- }
- }
-
- // Keep roughly half of the chunks
-
- {
- InsertChunks();
-
- GcContext Ctx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-
- for (const IoHash& Key : Keys)
- {
- if (Key.Hash[0] & 1)
- {
- Ctx.AddRetainedCids(std::vector<IoHash>{Key});
- }
- }
-
- FileCas.CollectGarbage(Ctx);
-
- for (const IoHash& Key : Keys)
- {
- if (Key.Hash[0] & 1)
- {
- CHECK(FileCas.FindChunk(Key));
- }
- else
- {
- CHECK(!FileCas.FindChunk(Key));
- }
- }
- }
-}
-
#endif
void