diff options
| author | Dan Engelbrecht <[email protected]> | 2024-10-03 16:42:57 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2024-10-03 16:42:57 +0200 |
| commit | b13b5f48bb497aaf9f9f3d74aceb6e474cf12898 (patch) | |
| tree | 24b1ed63ece11fb773a0ecf41ce6308969468198 /src/zenstore/filecas.cpp | |
| parent | 5.5.9-pre0 (diff) | |
| download | zen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.tar.xz zen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.zip | |
remove gc v1 (#121)
* kill gc v1
* block use of gc v1 from zen command line
* warn and flip to gcv2 if --gc-v2=false is specified for zenserver
Diffstat (limited to 'src/zenstore/filecas.cpp')
| -rw-r--r-- | src/zenstore/filecas.cpp | 170 |
1 files changed, 0 insertions, 170 deletions
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 71eead9b2..57b42beb2 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -1073,101 +1073,6 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx) NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } -void -FileCasStrategy::CollectGarbage(GcContext& GcCtx) -{ - ZEN_TRACE_CPU("FileCas::CollectGarbage"); - - ZEN_ASSERT(m_IsInitialized); - - if (GcCtx.SkipCid()) - { - return; - } - - ZEN_DEBUG("collecting garbage from {}", m_RootDirectory); - - std::vector<IoHash> ChunksToDelete; - std::atomic<uint64_t> ChunksToDeleteBytes{0}; - std::atomic<uint64_t> ChunkCount{0}, ChunkBytes{0}; - - std::vector<IoHash> CandidateCas; - CandidateCas.resize(1); - - uint64_t DeletedCount = 0; - uint64_t OldTotalSize = m_TotalSize.load(std::memory_order::relaxed); - - Stopwatch TotalTimer; - const auto _ = MakeGuard([&] { - ZEN_DEBUG("garbage collect for '{}' DONE after {}, deleted {} out of {} files, removed {} out of {}", - m_RootDirectory, - NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()), - DeletedCount, - ChunkCount.load(), - NiceBytes(OldTotalSize - m_TotalSize.load(std::memory_order::relaxed)), - NiceBytes(OldTotalSize)); - }); - - { - ZEN_TRACE_CPU("FileCas::CollectGarbage::Filter"); - IterateChunks([&](const IoHash& Hash, uint64_t Size) { - bool KeepThis = false; - CandidateCas[0] = Hash; - GcCtx.FilterCids(CandidateCas, [&](const IoHash& Hash) { - ZEN_UNUSED(Hash); - KeepThis = true; - }); - - if (!KeepThis) - { - ChunksToDelete.push_back(Hash); - ChunksToDeleteBytes.fetch_add(Size); - } - - ++ChunkCount; - ChunkBytes.fetch_add(Size); - }); - } - - // TODO, any entires we did not encounter during our IterateChunks should be removed from the index - - if (ChunksToDelete.empty()) - { - ZEN_DEBUG("gc for '{}' SKIPPED, nothing to delete", m_RootDirectory); - return; - } - - ZEN_DEBUG("deleting file CAS garbage for '{}': {} out of {} chunks ({})", - m_RootDirectory, - ChunksToDelete.size(), - ChunkCount.load(), - NiceBytes(ChunksToDeleteBytes)); - - if (GcCtx.IsDeletionMode() == false) - { - ZEN_DEBUG("NOTE: not actually deleting anything since deletion is disabled"); - - return; - } - - for (const IoHash& Hash : ChunksToDelete) - { - ZEN_TRACE("deleting chunk {}", Hash); - - std::error_code Ec; - DeleteChunk(Hash, Ec); - - if (Ec) - { - ZEN_WARN("gc for '{}' failed to delete file for chunk {}: '{}'", m_RootDirectory, Hash, Ec.message()); - continue; - } - DeletedCount++; - } - - GcCtx.AddDeletedCids(ChunksToDelete); -} - GcStorageSize FileCasStrategy::StorageSize() const { @@ -1834,81 +1739,6 @@ TEST_CASE("cas.file.move") # endif } -TEST_CASE("cas.file.gc") -{ - // specifying an absolute path here can be helpful when using procmon to dig into things - ScopedTemporaryDirectory TempDir; // {"d:\\filecas_testdir"}; - - GcManager Gc; - FileCasStrategy FileCas(Gc); - FileCas.Initialize(TempDir.Path() / "cas", /* IsNewStore */ true); - - const int kIterationCount = 100; - std::vector<IoHash> Keys{kIterationCount}; - - auto InsertChunks = [&] { - for (int i = 0; i < kIterationCount; ++i) - { - CbObjectWriter Cbo; - Cbo << "id" << i; - CbObject Obj = Cbo.Save(); - - IoBuffer ObjBuffer = Obj.GetBuffer().AsIoBuffer(); - IoHash Hash = IoHash::HashBuffer(ObjBuffer); - - FileCas.InsertChunk(ObjBuffer, Hash); - - Keys[i] = Hash; - } - }; - - // Drop everything - - { - InsertChunks(); - - GcContext Ctx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - FileCas.CollectGarbage(Ctx); - - for (const IoHash& Key : Keys) - { - IoBuffer Chunk = FileCas.FindChunk(Key); - - CHECK(!Chunk); - } - } - - // Keep roughly half of the chunks - - { - InsertChunks(); - - GcContext Ctx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - - for (const IoHash& Key : Keys) - { - if (Key.Hash[0] & 1) - { - Ctx.AddRetainedCids(std::vector<IoHash>{Key}); - } - } - - FileCas.CollectGarbage(Ctx); - - for (const IoHash& Key : Keys) - { - if (Key.Hash[0] & 1) - { - CHECK(FileCas.FindChunk(Key)); - } - else - { - CHECK(!FileCas.FindChunk(Key)); - } - } - } -} - #endif void |