diff options
| author | Dan Engelbrecht <[email protected]> | 2024-10-03 16:42:57 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2024-10-03 16:42:57 +0200 |
| commit | b13b5f48bb497aaf9f9f3d74aceb6e474cf12898 (patch) | |
| tree | 24b1ed63ece11fb773a0ecf41ce6308969468198 /src/zenstore/compactcas.cpp | |
| parent | 5.5.9-pre0 (diff) | |
| download | zen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.tar.xz zen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.zip | |
remove gc v1 (#121)
* kill gc v1
* block use of gc v1 from zen command line
* warn and flip to gcv2 if --gc-v2=false is specified for zenserver
Diffstat (limited to 'src/zenstore/compactcas.cpp')
| -rw-r--r-- | src/zenstore/compactcas.cpp | 639 |
1 files changed, 29 insertions, 610 deletions
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index a5d70a991..7f1300177 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -527,167 +527,6 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) ZEN_INFO("scrubbed {} chunks ({}) in '{}'", ChunkCount.load(), NiceBytes(ChunkBytes.load()), m_RootDirectory / m_ContainerBaseName); } -void -CasContainerStrategy::CollectGarbage(GcContext& GcCtx) -{ - ZEN_TRACE_CPU("CasContainer::CollectGarbage"); - - if (GcCtx.SkipCid()) - { - return; - } - - // It collects all the blocks that we want to delete chunks from. For each such - // block we keep a list of chunks to retain and a list of chunks to delete. - // - // If there is a block that we are currently writing to, that block is omitted - // from the garbage collection. - // - // Next it will iterate over all blocks that we want to remove chunks from. - // If the block is empty after removal of chunks we mark the block as pending - // delete - we want to delete it as soon as there are no IoBuffers using the - // block file. - // Once complete we update the m_LocationMap by removing the chunks. - // - // If the block is non-empty we write out the chunks we want to keep to a new - // block file (creating new block files as needed). - // - // We update the index as we complete each new block file. This makes it possible - // to break the GC if we want to limit time for execution. - // - // GC can very parallell to regular operation - it will block while taking - // a snapshot of the current m_LocationMap state and while moving blocks it will - // do a blocking operation and update the m_LocationMap after each new block is - // written and figuring out the path to the next new block. - - ZEN_DEBUG("collecting garbage from '{}'", m_RootDirectory / m_ContainerBaseName); - - uint64_t WriteBlockTimeUs = 0; - uint64_t WriteBlockLongestTimeUs = 0; - uint64_t ReadBlockTimeUs = 0; - uint64_t ReadBlockLongestTimeUs = 0; - - LocationMap_t LocationMap; - std::vector<BlockStoreDiskLocation> Locations; - BlockStore::ReclaimSnapshotState BlockStoreState; - { - ZEN_TRACE_CPU("CasContainer::CollectGarbage::State"); - - RwLock::SharedLockScope ___(m_LocationMapLock); - Stopwatch Timer; - const auto ____ = MakeGuard([&Timer, &ReadBlockTimeUs, &ReadBlockLongestTimeUs] { - uint64_t ElapsedUs = Timer.GetElapsedTimeUs(); - ReadBlockTimeUs += ElapsedUs; - ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs); - }); - LocationMap = m_LocationMap; - Locations = m_Locations; - BlockStoreState = m_BlockStore.GetReclaimSnapshotState(); - } - - uint64_t TotalChunkCount = LocationMap.size(); - - std::vector<IoHash> TotalChunkHashes; - TotalChunkHashes.reserve(TotalChunkCount); - for (const auto& Entry : LocationMap) - { - TotalChunkHashes.push_back(Entry.first); - } - - std::vector<BlockStoreLocation> ChunkLocations; - BlockStore::ChunkIndexArray KeepChunkIndexes; - std::vector<IoHash> ChunkIndexToChunkHash; - ChunkLocations.reserve(TotalChunkCount); - KeepChunkIndexes.reserve(TotalChunkCount); - ChunkIndexToChunkHash.reserve(TotalChunkCount); - - { - ZEN_TRACE_CPU("CasContainer::CollectGarbage::Filter"); - GcCtx.FilterCids(TotalChunkHashes, [&](const IoHash& ChunkHash, bool Keep) { - auto KeyIt = LocationMap.find(ChunkHash); - const BlockStoreDiskLocation& DiskLocation = Locations[KeyIt->second]; - BlockStoreLocation Location = DiskLocation.Get(m_PayloadAlignment); - size_t ChunkIndex = ChunkLocations.size(); - - ChunkLocations.push_back(Location); - ChunkIndexToChunkHash.push_back(ChunkHash); - if (Keep) - { - KeepChunkIndexes.push_back(ChunkIndex); - } - }); - } - - const bool PerformDelete = GcCtx.IsDeletionMode() && GcCtx.CollectSmallObjects(); - if (!PerformDelete) - { - m_BlockStore.ReclaimSpace(BlockStoreState, ChunkLocations, KeepChunkIndexes, m_PayloadAlignment, true); - return; - } - - std::vector<IoHash> DeletedChunks; - m_BlockStore.ReclaimSpace( - BlockStoreState, - ChunkLocations, - KeepChunkIndexes, - m_PayloadAlignment, - false, - [&](const BlockStore::MovedChunksArray& MovedChunks, const BlockStore::ChunkIndexArray& RemovedChunks) { - std::vector<CasDiskIndexEntry> LogEntries; - LogEntries.reserve(MovedChunks.size() + RemovedChunks.size()); - { - RwLock::ExclusiveLockScope __(m_LocationMapLock); - Stopwatch Timer; - const auto ____ = MakeGuard([&] { - uint64_t ElapsedUs = Timer.GetElapsedTimeUs(); - WriteBlockTimeUs += ElapsedUs; - WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs); - }); - for (const auto& Entry : MovedChunks) - { - size_t ChunkIndex = Entry.first; - const BlockStoreLocation& NewLocation = Entry.second; - const IoHash& ChunkHash = ChunkIndexToChunkHash[ChunkIndex]; - size_t LocationIndex = m_LocationMap[ChunkHash]; - BlockStoreDiskLocation& Location = m_Locations[LocationIndex]; - if (Locations[LocationMap[ChunkHash]] != Location) - { - // Entry has been updated while GC was running, ignore the move - continue; - } - Location = {NewLocation, m_PayloadAlignment}; - LogEntries.push_back({.Key = ChunkHash, .Location = Location}); - } - for (const size_t ChunkIndex : RemovedChunks) - { - const IoHash& ChunkHash = ChunkIndexToChunkHash[ChunkIndex]; - size_t LocationIndex = m_LocationMap[ChunkHash]; - const BlockStoreDiskLocation& Location = Locations[LocationIndex]; - if (Locations[LocationMap[ChunkHash]] != Location) - { - // Entry has been updated while GC was running, ignore the delete - continue; - } - LogEntries.push_back({.Key = ChunkHash, .Location = Location, .Flags = CasDiskIndexEntry::kTombstone}); - m_LocationMap.erase(ChunkHash); - DeletedChunks.push_back(ChunkHash); - } - } - - m_CasLog.Append(LogEntries); - m_CasLog.Flush(); - }, - [&GcCtx]() { return GcCtx.ClaimGCReserve(); }); - - if (!DeletedChunks.empty()) - { - // Clean up m_Locations vectors - RwLock::ExclusiveLockScope IndexLock(m_LocationMapLock); - CompactIndex(IndexLock); - } - GcCtx.AddDeletedCids(DeletedChunks); -} - class CasContainerStoreCompactor : public GcStoreCompactor { public: @@ -1456,412 +1295,7 @@ TEST_CASE("compactcas.compact.totalsize") } } -TEST_CASE("compactcas.gc.basic") -{ - ScopedTemporaryDirectory TempDir; - - GcManager Gc; - CasContainerStrategy Cas(Gc); - Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, true); - - IoBuffer Chunk = CreateRandomBlob(128); - IoHash ChunkHash = IoHash::HashBuffer(Chunk); - - const CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, ChunkHash); - CHECK(InsertResult.New); - Cas.Flush(); - - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - - Cas.CollectGarbage(GcCtx); - - CHECK(!Cas.HaveChunk(ChunkHash)); -} - -TEST_CASE("compactcas.gc.removefile") -{ - ScopedTemporaryDirectory TempDir; - - IoBuffer Chunk = CreateRandomBlob(128); - IoHash ChunkHash = IoHash::HashBuffer(Chunk); - { - GcManager Gc; - CasContainerStrategy Cas(Gc); - Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, true); - - const CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, ChunkHash); - CHECK(InsertResult.New); - const CasStore::InsertResult InsertResultDup = Cas.InsertChunk(Chunk, ChunkHash); - CHECK(!InsertResultDup.New); - Cas.Flush(); - } - - GcManager Gc; - CasContainerStrategy Cas(Gc); - Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, false); - - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - - Cas.CollectGarbage(GcCtx); - - CHECK(!Cas.HaveChunk(ChunkHash)); -} - -TEST_CASE("compactcas.gc.compact") -{ - { - ScopedTemporaryDirectory TempDir; - - GcManager Gc; - CasContainerStrategy Cas(Gc); - Cas.Initialize(TempDir.Path(), "cb", 2048, 1 << 4, true); - - uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5}; - std::vector<IoBuffer> Chunks; - Chunks.reserve(9); - for (uint64_t Size : ChunkSizes) - { - Chunks.push_back(CreateRandomBlob(Size)); - } - - std::vector<IoHash> ChunkHashes; - ChunkHashes.reserve(9); - for (const IoBuffer& Chunk : Chunks) - { - ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size())); - } - - CHECK(Cas.InsertChunk(Chunks[0], ChunkHashes[0]).New); - CHECK(Cas.InsertChunk(Chunks[1], ChunkHashes[1]).New); - CHECK(Cas.InsertChunk(Chunks[2], ChunkHashes[2]).New); - CHECK(Cas.InsertChunk(Chunks[3], ChunkHashes[3]).New); - CHECK(Cas.InsertChunk(Chunks[4], ChunkHashes[4]).New); - CHECK(Cas.InsertChunk(Chunks[5], ChunkHashes[5]).New); - CHECK(Cas.InsertChunk(Chunks[6], ChunkHashes[6]).New); - CHECK(Cas.InsertChunk(Chunks[7], ChunkHashes[7]).New); - CHECK(Cas.InsertChunk(Chunks[8], ChunkHashes[8]).New); - - CHECK(Cas.HaveChunk(ChunkHashes[0])); - CHECK(Cas.HaveChunk(ChunkHashes[1])); - CHECK(Cas.HaveChunk(ChunkHashes[2])); - CHECK(Cas.HaveChunk(ChunkHashes[3])); - CHECK(Cas.HaveChunk(ChunkHashes[4])); - CHECK(Cas.HaveChunk(ChunkHashes[5])); - CHECK(Cas.HaveChunk(ChunkHashes[6])); - CHECK(Cas.HaveChunk(ChunkHashes[7])); - CHECK(Cas.HaveChunk(ChunkHashes[8])); - - auto ValidateChunkExists = [&](size_t Index) { - IoBuffer Chunk = Cas.FindChunk(ChunkHashes[Index]); - bool Exists = !!Chunk; - CHECK(Exists); - IoHash Hash = IoHash::HashBuffer(Chunk); - if (ChunkHashes[Index] != Hash) - { - CHECK(fmt::format("{}", ChunkHashes[Index]) == fmt::format("{}", Hash)); - } - }; - - // Keep first and last - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - - std::vector<IoHash> KeepChunks; - KeepChunks.push_back(ChunkHashes[0]); - KeepChunks.push_back(ChunkHashes[8]); - GcCtx.AddRetainedCids(KeepChunks); - - Cas.Flush(); - Cas.CollectGarbage(GcCtx); - - CHECK(Cas.HaveChunk(ChunkHashes[0])); - CHECK(!Cas.HaveChunk(ChunkHashes[1])); - CHECK(!Cas.HaveChunk(ChunkHashes[2])); - CHECK(!Cas.HaveChunk(ChunkHashes[3])); - CHECK(!Cas.HaveChunk(ChunkHashes[4])); - CHECK(!Cas.HaveChunk(ChunkHashes[5])); - CHECK(!Cas.HaveChunk(ChunkHashes[6])); - CHECK(!Cas.HaveChunk(ChunkHashes[7])); - CHECK(Cas.HaveChunk(ChunkHashes[8])); - - ValidateChunkExists(0); - ValidateChunkExists(8); - - Cas.InsertChunk(Chunks[1], ChunkHashes[1]); - Cas.InsertChunk(Chunks[2], ChunkHashes[2]); - Cas.InsertChunk(Chunks[3], ChunkHashes[3]); - Cas.InsertChunk(Chunks[4], ChunkHashes[4]); - Cas.InsertChunk(Chunks[5], ChunkHashes[5]); - Cas.InsertChunk(Chunks[6], ChunkHashes[6]); - Cas.InsertChunk(Chunks[7], ChunkHashes[7]); - } - - // Keep last - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - std::vector<IoHash> KeepChunks; - KeepChunks.push_back(ChunkHashes[8]); - GcCtx.AddRetainedCids(KeepChunks); - - Cas.Flush(); - Cas.CollectGarbage(GcCtx); - - CHECK(!Cas.HaveChunk(ChunkHashes[0])); - CHECK(!Cas.HaveChunk(ChunkHashes[1])); - CHECK(!Cas.HaveChunk(ChunkHashes[2])); - CHECK(!Cas.HaveChunk(ChunkHashes[3])); - CHECK(!Cas.HaveChunk(ChunkHashes[4])); - CHECK(!Cas.HaveChunk(ChunkHashes[5])); - CHECK(!Cas.HaveChunk(ChunkHashes[6])); - CHECK(!Cas.HaveChunk(ChunkHashes[7])); - CHECK(Cas.HaveChunk(ChunkHashes[8])); - - ValidateChunkExists(8); - - Cas.InsertChunk(Chunks[1], ChunkHashes[1]); - Cas.InsertChunk(Chunks[2], ChunkHashes[2]); - Cas.InsertChunk(Chunks[3], ChunkHashes[3]); - Cas.InsertChunk(Chunks[4], ChunkHashes[4]); - Cas.InsertChunk(Chunks[5], ChunkHashes[5]); - Cas.InsertChunk(Chunks[6], ChunkHashes[6]); - Cas.InsertChunk(Chunks[7], ChunkHashes[7]); - } - - // Keep mixed - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - std::vector<IoHash> KeepChunks; - KeepChunks.push_back(ChunkHashes[1]); - KeepChunks.push_back(ChunkHashes[4]); - KeepChunks.push_back(ChunkHashes[7]); - GcCtx.AddRetainedCids(KeepChunks); - - Cas.Flush(); - Cas.CollectGarbage(GcCtx); - - CHECK(!Cas.HaveChunk(ChunkHashes[0])); - CHECK(Cas.HaveChunk(ChunkHashes[1])); - CHECK(!Cas.HaveChunk(ChunkHashes[2])); - CHECK(!Cas.HaveChunk(ChunkHashes[3])); - CHECK(Cas.HaveChunk(ChunkHashes[4])); - CHECK(!Cas.HaveChunk(ChunkHashes[5])); - CHECK(!Cas.HaveChunk(ChunkHashes[6])); - CHECK(Cas.HaveChunk(ChunkHashes[7])); - CHECK(!Cas.HaveChunk(ChunkHashes[8])); - - ValidateChunkExists(1); - ValidateChunkExists(4); - ValidateChunkExists(7); - - Cas.InsertChunk(Chunks[0], ChunkHashes[0]); - Cas.InsertChunk(Chunks[2], ChunkHashes[2]); - Cas.InsertChunk(Chunks[3], ChunkHashes[3]); - Cas.InsertChunk(Chunks[5], ChunkHashes[5]); - Cas.InsertChunk(Chunks[6], ChunkHashes[6]); - Cas.InsertChunk(Chunks[8], ChunkHashes[8]); - } - - // Keep multiple at end - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - std::vector<IoHash> KeepChunks; - KeepChunks.push_back(ChunkHashes[6]); - KeepChunks.push_back(ChunkHashes[7]); - KeepChunks.push_back(ChunkHashes[8]); - GcCtx.AddRetainedCids(KeepChunks); - - Cas.Flush(); - Cas.CollectGarbage(GcCtx); - - CHECK(!Cas.HaveChunk(ChunkHashes[0])); - CHECK(!Cas.HaveChunk(ChunkHashes[1])); - CHECK(!Cas.HaveChunk(ChunkHashes[2])); - CHECK(!Cas.HaveChunk(ChunkHashes[3])); - CHECK(!Cas.HaveChunk(ChunkHashes[4])); - CHECK(!Cas.HaveChunk(ChunkHashes[5])); - CHECK(Cas.HaveChunk(ChunkHashes[6])); - CHECK(Cas.HaveChunk(ChunkHashes[7])); - CHECK(Cas.HaveChunk(ChunkHashes[8])); - - ValidateChunkExists(6); - ValidateChunkExists(7); - ValidateChunkExists(8); - - Cas.InsertChunk(Chunks[0], ChunkHashes[0]); - Cas.InsertChunk(Chunks[1], ChunkHashes[1]); - Cas.InsertChunk(Chunks[2], ChunkHashes[2]); - Cas.InsertChunk(Chunks[3], ChunkHashes[3]); - Cas.InsertChunk(Chunks[4], ChunkHashes[4]); - Cas.InsertChunk(Chunks[5], ChunkHashes[5]); - } - - // Keep every other - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - std::vector<IoHash> KeepChunks; - KeepChunks.push_back(ChunkHashes[0]); - KeepChunks.push_back(ChunkHashes[2]); - KeepChunks.push_back(ChunkHashes[4]); - KeepChunks.push_back(ChunkHashes[6]); - KeepChunks.push_back(ChunkHashes[8]); - GcCtx.AddRetainedCids(KeepChunks); - - Cas.Flush(); - Cas.CollectGarbage(GcCtx); - - CHECK(Cas.HaveChunk(ChunkHashes[0])); - CHECK(!Cas.HaveChunk(ChunkHashes[1])); - CHECK(Cas.HaveChunk(ChunkHashes[2])); - CHECK(!Cas.HaveChunk(ChunkHashes[3])); - CHECK(Cas.HaveChunk(ChunkHashes[4])); - CHECK(!Cas.HaveChunk(ChunkHashes[5])); - CHECK(Cas.HaveChunk(ChunkHashes[6])); - CHECK(!Cas.HaveChunk(ChunkHashes[7])); - CHECK(Cas.HaveChunk(ChunkHashes[8])); - - ValidateChunkExists(0); - ValidateChunkExists(2); - ValidateChunkExists(4); - ValidateChunkExists(6); - ValidateChunkExists(8); - - Cas.InsertChunk(Chunks[1], ChunkHashes[1]); - Cas.InsertChunk(Chunks[3], ChunkHashes[3]); - Cas.InsertChunk(Chunks[5], ChunkHashes[5]); - Cas.InsertChunk(Chunks[7], ChunkHashes[7]); - } - - // Verify that we nicely appended blocks even after all GC operations - ValidateChunkExists(0); - ValidateChunkExists(1); - ValidateChunkExists(2); - ValidateChunkExists(3); - ValidateChunkExists(4); - ValidateChunkExists(5); - ValidateChunkExists(6); - ValidateChunkExists(7); - ValidateChunkExists(8); - } -} - -TEST_CASE("compactcas.gc.deleteblockonopen") -{ - ScopedTemporaryDirectory TempDir; - - uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84}; - std::vector<IoBuffer> Chunks; - Chunks.reserve(20); - for (uint64_t Size : ChunkSizes) - { - Chunks.push_back(CreateRandomBlob(Size)); - } - - std::vector<IoHash> ChunkHashes; - ChunkHashes.reserve(20); - for (const IoBuffer& Chunk : Chunks) - { - ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size())); - } - - { - GcManager Gc; - CasContainerStrategy Cas(Gc); - Cas.Initialize(TempDir.Path(), "test", 1024, 16, true); - - for (size_t i = 0; i < 20; i++) - { - CHECK(Cas.InsertChunk(Chunks[i], ChunkHashes[i]).New); - } - - // GC every other block - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - std::vector<IoHash> KeepChunks; - for (size_t i = 0; i < 20; i += 2) - { - KeepChunks.push_back(ChunkHashes[i]); - } - GcCtx.AddRetainedCids(KeepChunks); - - Cas.Flush(); - Cas.CollectGarbage(GcCtx); - - for (size_t i = 0; i < 20; i += 2) - { - CHECK(Cas.HaveChunk(ChunkHashes[i])); - CHECK(!Cas.HaveChunk(ChunkHashes[i + 1])); - CHECK(ChunkHashes[i] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[i]))); - } - } - } - { - // Re-open - GcManager Gc; - CasContainerStrategy Cas(Gc); - Cas.Initialize(TempDir.Path(), "test", 1024, 16, false); - - for (size_t i = 0; i < 20; i += 2) - { - CHECK(Cas.HaveChunk(ChunkHashes[i])); - CHECK(!Cas.HaveChunk(ChunkHashes[i + 1])); - CHECK(ChunkHashes[i] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[i]))); - } - } -} - -TEST_CASE("compactcas.gc.handleopeniobuffer") -{ - ScopedTemporaryDirectory TempDir; - - uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84}; - std::vector<IoBuffer> Chunks; - Chunks.reserve(20); - for (const uint64_t& Size : ChunkSizes) - { - Chunks.push_back(CreateRandomBlob(Size)); - } - - std::vector<IoHash> ChunkHashes; - ChunkHashes.reserve(20); - for (const IoBuffer& Chunk : Chunks) - { - ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size())); - } - - GcManager Gc; - CasContainerStrategy Cas(Gc); - Cas.Initialize(TempDir.Path(), "test", 1024, 16, true); - - for (size_t i = 0; i < 20; i++) - { - CHECK(Cas.InsertChunk(Chunks[i], ChunkHashes[i]).New); - } - - IoBuffer RetainChunk = Cas.FindChunk(ChunkHashes[5]); - Cas.Flush(); - - // GC everything - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - Cas.CollectGarbage(GcCtx); - - for (size_t i = 0; i < 20; i++) - { - CHECK(!Cas.HaveChunk(ChunkHashes[i])); - } - - CHECK(ChunkHashes[5] == IoHash::HashBuffer(RetainChunk)); -} - -TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType) +TEST_CASE("compactcas.threadedinsert") { // for (uint32_t i = 0; i < 100; ++i) { @@ -2004,56 +1438,41 @@ TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType) const tsl::robin_set<IoHash, IoHash::Hasher>& ChunksToDelete, const std::vector<IoHash>& KeepHashes, tsl::robin_set<IoHash, IoHash::Hasher>& GcChunkHashes) { - if (GCV2::Enabled) - { - std::atomic_bool IsCancelledFlag = false; - GcCtx Ctx = {.Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), - .CollectSmallObjects = true, - .IsDeleteMode = true, - .CompactBlockUsageThresholdPercent = 100}, - .IsCancelledFlag = IsCancelledFlag}; - GcReferenceStoreStats PrunerStats; - GcReferencePruner* Pruner = Cas.CreateReferencePruner(Ctx, PrunerStats); - CHECK(Pruner); - HashKeySet Deleted; - GcStats Stats; - GcStoreCompactor* Compactor = - Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::span<IoHash> { - std::vector<IoHash> Unreferenced; - HashKeySet Retain; - Retain.AddHashesToSet(KeepHashes); - for (const IoHash& ChunkHash : References) + std::atomic_bool IsCancelledFlag = false; + GcCtx Ctx = {.Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), + .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), + .CollectSmallObjects = true, + .IsDeleteMode = true, + .CompactBlockUsageThresholdPercent = 100}, + .IsCancelledFlag = IsCancelledFlag}; + GcReferenceStoreStats PrunerStats; + GcReferencePruner* Pruner = Cas.CreateReferencePruner(Ctx, PrunerStats); + CHECK(Pruner); + HashKeySet Deleted; + GcStats Stats; + GcStoreCompactor* Compactor = + Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::span<IoHash> { + std::vector<IoHash> Unreferenced; + HashKeySet Retain; + Retain.AddHashesToSet(KeepHashes); + for (const IoHash& ChunkHash : References) + { + if (!Retain.ContainsHash(ChunkHash)) { - if (!Retain.ContainsHash(ChunkHash)) - { - Unreferenced.push_back(ChunkHash); - } + Unreferenced.push_back(ChunkHash); } - Deleted.AddHashesToSet(Unreferenced); - return Unreferenced; - }); - if (Compactor) - { - Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) { - CHECK(ChunksToDelete.contains(ChunkHash)); - GcChunkHashes.erase(ChunkHash); - }); - GcCompactStoreStats CompactStats; - Compactor->CompactStore(Ctx, CompactStats, []() { return 0; }); - } - } - else + } + Deleted.AddHashesToSet(Unreferenced); + return Unreferenced; + }); + if (Compactor) { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - GcCtx.CollectSmallObjects(true); - GcCtx.AddRetainedCids(KeepHashes); - Cas.CollectGarbage(GcCtx); - const HashKeySet& Deleted = GcCtx.DeletedCids(); Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) { CHECK(ChunksToDelete.contains(ChunkHash)); GcChunkHashes.erase(ChunkHash); }); + GcCompactStoreStats CompactStats; + Compactor->CompactStore(Ctx, CompactStats, []() { return 0; }); } }; |