aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore/compactcas.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2024-10-03 16:42:57 +0200
committerGitHub Enterprise <[email protected]>2024-10-03 16:42:57 +0200
commitb13b5f48bb497aaf9f9f3d74aceb6e474cf12898 (patch)
tree24b1ed63ece11fb773a0ecf41ce6308969468198 /src/zenstore/compactcas.cpp
parent5.5.9-pre0 (diff)
downloadzen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.tar.xz
zen-b13b5f48bb497aaf9f9f3d74aceb6e474cf12898.zip
remove gc v1 (#121)
* kill gc v1 * block use of gc v1 from zen command line * warn and flip to gcv2 if --gc-v2=false is specified for zenserver
Diffstat (limited to 'src/zenstore/compactcas.cpp')
-rw-r--r--src/zenstore/compactcas.cpp639
1 files changed, 29 insertions, 610 deletions
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index a5d70a991..7f1300177 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -527,167 +527,6 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
ZEN_INFO("scrubbed {} chunks ({}) in '{}'", ChunkCount.load(), NiceBytes(ChunkBytes.load()), m_RootDirectory / m_ContainerBaseName);
}
-void
-CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
-{
- ZEN_TRACE_CPU("CasContainer::CollectGarbage");
-
- if (GcCtx.SkipCid())
- {
- return;
- }
-
- // It collects all the blocks that we want to delete chunks from. For each such
- // block we keep a list of chunks to retain and a list of chunks to delete.
- //
- // If there is a block that we are currently writing to, that block is omitted
- // from the garbage collection.
- //
- // Next it will iterate over all blocks that we want to remove chunks from.
- // If the block is empty after removal of chunks we mark the block as pending
- // delete - we want to delete it as soon as there are no IoBuffers using the
- // block file.
- // Once complete we update the m_LocationMap by removing the chunks.
- //
- // If the block is non-empty we write out the chunks we want to keep to a new
- // block file (creating new block files as needed).
- //
- // We update the index as we complete each new block file. This makes it possible
- // to break the GC if we want to limit time for execution.
- //
- // GC can very parallell to regular operation - it will block while taking
- // a snapshot of the current m_LocationMap state and while moving blocks it will
- // do a blocking operation and update the m_LocationMap after each new block is
- // written and figuring out the path to the next new block.
-
- ZEN_DEBUG("collecting garbage from '{}'", m_RootDirectory / m_ContainerBaseName);
-
- uint64_t WriteBlockTimeUs = 0;
- uint64_t WriteBlockLongestTimeUs = 0;
- uint64_t ReadBlockTimeUs = 0;
- uint64_t ReadBlockLongestTimeUs = 0;
-
- LocationMap_t LocationMap;
- std::vector<BlockStoreDiskLocation> Locations;
- BlockStore::ReclaimSnapshotState BlockStoreState;
- {
- ZEN_TRACE_CPU("CasContainer::CollectGarbage::State");
-
- RwLock::SharedLockScope ___(m_LocationMapLock);
- Stopwatch Timer;
- const auto ____ = MakeGuard([&Timer, &ReadBlockTimeUs, &ReadBlockLongestTimeUs] {
- uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
- ReadBlockTimeUs += ElapsedUs;
- ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
- });
- LocationMap = m_LocationMap;
- Locations = m_Locations;
- BlockStoreState = m_BlockStore.GetReclaimSnapshotState();
- }
-
- uint64_t TotalChunkCount = LocationMap.size();
-
- std::vector<IoHash> TotalChunkHashes;
- TotalChunkHashes.reserve(TotalChunkCount);
- for (const auto& Entry : LocationMap)
- {
- TotalChunkHashes.push_back(Entry.first);
- }
-
- std::vector<BlockStoreLocation> ChunkLocations;
- BlockStore::ChunkIndexArray KeepChunkIndexes;
- std::vector<IoHash> ChunkIndexToChunkHash;
- ChunkLocations.reserve(TotalChunkCount);
- KeepChunkIndexes.reserve(TotalChunkCount);
- ChunkIndexToChunkHash.reserve(TotalChunkCount);
-
- {
- ZEN_TRACE_CPU("CasContainer::CollectGarbage::Filter");
- GcCtx.FilterCids(TotalChunkHashes, [&](const IoHash& ChunkHash, bool Keep) {
- auto KeyIt = LocationMap.find(ChunkHash);
- const BlockStoreDiskLocation& DiskLocation = Locations[KeyIt->second];
- BlockStoreLocation Location = DiskLocation.Get(m_PayloadAlignment);
- size_t ChunkIndex = ChunkLocations.size();
-
- ChunkLocations.push_back(Location);
- ChunkIndexToChunkHash.push_back(ChunkHash);
- if (Keep)
- {
- KeepChunkIndexes.push_back(ChunkIndex);
- }
- });
- }
-
- const bool PerformDelete = GcCtx.IsDeletionMode() && GcCtx.CollectSmallObjects();
- if (!PerformDelete)
- {
- m_BlockStore.ReclaimSpace(BlockStoreState, ChunkLocations, KeepChunkIndexes, m_PayloadAlignment, true);
- return;
- }
-
- std::vector<IoHash> DeletedChunks;
- m_BlockStore.ReclaimSpace(
- BlockStoreState,
- ChunkLocations,
- KeepChunkIndexes,
- m_PayloadAlignment,
- false,
- [&](const BlockStore::MovedChunksArray& MovedChunks, const BlockStore::ChunkIndexArray& RemovedChunks) {
- std::vector<CasDiskIndexEntry> LogEntries;
- LogEntries.reserve(MovedChunks.size() + RemovedChunks.size());
- {
- RwLock::ExclusiveLockScope __(m_LocationMapLock);
- Stopwatch Timer;
- const auto ____ = MakeGuard([&] {
- uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
- WriteBlockTimeUs += ElapsedUs;
- WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
- });
- for (const auto& Entry : MovedChunks)
- {
- size_t ChunkIndex = Entry.first;
- const BlockStoreLocation& NewLocation = Entry.second;
- const IoHash& ChunkHash = ChunkIndexToChunkHash[ChunkIndex];
- size_t LocationIndex = m_LocationMap[ChunkHash];
- BlockStoreDiskLocation& Location = m_Locations[LocationIndex];
- if (Locations[LocationMap[ChunkHash]] != Location)
- {
- // Entry has been updated while GC was running, ignore the move
- continue;
- }
- Location = {NewLocation, m_PayloadAlignment};
- LogEntries.push_back({.Key = ChunkHash, .Location = Location});
- }
- for (const size_t ChunkIndex : RemovedChunks)
- {
- const IoHash& ChunkHash = ChunkIndexToChunkHash[ChunkIndex];
- size_t LocationIndex = m_LocationMap[ChunkHash];
- const BlockStoreDiskLocation& Location = Locations[LocationIndex];
- if (Locations[LocationMap[ChunkHash]] != Location)
- {
- // Entry has been updated while GC was running, ignore the delete
- continue;
- }
- LogEntries.push_back({.Key = ChunkHash, .Location = Location, .Flags = CasDiskIndexEntry::kTombstone});
- m_LocationMap.erase(ChunkHash);
- DeletedChunks.push_back(ChunkHash);
- }
- }
-
- m_CasLog.Append(LogEntries);
- m_CasLog.Flush();
- },
- [&GcCtx]() { return GcCtx.ClaimGCReserve(); });
-
- if (!DeletedChunks.empty())
- {
- // Clean up m_Locations vectors
- RwLock::ExclusiveLockScope IndexLock(m_LocationMapLock);
- CompactIndex(IndexLock);
- }
- GcCtx.AddDeletedCids(DeletedChunks);
-}
-
class CasContainerStoreCompactor : public GcStoreCompactor
{
public:
@@ -1456,412 +1295,7 @@ TEST_CASE("compactcas.compact.totalsize")
}
}
-TEST_CASE("compactcas.gc.basic")
-{
- ScopedTemporaryDirectory TempDir;
-
- GcManager Gc;
- CasContainerStrategy Cas(Gc);
- Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, true);
-
- IoBuffer Chunk = CreateRandomBlob(128);
- IoHash ChunkHash = IoHash::HashBuffer(Chunk);
-
- const CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, ChunkHash);
- CHECK(InsertResult.New);
- Cas.Flush();
-
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
-
- Cas.CollectGarbage(GcCtx);
-
- CHECK(!Cas.HaveChunk(ChunkHash));
-}
-
-TEST_CASE("compactcas.gc.removefile")
-{
- ScopedTemporaryDirectory TempDir;
-
- IoBuffer Chunk = CreateRandomBlob(128);
- IoHash ChunkHash = IoHash::HashBuffer(Chunk);
- {
- GcManager Gc;
- CasContainerStrategy Cas(Gc);
- Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, true);
-
- const CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, ChunkHash);
- CHECK(InsertResult.New);
- const CasStore::InsertResult InsertResultDup = Cas.InsertChunk(Chunk, ChunkHash);
- CHECK(!InsertResultDup.New);
- Cas.Flush();
- }
-
- GcManager Gc;
- CasContainerStrategy Cas(Gc);
- Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, false);
-
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
-
- Cas.CollectGarbage(GcCtx);
-
- CHECK(!Cas.HaveChunk(ChunkHash));
-}
-
-TEST_CASE("compactcas.gc.compact")
-{
- {
- ScopedTemporaryDirectory TempDir;
-
- GcManager Gc;
- CasContainerStrategy Cas(Gc);
- Cas.Initialize(TempDir.Path(), "cb", 2048, 1 << 4, true);
-
- uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5};
- std::vector<IoBuffer> Chunks;
- Chunks.reserve(9);
- for (uint64_t Size : ChunkSizes)
- {
- Chunks.push_back(CreateRandomBlob(Size));
- }
-
- std::vector<IoHash> ChunkHashes;
- ChunkHashes.reserve(9);
- for (const IoBuffer& Chunk : Chunks)
- {
- ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
- }
-
- CHECK(Cas.InsertChunk(Chunks[0], ChunkHashes[0]).New);
- CHECK(Cas.InsertChunk(Chunks[1], ChunkHashes[1]).New);
- CHECK(Cas.InsertChunk(Chunks[2], ChunkHashes[2]).New);
- CHECK(Cas.InsertChunk(Chunks[3], ChunkHashes[3]).New);
- CHECK(Cas.InsertChunk(Chunks[4], ChunkHashes[4]).New);
- CHECK(Cas.InsertChunk(Chunks[5], ChunkHashes[5]).New);
- CHECK(Cas.InsertChunk(Chunks[6], ChunkHashes[6]).New);
- CHECK(Cas.InsertChunk(Chunks[7], ChunkHashes[7]).New);
- CHECK(Cas.InsertChunk(Chunks[8], ChunkHashes[8]).New);
-
- CHECK(Cas.HaveChunk(ChunkHashes[0]));
- CHECK(Cas.HaveChunk(ChunkHashes[1]));
- CHECK(Cas.HaveChunk(ChunkHashes[2]));
- CHECK(Cas.HaveChunk(ChunkHashes[3]));
- CHECK(Cas.HaveChunk(ChunkHashes[4]));
- CHECK(Cas.HaveChunk(ChunkHashes[5]));
- CHECK(Cas.HaveChunk(ChunkHashes[6]));
- CHECK(Cas.HaveChunk(ChunkHashes[7]));
- CHECK(Cas.HaveChunk(ChunkHashes[8]));
-
- auto ValidateChunkExists = [&](size_t Index) {
- IoBuffer Chunk = Cas.FindChunk(ChunkHashes[Index]);
- bool Exists = !!Chunk;
- CHECK(Exists);
- IoHash Hash = IoHash::HashBuffer(Chunk);
- if (ChunkHashes[Index] != Hash)
- {
- CHECK(fmt::format("{}", ChunkHashes[Index]) == fmt::format("{}", Hash));
- }
- };
-
- // Keep first and last
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
-
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[0]);
- KeepChunks.push_back(ChunkHashes[8]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- Cas.Flush();
- Cas.CollectGarbage(GcCtx);
-
- CHECK(Cas.HaveChunk(ChunkHashes[0]));
- CHECK(!Cas.HaveChunk(ChunkHashes[1]));
- CHECK(!Cas.HaveChunk(ChunkHashes[2]));
- CHECK(!Cas.HaveChunk(ChunkHashes[3]));
- CHECK(!Cas.HaveChunk(ChunkHashes[4]));
- CHECK(!Cas.HaveChunk(ChunkHashes[5]));
- CHECK(!Cas.HaveChunk(ChunkHashes[6]));
- CHECK(!Cas.HaveChunk(ChunkHashes[7]));
- CHECK(Cas.HaveChunk(ChunkHashes[8]));
-
- ValidateChunkExists(0);
- ValidateChunkExists(8);
-
- Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
- Cas.InsertChunk(Chunks[2], ChunkHashes[2]);
- Cas.InsertChunk(Chunks[3], ChunkHashes[3]);
- Cas.InsertChunk(Chunks[4], ChunkHashes[4]);
- Cas.InsertChunk(Chunks[5], ChunkHashes[5]);
- Cas.InsertChunk(Chunks[6], ChunkHashes[6]);
- Cas.InsertChunk(Chunks[7], ChunkHashes[7]);
- }
-
- // Keep last
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[8]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- Cas.Flush();
- Cas.CollectGarbage(GcCtx);
-
- CHECK(!Cas.HaveChunk(ChunkHashes[0]));
- CHECK(!Cas.HaveChunk(ChunkHashes[1]));
- CHECK(!Cas.HaveChunk(ChunkHashes[2]));
- CHECK(!Cas.HaveChunk(ChunkHashes[3]));
- CHECK(!Cas.HaveChunk(ChunkHashes[4]));
- CHECK(!Cas.HaveChunk(ChunkHashes[5]));
- CHECK(!Cas.HaveChunk(ChunkHashes[6]));
- CHECK(!Cas.HaveChunk(ChunkHashes[7]));
- CHECK(Cas.HaveChunk(ChunkHashes[8]));
-
- ValidateChunkExists(8);
-
- Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
- Cas.InsertChunk(Chunks[2], ChunkHashes[2]);
- Cas.InsertChunk(Chunks[3], ChunkHashes[3]);
- Cas.InsertChunk(Chunks[4], ChunkHashes[4]);
- Cas.InsertChunk(Chunks[5], ChunkHashes[5]);
- Cas.InsertChunk(Chunks[6], ChunkHashes[6]);
- Cas.InsertChunk(Chunks[7], ChunkHashes[7]);
- }
-
- // Keep mixed
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[1]);
- KeepChunks.push_back(ChunkHashes[4]);
- KeepChunks.push_back(ChunkHashes[7]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- Cas.Flush();
- Cas.CollectGarbage(GcCtx);
-
- CHECK(!Cas.HaveChunk(ChunkHashes[0]));
- CHECK(Cas.HaveChunk(ChunkHashes[1]));
- CHECK(!Cas.HaveChunk(ChunkHashes[2]));
- CHECK(!Cas.HaveChunk(ChunkHashes[3]));
- CHECK(Cas.HaveChunk(ChunkHashes[4]));
- CHECK(!Cas.HaveChunk(ChunkHashes[5]));
- CHECK(!Cas.HaveChunk(ChunkHashes[6]));
- CHECK(Cas.HaveChunk(ChunkHashes[7]));
- CHECK(!Cas.HaveChunk(ChunkHashes[8]));
-
- ValidateChunkExists(1);
- ValidateChunkExists(4);
- ValidateChunkExists(7);
-
- Cas.InsertChunk(Chunks[0], ChunkHashes[0]);
- Cas.InsertChunk(Chunks[2], ChunkHashes[2]);
- Cas.InsertChunk(Chunks[3], ChunkHashes[3]);
- Cas.InsertChunk(Chunks[5], ChunkHashes[5]);
- Cas.InsertChunk(Chunks[6], ChunkHashes[6]);
- Cas.InsertChunk(Chunks[8], ChunkHashes[8]);
- }
-
- // Keep multiple at end
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[6]);
- KeepChunks.push_back(ChunkHashes[7]);
- KeepChunks.push_back(ChunkHashes[8]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- Cas.Flush();
- Cas.CollectGarbage(GcCtx);
-
- CHECK(!Cas.HaveChunk(ChunkHashes[0]));
- CHECK(!Cas.HaveChunk(ChunkHashes[1]));
- CHECK(!Cas.HaveChunk(ChunkHashes[2]));
- CHECK(!Cas.HaveChunk(ChunkHashes[3]));
- CHECK(!Cas.HaveChunk(ChunkHashes[4]));
- CHECK(!Cas.HaveChunk(ChunkHashes[5]));
- CHECK(Cas.HaveChunk(ChunkHashes[6]));
- CHECK(Cas.HaveChunk(ChunkHashes[7]));
- CHECK(Cas.HaveChunk(ChunkHashes[8]));
-
- ValidateChunkExists(6);
- ValidateChunkExists(7);
- ValidateChunkExists(8);
-
- Cas.InsertChunk(Chunks[0], ChunkHashes[0]);
- Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
- Cas.InsertChunk(Chunks[2], ChunkHashes[2]);
- Cas.InsertChunk(Chunks[3], ChunkHashes[3]);
- Cas.InsertChunk(Chunks[4], ChunkHashes[4]);
- Cas.InsertChunk(Chunks[5], ChunkHashes[5]);
- }
-
- // Keep every other
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[0]);
- KeepChunks.push_back(ChunkHashes[2]);
- KeepChunks.push_back(ChunkHashes[4]);
- KeepChunks.push_back(ChunkHashes[6]);
- KeepChunks.push_back(ChunkHashes[8]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- Cas.Flush();
- Cas.CollectGarbage(GcCtx);
-
- CHECK(Cas.HaveChunk(ChunkHashes[0]));
- CHECK(!Cas.HaveChunk(ChunkHashes[1]));
- CHECK(Cas.HaveChunk(ChunkHashes[2]));
- CHECK(!Cas.HaveChunk(ChunkHashes[3]));
- CHECK(Cas.HaveChunk(ChunkHashes[4]));
- CHECK(!Cas.HaveChunk(ChunkHashes[5]));
- CHECK(Cas.HaveChunk(ChunkHashes[6]));
- CHECK(!Cas.HaveChunk(ChunkHashes[7]));
- CHECK(Cas.HaveChunk(ChunkHashes[8]));
-
- ValidateChunkExists(0);
- ValidateChunkExists(2);
- ValidateChunkExists(4);
- ValidateChunkExists(6);
- ValidateChunkExists(8);
-
- Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
- Cas.InsertChunk(Chunks[3], ChunkHashes[3]);
- Cas.InsertChunk(Chunks[5], ChunkHashes[5]);
- Cas.InsertChunk(Chunks[7], ChunkHashes[7]);
- }
-
- // Verify that we nicely appended blocks even after all GC operations
- ValidateChunkExists(0);
- ValidateChunkExists(1);
- ValidateChunkExists(2);
- ValidateChunkExists(3);
- ValidateChunkExists(4);
- ValidateChunkExists(5);
- ValidateChunkExists(6);
- ValidateChunkExists(7);
- ValidateChunkExists(8);
- }
-}
-
-TEST_CASE("compactcas.gc.deleteblockonopen")
-{
- ScopedTemporaryDirectory TempDir;
-
- uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84};
- std::vector<IoBuffer> Chunks;
- Chunks.reserve(20);
- for (uint64_t Size : ChunkSizes)
- {
- Chunks.push_back(CreateRandomBlob(Size));
- }
-
- std::vector<IoHash> ChunkHashes;
- ChunkHashes.reserve(20);
- for (const IoBuffer& Chunk : Chunks)
- {
- ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
- }
-
- {
- GcManager Gc;
- CasContainerStrategy Cas(Gc);
- Cas.Initialize(TempDir.Path(), "test", 1024, 16, true);
-
- for (size_t i = 0; i < 20; i++)
- {
- CHECK(Cas.InsertChunk(Chunks[i], ChunkHashes[i]).New);
- }
-
- // GC every other block
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- for (size_t i = 0; i < 20; i += 2)
- {
- KeepChunks.push_back(ChunkHashes[i]);
- }
- GcCtx.AddRetainedCids(KeepChunks);
-
- Cas.Flush();
- Cas.CollectGarbage(GcCtx);
-
- for (size_t i = 0; i < 20; i += 2)
- {
- CHECK(Cas.HaveChunk(ChunkHashes[i]));
- CHECK(!Cas.HaveChunk(ChunkHashes[i + 1]));
- CHECK(ChunkHashes[i] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[i])));
- }
- }
- }
- {
- // Re-open
- GcManager Gc;
- CasContainerStrategy Cas(Gc);
- Cas.Initialize(TempDir.Path(), "test", 1024, 16, false);
-
- for (size_t i = 0; i < 20; i += 2)
- {
- CHECK(Cas.HaveChunk(ChunkHashes[i]));
- CHECK(!Cas.HaveChunk(ChunkHashes[i + 1]));
- CHECK(ChunkHashes[i] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[i])));
- }
- }
-}
-
-TEST_CASE("compactcas.gc.handleopeniobuffer")
-{
- ScopedTemporaryDirectory TempDir;
-
- uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84};
- std::vector<IoBuffer> Chunks;
- Chunks.reserve(20);
- for (const uint64_t& Size : ChunkSizes)
- {
- Chunks.push_back(CreateRandomBlob(Size));
- }
-
- std::vector<IoHash> ChunkHashes;
- ChunkHashes.reserve(20);
- for (const IoBuffer& Chunk : Chunks)
- {
- ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
- }
-
- GcManager Gc;
- CasContainerStrategy Cas(Gc);
- Cas.Initialize(TempDir.Path(), "test", 1024, 16, true);
-
- for (size_t i = 0; i < 20; i++)
- {
- CHECK(Cas.InsertChunk(Chunks[i], ChunkHashes[i]).New);
- }
-
- IoBuffer RetainChunk = Cas.FindChunk(ChunkHashes[5]);
- Cas.Flush();
-
- // GC everything
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- Cas.CollectGarbage(GcCtx);
-
- for (size_t i = 0; i < 20; i++)
- {
- CHECK(!Cas.HaveChunk(ChunkHashes[i]));
- }
-
- CHECK(ChunkHashes[5] == IoHash::HashBuffer(RetainChunk));
-}
-
-TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
+TEST_CASE("compactcas.threadedinsert")
{
// for (uint32_t i = 0; i < 100; ++i)
{
@@ -2004,56 +1438,41 @@ TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
const tsl::robin_set<IoHash, IoHash::Hasher>& ChunksToDelete,
const std::vector<IoHash>& KeepHashes,
tsl::robin_set<IoHash, IoHash::Hasher>& GcChunkHashes) {
- if (GCV2::Enabled)
- {
- std::atomic_bool IsCancelledFlag = false;
- GcCtx Ctx = {.Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24),
- .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
- .CollectSmallObjects = true,
- .IsDeleteMode = true,
- .CompactBlockUsageThresholdPercent = 100},
- .IsCancelledFlag = IsCancelledFlag};
- GcReferenceStoreStats PrunerStats;
- GcReferencePruner* Pruner = Cas.CreateReferencePruner(Ctx, PrunerStats);
- CHECK(Pruner);
- HashKeySet Deleted;
- GcStats Stats;
- GcStoreCompactor* Compactor =
- Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::span<IoHash> {
- std::vector<IoHash> Unreferenced;
- HashKeySet Retain;
- Retain.AddHashesToSet(KeepHashes);
- for (const IoHash& ChunkHash : References)
+ std::atomic_bool IsCancelledFlag = false;
+ GcCtx Ctx = {.Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24),
+ .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
+ .CollectSmallObjects = true,
+ .IsDeleteMode = true,
+ .CompactBlockUsageThresholdPercent = 100},
+ .IsCancelledFlag = IsCancelledFlag};
+ GcReferenceStoreStats PrunerStats;
+ GcReferencePruner* Pruner = Cas.CreateReferencePruner(Ctx, PrunerStats);
+ CHECK(Pruner);
+ HashKeySet Deleted;
+ GcStats Stats;
+ GcStoreCompactor* Compactor =
+ Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::span<IoHash> {
+ std::vector<IoHash> Unreferenced;
+ HashKeySet Retain;
+ Retain.AddHashesToSet(KeepHashes);
+ for (const IoHash& ChunkHash : References)
+ {
+ if (!Retain.ContainsHash(ChunkHash))
{
- if (!Retain.ContainsHash(ChunkHash))
- {
- Unreferenced.push_back(ChunkHash);
- }
+ Unreferenced.push_back(ChunkHash);
}
- Deleted.AddHashesToSet(Unreferenced);
- return Unreferenced;
- });
- if (Compactor)
- {
- Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
- CHECK(ChunksToDelete.contains(ChunkHash));
- GcChunkHashes.erase(ChunkHash);
- });
- GcCompactStoreStats CompactStats;
- Compactor->CompactStore(Ctx, CompactStats, []() { return 0; });
- }
- }
- else
+ }
+ Deleted.AddHashesToSet(Unreferenced);
+ return Unreferenced;
+ });
+ if (Compactor)
{
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- GcCtx.AddRetainedCids(KeepHashes);
- Cas.CollectGarbage(GcCtx);
- const HashKeySet& Deleted = GcCtx.DeletedCids();
Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
CHECK(ChunksToDelete.contains(ChunkHash));
GcChunkHashes.erase(ChunkHash);
});
+ GcCompactStoreStats CompactStats;
+ Compactor->CompactStore(Ctx, CompactStats, []() { return 0; });
}
};