remove gc v1 (#121)

* kill gc v1
* block use of gc v1 from zen command line
* warn and flip to gcv2 if --gc-v2=false is specified for zenserver

1 files changed, 17 insertions, 283 deletions

diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index ac8b70c1c..d30bd93cc 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -127,14 +127,12 @@ ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const st
 
 	m_DiskLayer.DiscoverBuckets();
 
-	m_Gc.AddGcContributor(this);
 	m_Gc.AddGcStorage(this);
 }
 
 ZenCacheNamespace::~ZenCacheNamespace()
 {
 	m_Gc.RemoveGcStorage(this);
-	m_Gc.RemoveGcContributor(this);
 }
 
 struct ZenCacheNamespace::PutBatchHandle
@@ -307,26 +305,6 @@ ZenCacheNamespace::ScrubStorage(ScrubContext& Ctx)
 	m_DiskLayer.ScrubStorage(Ctx);
 }
 
-void
-ZenCacheNamespace::GatherReferences(GcContext& GcCtx)
-{
-	ZEN_TRACE_CPU("Z$::ZenCacheNamespace::GatherReferences");
-
-	Stopwatch  Timer;
-	const auto Guard =
-		MakeGuard([&] { ZEN_DEBUG("cache gathered all references from '{}' in {}", m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
-
-	m_DiskLayer.GatherReferences(GcCtx);
-}
-
-void
-ZenCacheNamespace::CollectGarbage(GcContext& GcCtx)
-{
-	ZEN_TRACE_CPU("Z$::Namespace::CollectGarbage");
-
-	m_DiskLayer.CollectGarbage(GcCtx);
-}
-
 GcStorageSize
 ZenCacheNamespace::StorageSize() const
 {
@@ -1452,186 +1430,7 @@ TEST_CASE("cachestore.size")
 	}
 }
 
-TEST_CASE("cachestore.gc")
-{
-	using namespace testutils;
-
-	auto JobQueue = MakeJobQueue(1, "testqueue");
-
-	SUBCASE("gather references does NOT add references for expired cache entries")
-	{
-		ScopedTemporaryDirectory TempDir;
-		std::vector<IoHash>		 Cids{CreateKey(1), CreateKey(2), CreateKey(3)};
-
-		const auto CollectAndFilter = [](GcManager&				 Gc,
-										 GcClock::TimePoint		 Time,
-										 GcClock::Duration		 MaxDuration,
-										 std::span<const IoHash> Cids,
-										 std::vector<IoHash>&	 OutKeep) {
-			GcContext GcCtx(Time - MaxDuration, Time - MaxDuration);
-			Gc.CollectGarbage(GcCtx);
-			OutKeep.clear();
-			GcCtx.FilterCids(Cids, [&OutKeep](const IoHash& Hash) { OutKeep.push_back(Hash); });
-		};
-
-		{
-			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
-			const auto		  Bucket = "teardrinker"sv;
-
-			// Create a cache record
-			const IoHash   Key = CreateKey(42);
-			CbObjectWriter Record;
-			Record << "Key"sv
-				   << "SomeRecord"sv;
-
-			for (size_t Idx = 0; auto& Cid : Cids)
-			{
-				Record.AddBinaryAttachment(fmt::format("attachment-{}", Idx++), Cid);
-			}
-
-			IoBuffer Buffer = Record.Save().GetBuffer().AsIoBuffer();
-			Buffer.SetContentType(ZenContentType::kCbObject);
-
-			Zcs.Put(Bucket, Key, {.Value = Buffer}, Cids);
-
-			std::vector<IoHash> Keep;
-
-			// Collect garbage with 1 hour max cache duration
-			{
-				CollectAndFilter(Gc, GcClock::Now(), std::chrono::hours(1), Cids, Keep);
-				CHECK_EQ(Cids.size(), Keep.size());
-			}
-
-			// Move forward in time
-			{
-				CollectAndFilter(Gc, GcClock::Now() + std::chrono::hours(2), std::chrono::hours(1), Cids, Keep);
-				CHECK_EQ(0, Keep.size());
-			}
-		}
-
-		// Expect timestamps to be serialized
-		{
-			GcManager			Gc;
-			ZenCacheNamespace	Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
-			std::vector<IoHash> Keep;
-
-			// Collect garbage with 1 hour max cache duration
-			{
-				CollectAndFilter(Gc, GcClock::Now(), std::chrono::hours(1), Cids, Keep);
-				CHECK_EQ(3, Keep.size());
-			}
-
-			// Move forward in time
-			{
-				CollectAndFilter(Gc, GcClock::Now() + std::chrono::hours(2), std::chrono::hours(1), Cids, Keep);
-				CHECK_EQ(0, Keep.size());
-			}
-		}
-	}
-
-	SUBCASE("gc removes standalone values")
-	{
-		ScopedTemporaryDirectory TempDir;
-		GcManager				 Gc;
-		ZenCacheNamespace		 Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
-		const auto				 Bucket		 = "fortysixandtwo"sv;
-		const GcClock::TimePoint CurrentTime = GcClock::Now();
-
-		std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
-
-		for (const auto& Key : Keys)
-		{
-			IoBuffer Value = CreateRandomBlob(128 << 10);
-			Zcs.Put(Bucket, Key, {.Value = Value}, {});
-		}
-
-		{
-			GcContext GcCtx(CurrentTime - std::chrono::hours(46), CurrentTime - std::chrono::hours(46));
-
-			Gc.CollectGarbage(GcCtx);
-
-			for (const auto& Key : Keys)
-			{
-				ZenCacheValue CacheValue;
-				const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
-				CHECK(Exists);
-			}
-		}
-
-		// Move forward in time and collect again
-		{
-			GcContext GcCtx(CurrentTime + std::chrono::minutes(2), CurrentTime + std::chrono::minutes(2));
-			Gc.CollectGarbage(GcCtx);
-
-			for (const auto& Key : Keys)
-			{
-				ZenCacheValue CacheValue;
-				const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
-				CHECK(!Exists);
-			}
-
-			CHECK_EQ(0, Zcs.StorageSize().DiskSize);
-		}
-	}
-
-	SUBCASE("gc removes small objects")
-	{
-		ScopedTemporaryDirectory TempDir;
-		GcManager				 Gc;
-		{
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
-			const auto		  Bucket = "rightintwo"sv;
-
-			std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
-
-			for (const auto& Key : Keys)
-			{
-				IoBuffer Value = CreateRandomBlob(128);
-				Zcs.Put(Bucket, Key, {.Value = Value}, {});
-			}
-
-			{
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(2), GcClock::Now() - std::chrono::hours(2));
-				GcCtx.CollectSmallObjects(true);
-
-				Gc.CollectGarbage(GcCtx);
-
-				for (const auto& Key : Keys)
-				{
-					ZenCacheValue CacheValue;
-					const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
-					CHECK(Exists);
-				}
-			}
-
-			// Move forward in time and collect again
-			{
-				GcContext GcCtx(GcClock::Now() + std::chrono::minutes(2), GcClock::Now() + std::chrono::minutes(2));
-				GcCtx.CollectSmallObjects(true);
-
-				Zcs.Flush();
-				Gc.CollectGarbage(GcCtx);
-
-				for (const auto& Key : Keys)
-				{
-					ZenCacheValue CacheValue;
-					const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
-					CHECK(!Exists);
-				}
-				// GC could not remove the currently written block so size will not be zero
-				CHECK_NE(0, Zcs.StorageSize().DiskSize);
-			}
-		}
-		{
-			// Unreferenced blocks will be pruned so size should now be zero
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
-			CHECK_EQ(0, Zcs.StorageSize().DiskSize);
-		}
-	}
-}
-
-TEST_CASE_TEMPLATE("cachestore.threadedinsert", GCV2, FalseType, TrueType)	// * doctest::skip(true))
+TEST_CASE("cachestore.threadedinsert")	// * doctest::skip(true))
 {
 	//	for (uint32_t i = 0; i < 100; ++i)
 	{
@@ -1699,39 +1498,24 @@ TEST_CASE_TEMPLATE("cachestore.threadedinsert", GCV2, FalseType, TrueType)	// *
 			}
 		}
 
-		auto DoGC = [](GcManager&												Gc,
-					   ZenCacheNamespace&										Zcs,
-					   std::unordered_map<IoHash, std::string, IoHash::Hasher>& GcChunkHashes,
-					   const std::vector<IoHash>&								KeepHashes) {
-			if (GCV2::Enabled)
+		auto DoGC = [](GcManager& Gc, ZenCacheNamespace& Zcs, std::unordered_map<IoHash, std::string, IoHash::Hasher>& GcChunkHashes) {
+			GcSettings Settings = {.CacheExpireTime					  = GcClock::Now() - std::chrono::hours(24),
+								   .ProjectStoreExpireTime			  = GcClock::Now() - std::chrono::hours(24),
+								   .CollectSmallObjects				  = true,
+								   .IsDeleteMode					  = true,
+								   .CompactBlockUsageThresholdPercent = 100};
+			Gc.CollectGarbage(Settings);
+			// Cheating as we don't get the list of deleted hashes back from this call
+			std::unordered_map<IoHash, std::string, IoHash::Hasher> RemainingChunkHashes;
+			for (const auto& It : GcChunkHashes)
 			{
-				GcSettings Settings = {.CacheExpireTime					  = GcClock::Now() - std::chrono::hours(24),
-									   .ProjectStoreExpireTime			  = GcClock::Now() - std::chrono::hours(24),
-									   .CollectSmallObjects				  = true,
-									   .IsDeleteMode					  = true,
-									   .CompactBlockUsageThresholdPercent = 100};
-				Gc.CollectGarbage(Settings);
-				// Cheating as we don't get the list of deleted hashes back from this call
-				std::unordered_map<IoHash, std::string, IoHash::Hasher> RemainingChunkHashes;
-				for (const auto& It : GcChunkHashes)
+				ZenCacheValue Tmp;
+				if (Zcs.Get(It.second, It.first, Tmp))
 				{
-					ZenCacheValue Tmp;
-					if (Zcs.Get(It.second, It.first, Tmp))
-					{
-						RemainingChunkHashes.insert(It);
-					}
+					RemainingChunkHashes.insert(It);
 				}
-				GcChunkHashes.swap(RemainingChunkHashes);
-			}
-			else
-			{
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-				GcCtx.CollectSmallObjects(true);
-				GcCtx.AddRetainedCids(KeepHashes);
-				Zcs.CollectGarbage(GcCtx);
-				const HashKeySet& Deleted = GcCtx.DeletedCids();
-				Deleted.IterateHashes([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
 			}
+			GcChunkHashes.swap(RemainingChunkHashes);
 		};
 
 		const uint64_t TotalSize = Zcs.StorageSize().DiskSize;
@@ -1813,32 +1597,7 @@ TEST_CASE_TEMPLATE("cachestore.threadedinsert", GCV2, FalseType, TrueType)	// *
 						GcChunkHashes[Chunk.first] = Chunk.second.Bucket;
 					}
 				}
-				std::vector<IoHash> KeepHashes;
-				KeepHashes.reserve(GcChunkHashes.size());
-				for (const auto& Entry : GcChunkHashes)
-				{
-					KeepHashes.push_back(Entry.first);
-				}
-				size_t C = 0;
-				while (C < KeepHashes.size())
-				{
-					if (C % 155 == 0)
-					{
-						if (C < KeepHashes.size() - 1)
-						{
-							KeepHashes[C] = KeepHashes[KeepHashes.size() - 1];
-							KeepHashes.pop_back();
-						}
-						if (C + 3 < KeepHashes.size() - 1)
-						{
-							KeepHashes[C + 3] = KeepHashes[KeepHashes.size() - 1];
-							KeepHashes.pop_back();
-						}
-					}
-					C++;
-				}
-
-				DoGC(Gc, Zcs, GcChunkHashes, KeepHashes);
+				DoGC(Gc, Zcs, GcChunkHashes);
 			}
 
 			while (WorkCompleted < NewChunks.size() + Chunks.size())
@@ -1856,32 +1615,7 @@ TEST_CASE_TEMPLATE("cachestore.threadedinsert", GCV2, FalseType, TrueType)	// *
 						GcChunkHashes[Chunk.first] = Chunk.second.Bucket;
 					}
 				}
-				std::vector<IoHash> KeepHashes;
-				KeepHashes.reserve(GcChunkHashes.size());
-				for (const auto& Entry : GcChunkHashes)
-				{
-					KeepHashes.push_back(Entry.first);
-				}
-				size_t C = 0;
-				while (C < KeepHashes.size())
-				{
-					if (C % 155 == 0)
-					{
-						if (C < KeepHashes.size() - 1)
-						{
-							KeepHashes[C] = KeepHashes[KeepHashes.size() - 1];
-							KeepHashes.pop_back();
-						}
-						if (C + 3 < KeepHashes.size() - 1)
-						{
-							KeepHashes[C + 3] = KeepHashes[KeepHashes.size() - 1];
-							KeepHashes.pop_back();
-						}
-					}
-					C++;
-				}
-
-				DoGC(Gc, Zcs, GcChunkHashes, KeepHashes);
+				DoGC(Gc, Zcs, GcChunkHashes);
 			}
 		}
 		{