add separate PreCache step for GcReferenceChecker (#578)

- Improvement: GCv2: Use separate PreCache step to improve concurrency when checking references
- Improvement: GCv2: Improved verbose logging
- Improvement: GCv2: Sort chunks to read by block/offset when finding references
- Improvement: GCv2: Exit as soon as no more unreferenced items are left

1 files changed, 66 insertions, 33 deletions

diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 25dfd103d..9155e209c 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -816,16 +816,28 @@ namespace testutils {
 		return {Key, Buffer};
 	}
 
+	struct FalseType
+	{
+		static const bool Enabled = false;
+	};
+	struct TrueType
+	{
+		static const bool Enabled = true;
+	};
+
 }  // namespace testutils
 
-TEST_CASE("z$.store")
+TEST_CASE_TEMPLATE("z$.store", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	ScopedTemporaryDirectory TempDir;
 
 	GcManager Gc;
 
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+	ZenCacheNamespace Zcs(Gc,
+						  *JobQueue,
+						  TempDir.Path() / "cache",
+						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 	const int kIterationCount = 100;
 
@@ -859,7 +871,7 @@ TEST_CASE("z$.store")
 	}
 }
 
-TEST_CASE("z$.size")
+TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	auto JobQueue = MakeJobQueue(1, "testqueue");
 
@@ -881,7 +893,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			CbObject CacheValue = CreateCacheValue(Zcs.GetConfig().DiskLayerConfig.BucketConfig.MemCacheSizeThreshold - 256);
 
@@ -915,7 +930,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -939,7 +957,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			CbObject CacheValue = CreateCacheValue(Zcs.GetConfig().DiskLayerConfig.BucketConfig.MemCacheSizeThreshold + 64);
 
@@ -959,7 +980,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -974,7 +998,7 @@ TEST_CASE("z$.size")
 	}
 }
 
-TEST_CASE("z$.gc")
+TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	using namespace testutils;
 
@@ -1001,7 +1025,7 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace Zcs(Gc,
 								  *JobQueue,
 								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			const auto		  Bucket = "teardrinker"sv;
 
 			// Create a cache record
@@ -1041,7 +1065,7 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace	Zcs(Gc,
 									*JobQueue,
 									TempDir.Path() / "cache",
-									{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+									{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			std::vector<IoHash> Keep;
 
 			// Collect garbage with 1 hour max cache duration
@@ -1065,7 +1089,7 @@ TEST_CASE("z$.gc")
 		ZenCacheNamespace		 Zcs(Gc,
 								 *JobQueue,
 								 TempDir.Path() / "cache",
-								 {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								 {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		const auto				 Bucket		 = "fortysixandtwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -1114,7 +1138,7 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace Zcs(Gc,
 								  *JobQueue,
 								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			const auto		  Bucket = "rightintwo"sv;
 
 			std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
@@ -1162,13 +1186,13 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace Zcs(Gc,
 								  *JobQueue,
 								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			CHECK_EQ(0, Zcs.StorageSize().DiskSize);
 		}
 	}
 }
 
-TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
+TEST_CASE_TEMPLATE("z$.threadedinsert", ReferenceCaching, testutils::FalseType, testutils::TrueType)  // * doctest::skip(true))
 {
 	//	for (uint32_t i = 0; i < 100; ++i)
 	{
@@ -1219,7 +1243,10 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 		WorkerThreadPool  ThreadPool(4);
 		GcManager		  Gc;
 		auto			  JobQueue = MakeJobQueue(1, "testqueue");
-		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path(), {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path(),
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;
@@ -1648,7 +1675,7 @@ TEST_CASE("z$.drop.namespace")
 	}
 }
 
-TEST_CASE("z$.blocked.disklayer.put")
+TEST_CASE_TEMPLATE("z$.blocked.disklayer.put", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	ScopedTemporaryDirectory TempDir;
 
@@ -1665,7 +1692,10 @@ TEST_CASE("z$.blocked.disklayer.put")
 
 	GcManager		  Gc;
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+	ZenCacheNamespace Zcs(Gc,
+						  *JobQueue,
+						  TempDir.Path() / "cache",
+						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 	CbObject CacheValue = CreateCacheValue(64 * 1024 + 64);
 
@@ -1701,7 +1731,7 @@ TEST_CASE("z$.blocked.disklayer.put")
 	CHECK(memcmp(NewView.GetData(), Buffer2.GetData(), NewView.GetSize()) == 0);
 }
 
-TEST_CASE("z$.scrub")
+TEST_CASE_TEMPLATE("z$.scrub", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	ScopedTemporaryDirectory TempDir;
 
@@ -1760,7 +1790,10 @@ TEST_CASE("z$.scrub")
 	GcManager			  Gc;
 	CidStore			  CidStore(Gc);
 	auto				  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace	  Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+	ZenCacheNamespace	  Zcs(Gc,
+						  *JobQueue,
+						  TempDir.Path() / "cache",
+						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
@@ -1795,7 +1828,7 @@ TEST_CASE("z$.scrub")
 	CHECK(ScrubCtx.BadCids().GetSize() == 0);
 }
 
-TEST_CASE("z$.newgc.basics")
+TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	using namespace testutils;
 
@@ -1915,7 +1948,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 		// Create some basic data
 		{
@@ -1949,7 +1982,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(1),
@@ -1983,7 +2016,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2017,7 +2050,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2051,7 +2084,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2086,7 +2119,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2121,7 +2154,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2162,7 +2195,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2198,7 +2231,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
@@ -2238,7 +2271,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
@@ -2277,7 +2310,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
@@ -2317,7 +2350,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		// Prime so we can check GC of memory layer
@@ -2370,7 +2403,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		auto Attachments =