improved block store logging and more gcv2 tests (#659)

* improved gc/blockstore logging
* more gcv2 tests

1 files changed, 96 insertions, 50 deletions

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 64c1dadf8..b308ddd4f 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -565,7 +565,7 @@ public:
 				return;
 			}
 			ZEN_INFO("GCV2: compactcas [COMPACT] '{}': RemovedDisk: {} in {}",
-					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+					 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 					 NiceBytes(Stats.RemovedDisk),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
@@ -629,7 +629,7 @@ public:
 						if (Ctx.Settings.Verbose)
 						{
 							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': compacting {} blocks",
-									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 									 BlocksToCompact.size());
 						}
 
@@ -669,14 +669,16 @@ public:
 								}
 								return true;
 							},
-							ClaimDiskReserveCallback);
+							ClaimDiskReserveCallback,
+							fmt::format("GCV2: compactcas [COMPACT] '{}': ",
+										m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName));
 					}
 					else
 					{
 						if (Ctx.Settings.Verbose)
 						{
 							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': skipped compacting of {} eligible blocks",
-									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 									 BlocksToCompact.size());
 						}
 					}
@@ -708,7 +710,7 @@ public:
 				return;
 			}
 			ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Checked: {}, Deleted: {}, FreedMemory: {} in {}",
-					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+					 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 					 Stats.CheckedCount,
 					 Stats.DeletedCount,
 					 NiceBytes(Stats.FreedMemory),
@@ -725,41 +727,43 @@ public:
 			return nullptr;
 		}
 
-		std::vector<CasDiskIndexEntry> ExpiredEntries;
-		ExpiredEntries.reserve(UnusedCids.size());
+		if (!Ctx.Settings.CollectSmallObjects)
+		{
+			return nullptr;
+		}
 
+		if (Ctx.Settings.IsDeleteMode)
 		{
-			RwLock::ExclusiveLockScope __(m_CasContainerStrategy.m_LocationMapLock);
-			if (Ctx.IsCancelledFlag.load())
-			{
-				return nullptr;
-			}
+			std::vector<CasDiskIndexEntry> ExpiredEntries;
+			ExpiredEntries.reserve(UnusedCids.size());
 
-			for (const IoHash& Cid : UnusedCids)
 			{
-				auto It = m_CasContainerStrategy.m_LocationMap.find(Cid);
-				if (It == m_CasContainerStrategy.m_LocationMap.end())
+				RwLock::ExclusiveLockScope __(m_CasContainerStrategy.m_LocationMapLock);
+				if (Ctx.IsCancelledFlag.load())
 				{
-					continue;
+					return nullptr;
 				}
-				if (Ctx.Settings.CollectSmallObjects)
+
+				for (const IoHash& Cid : UnusedCids)
 				{
-					CasDiskIndexEntry ExpiredEntry = {.Key		= Cid,
-													  .Location = m_CasContainerStrategy.m_Locations[It->second],
-													  .Flags	= CasDiskIndexEntry::kTombstone};
-					ExpiredEntries.push_back(ExpiredEntry);
+					if (auto It = m_CasContainerStrategy.m_LocationMap.find(Cid); It != m_CasContainerStrategy.m_LocationMap.end())
+					{
+						ExpiredEntries.push_back({.Key		= Cid,
+												  .Location = m_CasContainerStrategy.m_Locations[It->second],
+												  .Flags	= CasDiskIndexEntry::kTombstone});
+					}
 				}
-			}
 
-			if (Ctx.Settings.IsDeleteMode)
-			{
-				for (const CasDiskIndexEntry& Entry : ExpiredEntries)
+				if (!ExpiredEntries.empty())
 				{
-					m_CasContainerStrategy.m_LocationMap.erase(Entry.Key);
-					Stats.DeletedCount++;
+					for (const CasDiskIndexEntry& Entry : ExpiredEntries)
+					{
+						m_CasContainerStrategy.m_LocationMap.erase(Entry.Key);
+						Stats.DeletedCount++;
+					}
+					m_CasContainerStrategy.m_CasLog.Append(ExpiredEntries);
+					m_CasContainerStrategy.m_CasLog.Flush();
 				}
-				m_CasContainerStrategy.m_CasLog.Append(ExpiredEntries);
-				m_CasContainerStrategy.m_CasLog.Flush();
 			}
 		}
 
@@ -1695,9 +1699,9 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
 	CHECK(ChunkHashes[5] == IoHash::HashBuffer(RetainChunk));
 }
 
-TEST_CASE("compactcas.threadedinsert")
+TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
 {
-	//	for (uint32_t i = 0; i < 100; ++i)
+	// for (uint32_t i = 0; i < 100; ++i)
 	{
 		ScopedTemporaryDirectory TempDir;
 
@@ -1812,7 +1816,8 @@ TEST_CASE("compactcas.threadedinsert")
 
 			std::unordered_set<IoHash, IoHash::Hasher> ChunksToDelete;
 			std::vector<IoHash>						   KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end());
-			size_t									   C = 0;
+
+			size_t C = 0;
 			while (C < KeepHashes.size())
 			{
 				if (C % 155 == 0)
@@ -1833,17 +1838,66 @@ TEST_CASE("compactcas.threadedinsert")
 				C++;
 			}
 
+			auto DoGC = [](CasContainerStrategy&							 Cas,
+						   const std::unordered_set<IoHash, IoHash::Hasher>& ChunksToDelete,
+						   const std::vector<IoHash>&						 KeepHashes,
+						   std::unordered_set<IoHash, IoHash::Hasher>&		 GcChunkHashes) {
+				if (GCV2::Enabled)
+				{
+					std::atomic_bool	  IsCancelledFlag = false;
+					GcCtx				  Ctx			  = {.Settings		  = {.CacheExpireTime					= GcClock::Now() - std::chrono::hours(24),
+																				 .ProjectStoreExpireTime			= GcClock::Now() - std::chrono::hours(24),
+																				 .CollectSmallObjects				= true,
+																				 .IsDeleteMode						= true,
+																				 .CompactBlockUsageThresholdPercent = 100},
+															 .IsCancelledFlag = IsCancelledFlag};
+					GcReferenceStoreStats PrunerStats;
+					GcReferencePruner*	  Pruner = Cas.CreateReferencePruner(Ctx, PrunerStats);
+					CHECK(Pruner);
+					HashKeySet		  Deleted;
+					GcStats			  Stats;
+					GcStoreCompactor* Compactor =
+						Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::vector<IoHash> {
+							std::vector<IoHash> Unreferenced;
+							HashKeySet			Retain;
+							Retain.AddHashesToSet(KeepHashes);
+							for (const IoHash& ChunkHash : References)
+							{
+								if (!Retain.ContainsHash(ChunkHash))
+								{
+									Unreferenced.push_back(ChunkHash);
+								}
+							}
+							Deleted.AddHashesToSet(Unreferenced);
+							return Unreferenced;
+						});
+					if (Compactor)
+					{
+						Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
+							CHECK(ChunksToDelete.contains(ChunkHash));
+							GcChunkHashes.erase(ChunkHash);
+						});
+						GcCompactStoreStats CompactStats;
+						Compactor->CompactStore(Ctx, CompactStats, []() { return 0; });
+					}
+				}
+				else
+				{
+					GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
+					GcCtx.CollectSmallObjects(true);
+					GcCtx.AddRetainedCids(KeepHashes);
+					Cas.CollectGarbage(GcCtx);
+					const HashKeySet& Deleted = GcCtx.DeletedCids();
+					Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
+						CHECK(ChunksToDelete.contains(ChunkHash));
+						GcChunkHashes.erase(ChunkHash);
+					});
+				}
+			};
+
 			while (AddedChunkCount.load() < NewChunks.size())
 			{
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-				GcCtx.CollectSmallObjects(true);
-				GcCtx.AddRetainedCids(KeepHashes);
-				Cas.CollectGarbage(GcCtx);
-				const HashKeySet& Deleted = GcCtx.DeletedCids();
-				Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
-					CHECK(ChunksToDelete.contains(ChunkHash));
-					GcChunkHashes.erase(ChunkHash);
-				});
+				DoGC(Cas, ChunksToDelete, KeepHashes, GcChunkHashes);
 			}
 
 			while (WorkCompleted < NewChunks.size() + Chunks.size())
@@ -1851,15 +1905,7 @@ TEST_CASE("compactcas.threadedinsert")
 				Sleep(1);
 			}
 
-			GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-			GcCtx.CollectSmallObjects(true);
-			GcCtx.AddRetainedCids(KeepHashes);
-			Cas.CollectGarbage(GcCtx);
-			const HashKeySet& Deleted = GcCtx.DeletedCids();
-			Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
-				CHECK(ChunksToDelete.contains(ChunkHash));
-				GcChunkHashes.erase(ChunkHash);
-			});
+			DoGC(Cas, ChunksToDelete, KeepHashes, GcChunkHashes);
 		}
 		{
 			WorkCompleted = 0;