Remove Cid to CAS chunk mapping after GC.

1 files changed, 97 insertions, 4 deletions

diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index bb26af87b..d5cb4901b 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -9,6 +9,8 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
 #include <zencore/timer.h>
 #include <zenstore/CAS.h>
 #include <zenstore/cidstore.h>
@@ -16,6 +18,12 @@
 #include <fmt/format.h>
 #include <filesystem>
 
+#if ZEN_WITH_TESTS
+#	include <zencore/compress.h>
+#	include <algorithm>
+#	include <random>
+#endif
+
 namespace zen {
 
 using namespace std::literals;
@@ -60,6 +68,7 @@ struct GcContext::GcState
 
 	CacheBuckets	   m_CacheBuckets;
 	CasChunkSet		   m_CasChunks;
+	CasChunkSet		   m_DeletedCasChunks;
 	CasChunkSet		   m_CidChunks;
 	GcClock::TimePoint m_GcTime;
 	GcClock::Duration  m_MaxCacheDuration	 = std::chrono::hours(24);
@@ -113,6 +122,24 @@ GcContext::FilterCas(std::span<const IoHash> Cas, std::function<void(const IoHas
 	m_State->m_CasChunks.FilterChunks(Cas, [&](const IoHash& Hash) { KeepFunc(Hash); });
 }
 
+void
+GcContext::FilterCas(std::span<const IoHash> Cas, std::function<void(const IoHash&, bool)>&& FilterFunc)
+{
+	m_State->m_CasChunks.FilterChunks(Cas, std::move(FilterFunc));
+}
+
+void
+GcContext::DeletedCas(std::span<const IoHash> Cas)
+{
+	m_State->m_DeletedCasChunks.AddChunksToSet(Cas);
+}
+
+CasChunkSet&
+GcContext::DeletedCas()
+{
+	return m_State->m_DeletedCasChunks;
+}
+
 std::span<const IoHash>
 GcContext::ValidCacheKeys(const std::string& Bucket) const
 {
@@ -244,11 +271,10 @@ CasGc::CollectGarbage(GcContext& GcCtx)
 	if (CidStore* CidStore = m_CidStore)
 	{
 		std::vector<IoHash> CasHashes;
+		uint64_t			UnknownChunks = 0;
 
-		int UnknownChunks = 0;
-
-		GcCtx.IterateCids([&](const IoHash& Hash) {
-			IoHash Cas = CidStore->RemapCid(Hash);
+		GcCtx.IterateCids([&](const IoHash& Cid) {
+			IoHash Cas = CidStore->RemapCid(Cid);
 
 			if (Cas == IoHash::Zero)
 			{
@@ -274,6 +300,11 @@ CasGc::CollectGarbage(GcContext& GcCtx)
 	{
 		Storage->CollectGarbage(GcCtx);
 	}
+
+	if (CidStore* CidStore = m_CidStore)
+	{
+		CidStore->RemoveCids(GcCtx.DeletedCas());
+	}
 }
 
 void
@@ -510,4 +541,66 @@ GcScheduler::NextGcTime(GcClock::TimePoint CurrentTime)
 
 //////////////////////////////////////////////////////////////////////////
 
+#if ZEN_WITH_TESTS
+
+namespace {
+	IoHash CreateKey(size_t KeyValue) { return IoHash::HashBuffer(&KeyValue, sizeof(size_t)); }
+
+	static IoBuffer CreateChunk(uint64_t Size)
+	{
+		static std::random_device rd;
+		static std::mt19937		  g(rd());
+
+		const size_t		  Count = static_cast<size_t>(Size / sizeof(uint32_t));
+		std::vector<uint32_t> Values;
+		Values.resize(Count);
+		for (size_t Idx = 0; Idx < Count; ++Idx)
+		{
+			Values[Idx] = static_cast<uint32_t>(Idx);
+		}
+		std::shuffle(Values.begin(), Values.end(), g);
+
+		return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size() * sizeof(uint32_t));
+	}
+
+	static CompressedBuffer Compress(IoBuffer Buffer)
+	{
+		return CompressedBuffer::Compress(SharedBuffer::MakeView(Buffer.GetData(), Buffer.GetSize()));
+	}
+}  // namespace
+
+TEST_CASE("gc.basic")
+{
+	ScopedTemporaryDirectory TempDir;
+
+	CasStoreConfiguration CasConfig;
+	CasConfig.RootDirectory = TempDir.Path() / "cas";
+
+	CasGc					  Gc;
+	std::unique_ptr<CasStore> CasStore = CreateCasStore(Gc);
+	CidStore				  CidStore{*CasStore, TempDir.Path() / "cid"};
+
+	CasStore->Initialize(CasConfig);
+	Gc.SetCidStore(&CidStore);
+
+	IoBuffer Chunk			 = CreateChunk(128);
+	auto	 CompressedChunk = Compress(Chunk);
+
+	const auto InsertResult = CidStore.AddChunk(CompressedChunk);
+
+	GcContext GcCtx;
+	GcCtx.CollectSmallObjects(true);
+
+	Gc.CollectGarbage(GcCtx);
+
+	CHECK(!CidStore.ContainsChunk(InsertResult.DecompressedId));
+}
+
+#endif
+
+void
+gc_forcelink()
+{
+}
+
 }  // namespace zen