Merged main

5 files changed, 99 insertions, 21 deletions

diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index 1db2b50bf..a4bbfa340 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -32,6 +32,15 @@ CasChunkSet::AddChunkToSet(const IoHash& HashToAdd)
 }
 
 void
+CasChunkSet::AddChunksToSet(std::span<const IoHash> HashesToAdd)
+{
+	for (const IoHash& Hash : HashesToAdd)
+	{
+		m_ChunkSet.insert(Hash);
+	}
+}
+
+void
 CasChunkSet::RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate)
 {
 	for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;)
@@ -58,10 +67,45 @@ CasChunkSet::IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callba
 
 //////////////////////////////////////////////////////////////////////////
 
+struct GcContext::GcState
+{
+	CasChunkSet m_CasChunks;
+	CasChunkSet m_CidChunks;
+};
+
+GcContext::GcContext() : m_State(std::make_unique<GcState>())
+{
+}
+
+GcContext::~GcContext()
+{
+}
+
+void
+GcContext::ContributeCids(std::span<const IoHash> Cids)
+{
+	m_State->m_CidChunks.AddChunksToSet(Cids);
+}
+
+void
+GcContext::ContributeCas(std::span<const IoHash> Cas)
+{
+	m_State->m_CasChunks.AddChunksToSet(Cas);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+ScrubContext::ReportBadCasChunks(std::span<IoHash> BadCasChunks)
+{
+	m_BadCas.AddChunksToSet(BadCasChunks);
+}
+
 void
-ScrubContext::ReportBadChunks(std::span<IoHash> BadChunks)
+ScrubContext::ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes)
 {
-	ZEN_UNUSED(BadChunks);
+	m_ChunkCount.fetch_add(ChunkCount);
+	m_ByteCount.fetch_add(ChunkBytes);
 }
 
 /**
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index df5c32d25..7a5d7bcf4 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -204,7 +204,7 @@ struct CidStore::Impl
 
 		// TODO: Should compute a snapshot index here
 
-		Ctx.ReportBadChunks(BadChunks);
+		Ctx.ReportBadCasChunks(BadChunks);
 	}
 
 	uint64_t m_LastScrubTime = 0;
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 5fc3ac356..612f87c7c 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -254,7 +254,7 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
 	// be used to invalidate higher level data structures more efficiently
 	// than a full validation pass might be able to do
 
-	Ctx.ReportBadChunks(BadChunkHashes);
+	Ctx.ReportBadCasChunks(BadChunkHashes);
 }
 
 void
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 0b18848d5..ee641b80a 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -394,7 +394,8 @@ FileCasStrategy::Flush()
 void
 FileCasStrategy::Scrub(ScrubContext& Ctx)
 {
-	std::vector<IoHash> BadHashes;
+	std::vector<IoHash>	  BadHashes;
+	std::atomic<uint64_t> ChunkCount{0}, ChunkBytes{0};
 
 	IterateChunks([&](const IoHash& Hash, BasicFile& Payload) {
 		IoHashStream Hasher;
@@ -405,8 +406,13 @@ FileCasStrategy::Scrub(ScrubContext& Ctx)
 		{
 			BadHashes.push_back(Hash);
 		}
+
+		++ChunkCount;
+		ChunkBytes.fetch_add(Payload.FileSize());
 	});
 
+	Ctx.ReportScrubbed(ChunkCount, ChunkBytes);
+
 	if (!BadHashes.empty())
 	{
 		ZEN_ERROR("file CAS scrubbing: {} bad chunks found", BadHashes.size());
@@ -428,7 +434,9 @@ FileCasStrategy::Scrub(ScrubContext& Ctx)
 		}
 	}
 
-	Ctx.ReportBadChunks(BadHashes);
+	Ctx.ReportBadCasChunks(BadHashes);
+
+	ZEN_INFO("file CAS scrubbed: {} chunks ({})", ChunkCount.load(), NiceBytes(ChunkBytes));
 }
 
 void
diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
index 93454ca6f..d0698df7f 100644
--- a/zenstore/include/zenstore/CAS.h
+++ b/zenstore/include/zenstore/CAS.h
@@ -31,13 +31,41 @@ struct CasStoreConfiguration
 	uint64_t HugeValueThreshold = 1024 * 1024;
 };
 
+/** Manage a set of IoHash values
+ */
+
+class CasChunkSet
+{
+public:
+	void						AddChunkToSet(const IoHash& HashToAdd);
+	void						AddChunksToSet(std::span<const IoHash> HashesToAdd);
+	void						RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate);
+	void						IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback);
+	inline [[nodiscard]] bool	ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); }
+	inline [[nodiscard]] bool	IsEmpty() const { return m_ChunkSet.empty(); }
+	inline [[nodiscard]] size_t GetSize() const { return m_ChunkSet.size(); }
+
+private:
+	// Q: should we protect this with a lock, or is that a higher level concern?
+	std::unordered_set<IoHash> m_ChunkSet;
+};
+
 /** Garbage Collection context object
  */
 
 class GcContext
 {
 public:
+	GcContext();
+	~GcContext();
+
+	void ContributeCids(std::span<const IoHash> Cid);
+	void ContributeCas(std::span<const IoHash> Hash);
+
 private:
+	struct GcState;
+
+	std::unique_ptr<GcState> m_State;
 };
 
 /** Context object for data scrubbing
@@ -49,28 +77,26 @@ private:
 class ScrubContext
 {
 public:
-	virtual void	ReportBadChunks(std::span<IoHash> BadChunks);
+	virtual void	ReportBadCasChunks(std::span<IoHash> BadCasChunks);
 	inline uint64_t ScrubTimestamp() const { return m_ScrubTime; }
 	inline bool		RunRecovery() const { return m_Recover; }
+	void			ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes);
+
+	inline uint64_t ScrubbedChunks() const { return m_ChunkCount; }
+	inline uint64_t ScrubbedBytes() const { return m_ByteCount; }
 
 private:
-	uint64_t m_ScrubTime = GetHifreqTimerValue();
-	bool	 m_Recover	 = true;
+	uint64_t			  m_ScrubTime = GetHifreqTimerValue();
+	bool				  m_Recover	  = true;
+	std::atomic<uint64_t> m_ChunkCount{0};
+	std::atomic<uint64_t> m_ByteCount{0};
+	CasChunkSet			  m_BadCas;
+	CasChunkSet			  m_BadCid;
 };
 
-class CasChunkSet
-{
-public:
-	void						AddChunkToSet(const IoHash& HashToAdd);
-	void						RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate);
-	void						IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback);
-	inline [[nodiscard]] bool	ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); }
-	inline [[nodiscard]] bool	IsEmpty() const { return m_ChunkSet.empty(); }
-	inline [[nodiscard]] size_t GetSize() const { return m_ChunkSet.size(); }
+/** Content Addressable Storage interface
 
-private:
-	std::unordered_set<IoHash> m_ChunkSet;
-};
+  */
 
 class CasStore
 {