Merge branch 'gc' of https://github.com/EpicGames/zen into gc

4 files changed, 122 insertions, 19 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 73d61c124..72d7284a0 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -298,6 +298,36 @@ ProjectStore::Oplog::Flush()
 	m_Storage->Flush();
 }
 
+void
+ProjectStore::Oplog::Scrub(ScrubContext& Ctx) const
+{
+	ZEN_UNUSED(Ctx);
+}
+
+void
+ProjectStore::Oplog::GatherReferences(GcContext& GcCtx) const
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+
+	std::vector<IoHash> Hashes;
+
+	for (const auto& Kv : m_ChunkMap)
+	{
+		Hashes.push_back(Kv.second);
+	}
+
+	GcCtx.ContributeCids(Hashes);
+
+	Hashes.clear();
+
+	for (const auto& Kv : m_MetaMap)
+	{
+		Hashes.push_back(Kv.second);
+	}
+
+	GcCtx.ContributeCids(Hashes);
+}
+
 bool
 ProjectStore::Oplog::ExistsAt(std::filesystem::path BasePath)
 {
@@ -750,7 +780,7 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId)
 void
 ProjectStore::Project::IterateOplogs(std::function<void(const Oplog&)>&& Fn) const
 {
-	// TODO: should iterate over oplogs which are present on disk but not yet loaded
+	// TODO: should also iterate over oplogs which are present on disk but not yet loaded
 
 	RwLock::SharedLockScope _(m_ProjectLock);
 
@@ -769,13 +799,20 @@ ProjectStore::Project::Flush()
 void
 ProjectStore::Project::Scrub(ScrubContext& Ctx)
 {
-	ZEN_UNUSED(Ctx);
+	IterateOplogs([&](const Oplog& Ops) { Ops.Scrub(Ctx); });
+}
+
+void
+ProjectStore::Project::GatherReferences(GcContext& GcCtx)
+{
+	IterateOplogs([&](const Oplog& Ops) { Ops.GatherReferences(GcCtx); });
 }
 
 //////////////////////////////////////////////////////////////////////////
 
-ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath)
-: m_Log(zen::logging::Get("project"))
+ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, CasGc& Gc)
+: GcContributor(Gc)
+, m_Log(zen::logging::Get("project"))
 , m_ProjectBasePath(BasePath)
 , m_CidStore(Store)
 {
@@ -818,6 +855,17 @@ ProjectStore::Scrub(ScrubContext& Ctx)
 	}
 }
 
+void
+ProjectStore::GatherReferences(GcContext& GcCtx)
+{
+	RwLock::SharedLockScope _(m_ProjectsLock);
+
+	for (auto& Kv : m_Projects)
+	{
+		Kv.second.GatherReferences(GcCtx);
+	}
+}
+
 ProjectStore::Project*
 ProjectStore::OpenProject(std::string_view ProjectId)
 {
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index 83d3986bb..743abb5eb 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -9,6 +9,7 @@
 #include <zenstore/cas.h>
 #include <zenstore/caslog.h>
 #include <zenstore/cidstore.h>
+#include <zenstore/gc.h>
 
 #include <filesystem>
 #include <map>
@@ -49,12 +50,12 @@ static_assert(IsPow2(sizeof(OplogEntry)));
 
 /** Project Store
  */
-class ProjectStore : public RefCounted
+class ProjectStore : public RefCounted, public GcContributor
 {
 	struct OplogStorage;
 
 public:
-	ProjectStore(CidStore& Store, std::filesystem::path BasePath);
+	ProjectStore(CidStore& Store, std::filesystem::path BasePath, CasGc& Gc);
 	~ProjectStore();
 
 	struct Project;
@@ -105,7 +106,8 @@ public:
 
 		spdlog::logger& Log() { return m_OuterProject->Log(); }
 		void			Flush();
-		void			Scrub(ScrubContext& Ctx);
+		void			Scrub(ScrubContext& Ctx) const;
+		void			GatherReferences(GcContext& GcCtx) const;
 
 		std::size_t OplogCount() const { return m_LatestOpMap.size(); }
 
@@ -124,7 +126,7 @@ public:
 		std::filesystem::path m_BasePath;
 		std::filesystem::path m_TempPath;
 
-		RwLock							 m_OplogLock;
+		mutable RwLock					 m_OplogLock;
 		OidMap<IoHash>					 m_ChunkMap;		 // output data chunk id -> CAS address
 		OidMap<IoHash>					 m_MetaMap;			 // meta chunk id -> CAS address
 		OidMap<FileMapEntry>			 m_FileMap;			 // file id -> file map entry
@@ -161,6 +163,7 @@ public:
 		void					  Flush();
 		void					  Scrub(ScrubContext& Ctx);
 		spdlog::logger&			  Log();
+		void					  GatherReferences(GcContext& GcCtx);
 
 	private:
 		ProjectStore*				 m_ProjectStore;
@@ -188,6 +191,8 @@ public:
 	spdlog::logger&				 Log() { return m_Log; }
 	const std::filesystem::path& BasePath() const { return m_ProjectBasePath; }
 
+	virtual void GatherReferences(GcContext& GcCtx) override;
+
 private:
 	spdlog::logger&				   m_Log;
 	CidStore&					   m_CidStore;
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 33cec7db1..8da781a59 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -218,7 +218,7 @@ public:
 
 		ZEN_INFO("instantiating project service");
 
-		m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects");
+		m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects", m_Gc);
 		m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore});
 
 #if ZEN_USE_NAMED_PIPES
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 0714637c6..c83f87c1b 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -560,7 +560,7 @@ FileCasStrategy::CollectGarbage(GcContext& GcCtx)
 		return;
 	}
 
-	ZEN_INFO("deleting file CAS garbage: {} chunks ({})", ChunkCount.load(), NiceBytes(ChunksToDeleteBytes));
+	ZEN_INFO("deleting file CAS garbage: {} chunks ({})", ChunksToDelete.size(), NiceBytes(ChunksToDeleteBytes));
 
 	for (const IoHash& Hash : ChunksToDelete)
 	{
@@ -673,20 +673,70 @@ TEST_CASE("cas.file.gc")
 	FileCasStrategy FileCas(CasConfig, Gc);
 	FileCas.Initialize(/* IsNewStore */ true);
 
-	for (int i = 0; i < 1000; ++i)
+	const int			kIterationCount = 1000;
+	std::vector<IoHash> Keys{kIterationCount};
+
+	auto InsertChunks = [&] {
+		for (int i = 0; i < kIterationCount; ++i)
+		{
+			CbObjectWriter Cbo;
+			Cbo << "id" << i;
+			CbObject Obj = Cbo.Save();
+
+			IoBuffer ObjBuffer = Obj.GetBuffer().AsIoBuffer();
+			IoHash	 Hash	   = HashBuffer(ObjBuffer);
+
+			FileCas.InsertChunk(ObjBuffer, Hash);
+
+			Keys[i] = Hash;
+		}
+	};
+
+	// Drop everything
+
 	{
-		CbObjectWriter Cbo;
-		Cbo << "id" << i;
-		CbObject Obj = Cbo.Save();
+		InsertChunks();
 
-		IoBuffer ObjBuffer = Obj.GetBuffer().AsIoBuffer();
-		IoHash	 Hash	   = HashBuffer(ObjBuffer);
+		GcContext Ctx;
+		FileCas.CollectGarbage(Ctx);
 
-		FileCas.InsertChunk(ObjBuffer, Hash);
+		for (const IoHash& Key : Keys)
+		{
+			IoBuffer Chunk = FileCas.FindChunk(Key);
+
+			CHECK(!Chunk);
+		}
 	}
 
-	GcContext Ctx;
-	FileCas.CollectGarbage(Ctx);
+	// Keep roughly half of the chunks
+
+	{
+		InsertChunks();
+
+		GcContext Ctx;
+
+		for (const IoHash& Key : Keys)
+		{
+			if (Key.Hash[0] & 1)
+			{
+				Ctx.ContributeCas(std::vector<IoHash>{Key});
+			}
+		}
+
+		FileCas.CollectGarbage(Ctx);
+
+		for (const IoHash& Key : Keys)
+		{
+			if (Key.Hash[0] & 1)
+			{
+				CHECK(FileCas.FindChunk(Key));
+			}
+			else
+			{
+				CHECK(!FileCas.FindChunk(Key));
+			}
+		}
+	}
 }
 
 #endif