compact separate for gc referencer (#533)

- Refactor GCV2 so GcReferencer::RemoveExpiredData returns a store compactor, moving out the actual disk work from deleting items in the index.
- Refactor GCV2 GcResult to reuse GcCompactStoreStats and GcStats
- Make Compacting of stores non-parallell to not eat all the disk I/O when running GC

1 files changed, 199 insertions, 84 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 9fedd9165..617104ddc 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1435,31 +1435,40 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
 	return nullptr;
 }
 
-void
-ProjectStore::Project::DeleteOplog(std::string_view OplogId)
+std::filesystem::path
+ProjectStore::Project::RemoveOplog(std::string_view OplogId)
 {
+	RwLock::ExclusiveLockScope _(m_ProjectLock);
+
 	std::filesystem::path DeletePath;
+	if (auto OplogIt = m_Oplogs.find(std::string(OplogId)); OplogIt == m_Oplogs.end())
 	{
-		RwLock::ExclusiveLockScope _(m_ProjectLock);
+		std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
 
-		if (auto OplogIt = m_Oplogs.find(std::string(OplogId)); OplogIt == m_Oplogs.end())
+		if (Oplog::ExistsAt(OplogBasePath))
 		{
-			std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
-
-			if (Oplog::ExistsAt(OplogBasePath))
+			std::filesystem::path MovedDir;
+			if (PrepareDirectoryDelete(DeletePath, MovedDir))
 			{
-				DeletePath = OplogBasePath;
+				DeletePath = MovedDir;
 			}
 		}
-		else
-		{
-			std::unique_ptr<Oplog>& Oplog = OplogIt->second;
-			DeletePath					  = Oplog->PrepareForDelete(true);
-			m_DeletedOplogs.emplace_back(std::move(Oplog));
-			m_Oplogs.erase(OplogIt);
-		}
-		m_LastAccessTimes.erase(std::string(OplogId));
 	}
+	else
+	{
+		std::unique_ptr<Oplog>& Oplog = OplogIt->second;
+		DeletePath					  = Oplog->PrepareForDelete(true);
+		m_DeletedOplogs.emplace_back(std::move(Oplog));
+		m_Oplogs.erase(OplogIt);
+	}
+	m_LastAccessTimes.erase(std::string(OplogId));
+	return DeletePath;
+}
+
+void
+ProjectStore::Project::DeleteOplog(std::string_view OplogId)
+{
+	std::filesystem::path DeletePath = RemoveOplog(OplogId);
 
 	// Erase content on disk
 	if (!DeletePath.empty())
@@ -1567,9 +1576,29 @@ ProjectStore::Project::GatherReferences(GcContext& GcCtx)
 }
 
 uint64_t
+ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath)
+{
+	using namespace std::literals;
+
+	uint64_t			  Size				  = 0;
+	std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv;
+	if (std::filesystem::exists(AccessTimesFilePath))
+	{
+		Size += std::filesystem::file_size(AccessTimesFilePath);
+	}
+	std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv;
+	if (std::filesystem::exists(ProjectFilePath))
+	{
+		Size += std::filesystem::file_size(ProjectFilePath);
+	}
+
+	return Size;
+}
+
+uint64_t
 ProjectStore::Project::TotalSize() const
 {
-	uint64_t Result = 0;
+	uint64_t Result = TotalSize(m_OplogStoragePath);
 	{
 		std::vector<std::string> OpLogs = ScanForOplogs();
 		for (const std::string& OpLogId : OpLogs)
@@ -1954,7 +1983,7 @@ ProjectStore::StorageSize() const
 				std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv;
 				if (std::filesystem::exists(ProjectStateFilePath))
 				{
-					Result.DiskSize += std::filesystem::file_size(ProjectStateFilePath);
+					Result.DiskSize += Project::TotalSize(ProjectBasePath);
 					DirectoryContent DirContent;
 					GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
 					for (const std::filesystem::path& OplogBasePath : DirContent.Directories)
@@ -2068,12 +2097,8 @@ ProjectStore::UpdateProject(std::string_view ProjectId,
 }
 
 bool
-ProjectStore::DeleteProject(std::string_view ProjectId)
+ProjectStore::RemoveProject(std::string_view ProjectId, std::filesystem::path& OutDeletePath)
 {
-	ZEN_TRACE_CPU("Store::DeleteProject");
-
-	ZEN_INFO("deleting project {}", ProjectId);
-
 	RwLock::ExclusiveLockScope ProjectsLock(m_ProjectsLock);
 
 	auto ProjIt = m_Projects.find(std::string{ProjectId});
@@ -2083,20 +2108,34 @@ ProjectStore::DeleteProject(std::string_view ProjectId)
 		return true;
 	}
 
-	std::filesystem::path DeletePath;
-	bool				  Success = ProjIt->second->PrepareForDelete(DeletePath);
+	bool Success = ProjIt->second->PrepareForDelete(OutDeletePath);
 
 	if (!Success)
 	{
 		return false;
 	}
 	m_Projects.erase(ProjIt);
-	ProjectsLock.ReleaseNow();
+	return true;
+}
+
+bool
+ProjectStore::DeleteProject(std::string_view ProjectId)
+{
+	ZEN_TRACE_CPU("Store::DeleteProject");
+
+	ZEN_INFO("deleting project {}", ProjectId);
+
+	std::filesystem::path DeletePath;
+	if (!RemoveProject(ProjectId, DeletePath))
+	{
+		return false;
+	}
 
 	if (!DeletePath.empty())
 	{
 		DeleteDirectories(DeletePath);
 	}
+
 	return true;
 }
 
@@ -3042,29 +3081,106 @@ ProjectStore::GetGcName(GcCtx&)
 	return fmt::format("projectstore:'{}'", m_ProjectBasePath.string());
 }
 
-void
-ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
+class ProjectStoreGcStoreCompactor : public GcStoreCompactor
+{
+public:
+	ProjectStoreGcStoreCompactor(const std::filesystem::path&		  BasePath,
+								 std::vector<std::filesystem::path>&& OplogPathsToRemove,
+								 std::vector<std::filesystem::path>&& ProjectPathsToRemove)
+	: m_BasePath(BasePath)
+	, m_OplogPathsToRemove(std::move(OplogPathsToRemove))
+	, m_ProjectPathsToRemove(std::move(ProjectPathsToRemove))
+	{
+	}
+
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&)
+	{
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: projectstore [COMPACT] '{}': RemovedDisk: {} in {}",
+					 m_BasePath,
+					 NiceBytes(Stats.RemovedDisk),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		if (Ctx.Settings.IsDeleteMode)
+		{
+			for (const std::filesystem::path& OplogPath : m_OplogPathsToRemove)
+			{
+				uint64_t OplogSize = ProjectStore::Oplog::TotalSize(OplogPath);
+				if (DeleteDirectories(OplogPath))
+				{
+					ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed oplog folder '{}', removed {}",
+							  m_BasePath,
+							  OplogPath,
+							  NiceBytes(OplogSize));
+					Stats.RemovedDisk += OplogSize;
+				}
+				else
+				{
+					ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove oplog folder '{}'", m_BasePath, OplogPath);
+				}
+			}
+
+			for (const std::filesystem::path& ProjectPath : m_ProjectPathsToRemove)
+			{
+				uint64_t ProjectSize = ProjectStore::Project::TotalSize(ProjectPath);
+				if (DeleteDirectories(ProjectPath))
+				{
+					ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed project folder '{}', removed {}",
+							  m_BasePath,
+							  ProjectPath,
+							  NiceBytes(ProjectSize));
+					Stats.RemovedDisk += ProjectSize;
+				}
+				else
+				{
+					ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove project folder '{}'", m_BasePath, ProjectPath);
+				}
+			}
+		}
+		else
+		{
+			ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': Skipped deleting of {} oplogs and {} projects",
+					  m_BasePath,
+					  m_OplogPathsToRemove.size(),
+					  m_ProjectPathsToRemove.size());
+		}
+
+		m_ProjectPathsToRemove.clear();
+		m_OplogPathsToRemove.clear();
+	}
+
+private:
+	std::filesystem::path			   m_BasePath;
+	std::vector<std::filesystem::path> m_OplogPathsToRemove;
+	std::vector<std::filesystem::path> m_ProjectPathsToRemove;
+};
+
+GcStoreCompactor*
+ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 {
-	size_t	   ProjectCount		   = 0;
-	size_t	   ExpiredProjectCount = 0;
-	size_t	   OplogCount		   = 0;
-	size_t	   ExpiredOplogCount   = 0;
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
 		if (!Ctx.Settings.Verbose)
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+		ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {} in {}",
 				 m_ProjectBasePath,
-				 Stats.Count,
-				 Stats.Expired,
-				 Stats.Deleted,
-				 NiceBytes(Stats.RemovedDisk),
-				 NiceBytes(Stats.RemovedMemory),
+				 Stats.CheckedCount,
+				 Stats.FoundCount,
+				 Stats.DeletedCount,
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
+	std::vector<std::filesystem::path> OplogPathsToRemove;
+	std::vector<std::filesystem::path> ProjectPathsToRemove;
+
 	std::vector<Ref<Project>> ExpiredProjects;
 	std::vector<Ref<Project>> Projects;
 
@@ -3081,6 +3197,8 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
 		}
 	}
 
+	size_t OplogCount		 = 0;
+	size_t ExpiredOplogCount = 0;
 	for (const Ref<Project>& Project : Projects)
 	{
 		std::vector<std::string> ExpiredOplogs;
@@ -3101,70 +3219,69 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
 		{
 			for (const std::string& OplogId : ExpiredOplogs)
 			{
-				std::filesystem::path OplogBasePath = ProjectPath / OplogId;
-				uint64_t			  OplogSize		= Oplog::TotalSize(OplogBasePath);
-				ZEN_DEBUG("gc project store '{}': garbage collected oplog '{}' in project '{}'. Removing storage on disk",
-						  m_ProjectBasePath,
-						  OplogId,
-						  Project->Identifier);
-				Project->DeleteOplog(OplogId);
-				Stats.RemovedDisk += OplogSize;
+				std::filesystem::path RemovePath = Project->RemoveOplog(OplogId);
+				if (!RemovePath.empty())
+				{
+					OplogPathsToRemove.push_back(RemovePath);
+				}
 			}
-			Stats.Deleted += ExpiredOplogs.size();
+			Stats.DeletedCount += ExpiredOplogs.size();
 			Project->Flush();
 		}
 	}
-	ProjectCount = Projects.size();
-	Stats.Count += ProjectCount + OplogCount;
-	ExpiredProjectCount = ExpiredProjects.size();
+	size_t ProjectCount = Projects.size();
+
+	Stats.CheckedCount += ProjectCount + OplogCount;
 
 	if (ExpiredProjects.empty())
 	{
-		ZEN_DEBUG("gc project store '{}': no expired projects found", m_ProjectBasePath);
-		return;
+		ZEN_DEBUG("GCV2: projectstore [REMOVE EXPIRED] '{}': no expired projects found", m_ProjectBasePath);
+		return nullptr;
 	}
 
 	if (Ctx.Settings.IsDeleteMode)
 	{
 		for (const Ref<Project>& Project : ExpiredProjects)
 		{
-			std::filesystem::path PathToRemove;
-			std::string			  ProjectId = Project->Identifier;
+			std::string ProjectId = Project->Identifier;
 			{
 				{
 					RwLock::SharedLockScope Lock(m_ProjectsLock);
 					if (!Project->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime))
 					{
-						ZEN_DEBUG("gc project store '{}': skipped garbage collect of project '{}'. Project no longer expired.",
-								  m_ProjectBasePath,
-								  ProjectId);
+						ZEN_DEBUG(
+							"GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project no longer expired.",
+							m_ProjectBasePath,
+							ProjectId);
 						continue;
 					}
 				}
-				RwLock::ExclusiveLockScope __(m_ProjectsLock);
-				bool					   Success = Project->PrepareForDelete(PathToRemove);
+				std::filesystem::path RemovePath;
+				bool				  Success = RemoveProject(ProjectId, RemovePath);
 				if (!Success)
 				{
-					ZEN_DEBUG("gc project store '{}': skipped garbage collect of project '{}'. Project folder is locked.",
-							  m_ProjectBasePath,
-							  ProjectId);
+					ZEN_DEBUG(
+						"GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project folder is locked.",
+						m_ProjectBasePath,
+						ProjectId);
 					continue;
 				}
-				m_Projects.erase(ProjectId);
-			}
-
-			ZEN_DEBUG("gc project store '{}': sgarbage collected project '{}'. Removing storage on disk", m_ProjectBasePath, ProjectId);
-			if (PathToRemove.empty())
-			{
-				continue;
+				if (!RemovePath.empty())
+				{
+					ProjectPathsToRemove.push_back(RemovePath);
+				}
 			}
-
-			DeleteDirectories(PathToRemove);
 		}
-		Stats.Deleted += ExpiredProjects.size();
+		Stats.DeletedCount += ExpiredProjects.size();
 	}
 
-	Stats.Expired += ExpiredOplogCount + ExpiredProjectCount;
+	size_t ExpiredProjectCount = ExpiredProjects.size();
+	Stats.FoundCount += ExpiredOplogCount + ExpiredProjectCount;
+	if (!OplogPathsToRemove.empty() || !ProjectPathsToRemove.empty())
+	{
+		return new ProjectStoreGcStoreCompactor(m_ProjectBasePath, std::move(OplogPathsToRemove), std::move(ProjectPathsToRemove));
+	}
+	return nullptr;
 }
 
 class ProjectStoreReferenceChecker : public GcReferenceChecker
@@ -3182,16 +3299,15 @@ public:
 				}
 				ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': precached {} references in {} from {}/{}",
 						 m_Oplog.m_BasePath,
-						 m_UncachedReferences.size(),
+						 m_References.size(),
 						 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 						 m_Oplog.m_OuterProject->Identifier,
 						 m_Oplog.OplogId());
 			});
 
 			RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
-			m_Oplog.IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
-			});
+			m_Oplog.IterateOplog(
+				[&](CbObjectView Op) { Op.IterateAttachments([&](CbFieldView Visitor) { m_References.insert(Visitor.AsAttachment()); }); });
 			m_PreCachedLsn = m_Oplog.GetMaxOpIndex();
 		}
 	}
@@ -3208,7 +3324,7 @@ public:
 			}
 			ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
 					 m_Oplog.m_BasePath,
-					 m_UncachedReferences.size(),
+					 m_References.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 					 m_Oplog.m_OuterProject->Identifier,
 					 m_Oplog.OplogId());
@@ -3219,23 +3335,22 @@ public:
 		{
 			// TODO: Maybe we could just check the added oplog entries - we might get a few extra references from obsolete entries
 			// but I don't think that would be critical
-			m_UncachedReferences.clear();
-			m_Oplog.IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
-			});
+			m_References.clear();
+			m_Oplog.IterateOplog(
+				[&](CbObjectView Op) { Op.IterateAttachments([&](CbFieldView Visitor) { m_References.insert(Visitor.AsAttachment()); }); });
 		}
 	}
 
 	virtual void RemoveUsedReferencesFromSet(GcCtx&, HashSet& IoCids) override
 	{
-		for (const IoHash& ReferenceHash : m_UncachedReferences)
+		for (const IoHash& ReferenceHash : m_References)
 		{
 			IoCids.erase(ReferenceHash);
 		}
 	}
 	ProjectStore::Oplog&					 m_Oplog;
 	std::unique_ptr<RwLock::SharedLockScope> m_OplogLock;
-	HashSet									 m_UncachedReferences;
+	HashSet									 m_References;
 	int										 m_PreCachedLsn = -1;
 };