gc stop command (#569)v0.2.36-pre2

- Feature: New endpoint `/admin/gc-stop` to cancel a running garbage collect operation
- Feature: Added `zen gc-stop` command to cancel a running garbage collect operation
- Bugfix: GCv2 - make sure to discover all projects and oplogs before checking for expired data

8 files changed, 125 insertions, 7 deletions

diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 5786ae6c5..3422c6880 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -254,6 +254,60 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	return 1;
 }
 
+GcStopCommand::GcStopCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
+
+GcStopCommand::~GcStopCommand()
+{
+}
+
+int
+GcStopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw OptionParseException("unable to resolve server specification");
+	}
+
+	cpr::Session Session;
+	Session.SetUrl({fmt::format("{}/admin/gc-stop", m_HostName)});
+	cpr::Response Result = Session.Post();
+
+	if (static_cast<HttpResponseCode>(Result.status_code) == HttpResponseCode::Accepted)
+	{
+		ZEN_CONSOLE("OK: {}", "Cancel request accepted");
+		return 0;
+	}
+	else if (zen::IsHttpSuccessCode(Result.status_code))
+	{
+		ZEN_CONSOLE("OK: {}", "No GC running");
+		return 0;
+	}
+
+	if (Result.status_code)
+	{
+		ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+	}
+	else
+	{
+		ZEN_ERROR("GC status failed: {}", Result.error.message);
+	}
+
+	return 1;
+}
+
 ////////////////////////////////////////////
 
 JobCommand::JobCommand()
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 084c2f654..b5741c666 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -64,6 +64,20 @@ private:
 	bool			 m_Details = false;
 };
 
+class GcStopCommand : public StorageCommand
+{
+public:
+	GcStopCommand();
+	~GcStopCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+	cxxopts::Options m_Options{"gc-stop", "Request cancel of running garbage collection in zen storage"};
+	std::string		 m_HostName;
+};
+
 ////////////////////////////////////////////
 
 class JobCommand : public ZenCmdBase
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 7d3a9b9cd..c949008ff 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -271,6 +271,7 @@ main(int argc, char** argv)
 	FlushCommand			 FlushCmd;
 	GcCommand				 GcCmd;
 	GcStatusCommand			 GcStatusCmd;
+	GcStopCommand			 GcStopCmd;
 	ImportOplogCommand		 ImportOplogCmd;
 	JobCommand				 JobCmd;
 	OplogMirrorCommand		 OplogMirrorCmd;
@@ -313,6 +314,7 @@ main(int argc, char** argv)
 		{"down",				&DownCmd,				"Bring zen server down"},
 		{"drop",				&DropCmd,				"Drop cache namespace or bucket"},
 		{"gc-status",			&GcStatusCmd,			"Garbage collect zen storage status check"},
+		{"gc-stop",				&GcStopCmd,				"Request cancel of running garbage collection in zen storage"},
 		{"gc",					&GcCmd,					"Garbage collect zen storage"},
 		{"jobs",				&JobCmd,				"Show/cancel zen background jobs"},
 		{"logs",				&LoggingCmd,			"Show/control zen logging"},
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index f1d9f8d7c..1d5463a32 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -346,7 +346,19 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 
 			CbObjectWriter Response;
 			Response << "Status"sv << (Started ? "Started"sv : "Running"sv);
-			HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
+			HttpReq.WriteResponse(HttpResponseCode::Accepted, Response.Save());
+		},
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"gc-stop",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+			if (m_GcScheduler.CancelGC())
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::Accepted);
+			}
+			HttpReq.WriteResponse(HttpResponseCode::OK);
 		},
 		HttpVerb::kPost);
 
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 21aa6b44a..200aba3a9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3184,6 +3184,8 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 	std::vector<Ref<Project>> ExpiredProjects;
 	std::vector<Ref<Project>> Projects;
 
+	DiscoverProjects();
+
 	{
 		RwLock::SharedLockScope Lock(m_ProjectsLock);
 		for (auto& Kv : m_Projects)
@@ -3197,13 +3199,30 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		}
 	}
 
+	for (const Ref<Project>& Project : Projects)
+	{
+		std::vector<std::string> OpLogs = Project->ScanForOplogs();
+		for (const std::string& OpLogId : OpLogs)
+		{
+			Project->OpenOplog(OpLogId);
+			if (Ctx.IsCancelledFlag)
+			{
+				return nullptr;
+			}
+		}
+	}
+
 	size_t OplogCount		 = 0;
 	size_t ExpiredOplogCount = 0;
 	for (const Ref<Project>& Project : Projects)
 	{
+		if (Ctx.IsCancelledFlag)
+		{
+			break;
+		}
+
 		std::vector<std::string> ExpiredOplogs;
 		{
-			RwLock::ExclusiveLockScope __(m_ProjectsLock);
 			Project->IterateOplogs(
 				[&Ctx, &Project, &ExpiredOplogs, &OplogCount](const RwLock::SharedLockScope& Lock, ProjectStore::Oplog& Oplog) {
 					OplogCount++;
@@ -3233,7 +3252,7 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 
 	Stats.CheckedCount += ProjectCount + OplogCount;
 
-	if (ExpiredProjects.empty())
+	if (ExpiredProjects.empty() && ExpiredOplogCount == 0)
 	{
 		ZEN_DEBUG("GCV2: projectstore [REMOVE EXPIRED] '{}': no expired projects found", m_ProjectBasePath);
 		return nullptr;
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index ba9ff4f88..841f19295 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -604,8 +604,6 @@ ZenServer::Run()
 	SetNewState(kShuttingDown);
 
 	ZEN_INFO(ZEN_APP_NAME " exiting");
-
-	Flush();
 }
 
 void
@@ -640,8 +638,10 @@ ZenServer::Cleanup()
 		}
 
 		m_StatsReporter.Shutdown();
-
 		m_GcScheduler.Shutdown();
+
+		Flush();
+
 		m_AdminService.reset();
 		m_VfsService.reset();
 		m_ObjStoreService.reset();
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 2cd1f6aeb..262c94feb 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1295,8 +1295,9 @@ GcScheduler::Shutdown()
 		bool GcIsRunning = m_Status == static_cast<uint32_t>(GcSchedulerStatus::kRunning);
 		if (GcIsRunning)
 		{
-			m_GcManager.SetCancelGC(true);
+			ZEN_INFO("Requesting cancel running garbage collection");
 		}
+		m_GcManager.SetCancelGC(true);
 		m_Status = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
 		m_GcSignal.notify_one();
 
@@ -1353,6 +1354,20 @@ GcScheduler::TriggerScrub(const TriggerScrubParams& Params)
 	return false;
 }
 
+bool
+GcScheduler::CancelGC()
+{
+	std::unique_lock Lock(m_GcMutex);
+
+	if (static_cast<uint32_t>(GcSchedulerStatus::kRunning) == m_Status)
+	{
+		ZEN_INFO("Cancel requested for running garbage collection");
+		m_GcManager.SetCancelGC(true);
+		return true;
+	}
+	return false;
+}
+
 DiskSpace
 GcScheduler::CheckDiskSpace()
 {
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 486dca3c6..7a6249970 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -492,6 +492,8 @@ public:
 
 	bool TriggerScrub(const TriggerScrubParams& Params);
 
+	bool CancelGC();
+
 private:
 	void		 SchedulerThread();
 	void		 CollectGarbage(const GcClock::TimePoint& CacheExpireTime,