improved scrubbing of oplogs and filecas (#596)

- Improvement: Scrub command now validates compressed buffer hashes in filecas storage (used for large chunks)
- Improvement: Added --dry, --no-gc and --no-cas options to zen scrub command
- Improvement: Implemented oplog scrubbing (previously was a no-op)
- Improvement: Implemented support for running scrubbint at startup with --scrub=<options>

1 files changed, 64 insertions, 4 deletions

diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 418e3c325..336f715f4 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -120,6 +120,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	m_DebugOptionForcedCrash = ServerOptions.ShouldCrash;
 	m_IsPowerCycle			 = ServerOptions.IsPowerCycle;
 	const int ParentPid		 = ServerOptions.OwnerPid;
+	m_StartupScrubOptions	 = ServerOptions.ScrubOptions;
 
 	if (ParentPid)
 	{
@@ -554,10 +555,6 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 void
 ZenServer::Run()
 {
-	// This is disabled for now, awaiting better scheduling
-	//
-	// ScrubStorage();
-
 	if (m_ProcessMonitor.IsActive())
 	{
 		CheckOwnerPid();
@@ -594,6 +591,69 @@ ZenServer::Run()
 
 	OnReady();
 
+	if (!m_StartupScrubOptions.empty())
+	{
+		using namespace std::literals;
+
+		ZEN_INFO("triggering scrub with settings: '{}'", m_StartupScrubOptions);
+
+		bool							DoScrub = true;
+		bool							DoWait	= false;
+		GcScheduler::TriggerScrubParams ScrubParams;
+
+		ForEachStrTok(m_StartupScrubOptions, ',', [&](std::string_view Token) {
+			if (Token == "nocas"sv)
+			{
+				ScrubParams.SkipCas = true;
+			}
+			else if (Token == "nodelete"sv)
+			{
+				ScrubParams.SkipDelete = true;
+			}
+			else if (Token == "nogc"sv)
+			{
+				ScrubParams.SkipGc = true;
+			}
+			else if (Token == "no"sv)
+			{
+				DoScrub = false;
+			}
+			else if (Token == "wait"sv)
+			{
+				DoWait = true;
+			}
+			return true;
+		});
+
+		if (DoScrub)
+		{
+			m_GcScheduler.TriggerScrub(ScrubParams);
+
+			if (DoWait)
+			{
+				auto State = m_GcScheduler.Status();
+
+				while ((State != GcSchedulerStatus::kRunning) && (State != GcSchedulerStatus::kStopped))
+				{
+					Sleep(500);
+
+					State = m_GcScheduler.Status();
+				}
+
+				ZEN_INFO("waiting for Scrub/GC to complete...");
+
+				while (State == GcSchedulerStatus::kRunning)
+				{
+					Sleep(500);
+
+					State = m_GcScheduler.Status();
+				}
+
+				ZEN_INFO("Scrub/GC completed");
+			}
+		}
+	}
+
 	if (m_IsPowerCycle)
 	{
 		ZEN_INFO("Power cycle mode enabled -- shutting down");