Improved oplog import/export progress indicator at commandline (#206)

Nicer progress bar during oplog import/export
Verify that oplog has not been deleted from disk behind our back

1 files changed, 41 insertions, 23 deletions

diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp
index d26d0dd1e..b97484458 100644
--- a/src/zencore/jobqueue.cpp
+++ b/src/zencore/jobqueue.cpp
@@ -11,6 +11,10 @@
 #	include <zencore/testing.h>
 #endif	// ZEN_WITH_TESTS
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 #include <deque>
 #include <thread>
 #include <unordered_map>
@@ -49,9 +53,12 @@ public:
 
 		virtual bool IsCancelled() const override { return CancelFlag.load(); }
 		virtual void ReportMessage(std::string_view Message) override { Queue->ReportMessage(Id, Message); }
-		virtual void ReportProgress(std::string_view CurrentOp, uint32_t CurrentOpPercentComplete) override
+		virtual void ReportProgress(std::string_view CurrentOp,
+									std::string_view Details,
+									ptrdiff_t		 TotalCount,
+									ptrdiff_t		 RemainingCount) override
 		{
-			Queue->ReportProgress(Id, CurrentOp, CurrentOpPercentComplete);
+			Queue->ReportProgress(Id, CurrentOp, Details, TotalCount, RemainingCount);
 		}
 	};
 
@@ -254,16 +261,20 @@ public:
 	virtual std::optional<JobDetails> Get(JobId Id) override
 	{
 		auto Convert = [](Status Status, Job& Job) -> JobDetails {
-			return JobDetails{.Name			  = Job.Name,
-							  .Status		  = Status,
-							  .State		  = {.CurrentOp				   = Job.State.CurrentOp,
-												 .CurrentOpPercentComplete = Job.State.CurrentOpPercentComplete,
-												 .Messages				   = std::move(Job.State.Messages),
-												 .AbortReason			   = Job.State.AbortReason},
-							  .CreateTime	  = JobClock::TimePointFromTick(Job.CreateTick),
-							  .StartTime	  = JobClock::TimePointFromTick(Job.StartTick),
-							  .EndTime		  = JobClock::TimePointFromTick(Job.EndTick),
-							  .WorkerThreadId = Job.WorkerThreadId};
+			return JobDetails{
+				.Name			= Job.Name,
+				.Status			= Status,
+				.State			= {.CurrentOp		 = Job.State.CurrentOp,
+								   .CurrentOpDetails = Job.State.CurrentOpDetails,
+								   .TotalCount		 = Job.State.TotalCount,
+								   .RemainingCount	 = Job.State.RemainingCount,
+						   //												 .CurrentOpPercentComplete = Job.State.CurrentOpPercentComplete,
+								   .Messages	= std::move(Job.State.Messages),
+								   .AbortReason = Job.State.AbortReason},
+				.CreateTime		= JobClock::TimePointFromTick(Job.CreateTick),
+				.StartTime		= JobClock::TimePointFromTick(Job.StartTick),
+				.EndTime		= JobClock::TimePointFromTick(Job.EndTick),
+				.WorkerThreadId = Job.WorkerThreadId};
 		};
 
 		std::optional<JobDetails> Result;
@@ -304,13 +315,15 @@ public:
 		});
 	}
 
-	void ReportProgress(JobId Id, std::string_view CurrentOp, uint32_t CurrentOpPercentComplete)
+	void ReportProgress(JobId Id, std::string_view CurrentOp, std::string_view Details, ptrdiff_t TotalCount, ptrdiff_t RemainingCount)
 	{
 		QueueLock.WithExclusiveLock([&]() {
 			auto It = RunningJobs.find(Id.Id);
 			ZEN_ASSERT(It != RunningJobs.end());
-			It->second->State.CurrentOp				   = CurrentOp;
-			It->second->State.CurrentOpPercentComplete = CurrentOpPercentComplete;
+			It->second->State.CurrentOp		   = CurrentOp;
+			It->second->State.CurrentOpDetails = Details;
+			It->second->State.TotalCount	   = TotalCount;
+			It->second->State.RemainingCount   = RemainingCount;
 		});
 	}
 
@@ -442,13 +455,13 @@ TEST_CASE("JobQueue")
 				{
 					return;
 				}
-				Context.ReportProgress("going to sleep", 0);
+				Context.ReportProgress("going to sleep", "", 100, 100);
 				Sleep(10);
 				if (Context.IsCancelled())
 				{
 					return;
 				}
-				Context.ReportProgress("going to sleep again", 50);
+				Context.ReportProgress("going to sleep again", "", 100, 50);
 				if ((I & 0xFF) == 0x10)
 				{
 					zen::ThrowSystemError(8, fmt::format("Job {} forced to fail", I));
@@ -458,7 +471,7 @@ TEST_CASE("JobQueue")
 				{
 					return;
 				}
-				Context.ReportProgress("done", 100);
+				Context.ReportProgress("done", "", 100, 0);
 			});
 		});
 	}
@@ -507,11 +520,16 @@ TEST_CASE("JobQueue")
 					RemainingJobs.push_back(Id);
 					break;
 				case JobQueue::Status::Running:
-					ZEN_DEBUG("{} running. '{}' {}% '{}'",
-							  Id.Id,
-							  CurrentState->State.CurrentOp,
-							  CurrentState->State.CurrentOpPercentComplete,
-							  Join(CurrentState->State.Messages, " "sv));
+					ZEN_DEBUG(
+						"{} running. '{}{}' {}% '{}'",
+						Id.Id,
+						CurrentState->State.CurrentOp,
+						CurrentState->State.CurrentOpDetails.empty() ? ""sv : fmt::format(", {}", CurrentState->State.CurrentOpDetails),
+						CurrentState->State.TotalCount > 0
+							? gsl::narrow<uint32_t>((100 * (CurrentState->State.TotalCount - CurrentState->State.RemainingCount)) /
+													CurrentState->State.TotalCount)
+							: 0,
+						Join(CurrentState->State.Messages, " "sv));
 					RemainingJobs.push_back(Id);
 					break;
 				case JobQueue::Status::Aborted: