job queue and async oplog-import/export (#395)

- Feature: New http endpoint for background jobs `/admin/jobs/status` which will return a response listing the currently active background jobs and their status
- Feature: New http endpoint for background jobs information `/admin/jobs/status/{jobid}` which will return a response detailing status, pending messages and progress status
  - GET will return a response detailing status, pending messages and progress status
  - DELETE will mark the job for cancelling and return without waiting for completion
  - If status returned is "Complete" or "Aborted" the jobid will be removed from the server and can not be queried again
- Feature: New zen command `jobs` to list, get info about and cancel background jobs
  - If no options are given it will display a list of active background jobs
  - `--jobid` accepts an id (returned from for example `oplog-export` with `--async`) and will return a response detailing status, pending messages and progress status for that job
  - `--cancel` can be added when `--jobid` is given which will request zenserver to cancel the background job
- Feature: oplog import and export http rpc requests are now async operations that will run in the background
- Feature: `oplog-export` and `oplog-import` now reports progress to the console as work progress by default
- Feature: `oplog-export` and `oplog-import` can now be cancelled using Ctrl+C
- Feature: `oplog-export` and `oplog-import` has a new option `--async` which will only trigger the work and report a background job id back

1 files changed, 84 insertions, 0 deletions

diff --git a/src/zencore/include/zencore/jobqueue.h b/src/zencore/include/zencore/jobqueue.h
new file mode 100644
index 000000000..41a3288e1
--- /dev/null
+++ b/src/zencore/include/zencore/jobqueue.h
@@ -0,0 +1,84 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace zen {
+
+struct JobId
+{
+	uint64_t Id;
+};
+
+class JobQueue;
+
+struct JobContext
+{
+	JobQueue&		  Queue;
+	const JobId		  Id;
+	std::atomic_bool& CancelFlag;
+};
+
+class JobQueue
+{
+public:
+	typedef std::function<void(JobContext& Context)> JobFunction;
+	virtual ~JobQueue() = default;
+
+	virtual JobId QueueJob(JobFunction&& JobFunc) = 0;
+	virtual bool  CancelJob(JobId Id)			  = 0;
+	virtual void  Stop()						  = 0;
+
+	virtual void ReportMessage(JobId Id, std::string_view Message)										 = 0;
+	virtual void ReportProgress(JobId Id, std::string_view CurrentOp, uint32_t CurrentOpPercentComplete) = 0;
+
+	enum class Status : uint32_t
+	{
+		Queued,
+		Running,
+		Aborted,
+		Completed
+	};
+
+	struct State
+	{
+		std::string				 CurrentOp;
+		uint32_t				 CurrentOpPercentComplete = 0;
+		std::vector<std::string> Messages;
+	};
+
+	struct JobInfo
+	{
+		JobId  Id;
+		Status Status;
+	};
+
+	virtual std::vector<JobInfo> GetJobs() = 0;
+
+	struct JobDetails
+	{
+		Status								  Status;
+		State								  State;
+		std::chrono::system_clock::time_point CreateTime;
+		std::chrono::system_clock::time_point StartTime;
+		std::chrono::system_clock::time_point EndTime;
+	};
+
+	// Will only respond once when status is Complete or Aborted
+	virtual std::optional<JobDetails> Get(JobId Id) = 0;
+
+	static std::string_view ToString(Status Status);
+};
+
+std::unique_ptr<JobQueue> MakeJobQueue(int WorkerCount, std::string_view QueueName);
+
+void jobqueue_forcelink();
+
+}  // namespace zen