Improved oplog import/export progress indicator at commandline (#206)

Nicer progress bar during oplog import/export
Verify that oplog has not been deleted from disk behind our back

12 files changed, 457 insertions, 150 deletions

diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 8573c36aa..4e2d5123d 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -9,6 +9,7 @@
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/stream.h>
+#include <zencore/timer.h>
 #include <zencore/workthreadpool.h>
 #include <zenhttp/formatters.h>
 #include <zenhttp/httpclient.h>
@@ -79,7 +80,7 @@ namespace {
 		}
 	}
 
-	void ExecuteAsyncOperation(HttpClient& Http, std::string_view Url, IoBuffer&& Payload)
+	void ExecuteAsyncOperation(HttpClient& Http, std::string_view Url, IoBuffer&& Payload, bool PlainProgress)
 	{
 		signal(SIGINT, SignalCallbackHandler);
 #if ZEN_PLATFORM_WINDOWS
@@ -98,8 +99,20 @@ namespace {
 					throw std::runtime_error(fmt::format("invalid job id returned, received '{}'", JobIdText));
 				}
 
-				std::string LastCurrentOp;
-				uint32_t	LastCurrentOpPercentComplete = 0;
+				ProgressBar ProgressBar(PlainProgress);
+
+				auto OuputMessages = [&](CbObjectView StatusObject) {
+					CbArrayView Messages = StatusObject["Messages"sv].AsArrayView();
+					if (Messages.Num() > 0)
+					{
+						ProgressBar.ForceLinebreak();
+						for (auto M : Messages)
+						{
+							std::string_view Message = M.AsString();
+							ZEN_CONSOLE("{}", Message);
+						}
+					}
+				};
 
 				uint64_t JobId = JobIdMaybe.value();
 				while (true)
@@ -110,27 +123,47 @@ namespace {
 					{
 						StatusResult.ThrowError("failed to create project"sv);
 					}
+
 					CbObject		 StatusObject = StatusResult.AsObject();
 					std::string_view Status		  = StatusObject["Status"sv].AsString();
 
+					bool MessagesDone = false;
+
 					if (Status == "Running")
 					{
-						std::string_view CurrentOp				  = StatusObject["CurrentOp"sv].AsString();
-						uint32_t		 CurrentOpPercentComplete = StatusObject["CurrentOpPercentComplete"sv].AsUInt32();
-						if (CurrentOp != LastCurrentOp || CurrentOpPercentComplete != LastCurrentOpPercentComplete)
+						std::string_view CurrentOp		  = StatusObject["Op"sv].AsString();
+						std::string_view CurrentOpDetails = StatusObject["Details"sv].AsString();
+						uint64_t		 TotalCount		  = StatusObject["TotalCount"sv].AsUInt64();
+						uint64_t		 RemainingCount	  = StatusObject["RemainingCount"sv].AsUInt64();
+
+						if (!ProgressBar.IsSameTask(CurrentOp))
 						{
-							LastCurrentOp				 = CurrentOp;
-							LastCurrentOpPercentComplete = CurrentOpPercentComplete;
-							ZEN_CONSOLE("{} {}%", CurrentOp, CurrentOpPercentComplete);
+							ProgressBar.Finish();
 						}
+
+						if (!ProgressBar.HasActiveTask())
+						{
+							OuputMessages(StatusObject);
+							MessagesDone = true;
+						}
+
+						ProgressBar.UpdateState({.Task			 = std::string(CurrentOp),
+												 .Details		 = std::string(CurrentOpDetails),
+												 .TotalCount	 = TotalCount,
+												 .RemainingCount = RemainingCount},
+												false);
 					}
 
-					CbArrayView Messages = StatusObject["Messages"sv].AsArrayView();
-					for (auto M : Messages)
+					if ((Status == "Complete") || (Status == "Aborted"))
 					{
-						std::string_view Message = M.AsString();
-						ZEN_CONSOLE("{}", Message);
+						ProgressBar.Finish();
 					}
+
+					if (!MessagesDone)
+					{
+						OuputMessages(StatusObject);
+					}
+
 					if (Status == "Complete")
 					{
 						if (Cancelled)
@@ -178,16 +211,25 @@ namespace {
 #endif	// ZEN_PLATFORM_WINDOWS
 						if (HttpClient::Response DeleteResult = Http.Delete(fmt::format("/admin/jobs/{}", JobId)))
 						{
+							ProgressBar.ForceLinebreak();
 							ZEN_CONSOLE("Requested cancel...");
 							Cancelled = true;
 						}
 						else
 						{
+							ProgressBar.ForceLinebreak();
 							ZEN_CONSOLE("Failed cancelling job {}", DeleteResult);
 						}
 						continue;
 					}
-					Sleep(100);
+					if (PlainProgress)
+					{
+						Sleep(5000);
+					}
+					else
+					{
+						Sleep(500);
+					}
 				}
 			}
 			else
@@ -749,7 +791,7 @@ ExportOplogCommand::ExportOplogCommand()
 	m_Options.add_option("",
 						 "",
 						 "ignore-missing-attachments",
-						 "Continue importing oplog even if attachments are missing",
+						 "Continue exporting oplog even if attachments are missing",
 						 cxxopts::value(m_IgnoreMissingAttachments),
 						 "<ignore>");
 	m_Options.add_option("",
@@ -818,6 +860,8 @@ ExportOplogCommand::ExportOplogCommand()
 						 cxxopts::value(m_FileForceEnableTempBlocks),
 						 "<forcetempblocks>");
 
+	m_Options.add_option("", "", "plainprogress", "Use (legacy) plain progress update", cxxopts::value(m_PlainProgress), "<plainprogress>");
+
 	m_Options.parse_positional({"project", "oplog"});
 }
 
@@ -1113,7 +1157,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 	}
 	else
 	{
-		ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload));
+		ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload), m_PlainProgress);
 	}
 	return 0;
 }
@@ -1182,6 +1226,8 @@ ImportOplogCommand::ImportOplogCommand()
 	m_Options.add_option("", "", "file", "Local folder path", cxxopts::value(m_FileDirectoryPath), "<path>");
 	m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), "<filename>");
 
+	m_Options.add_option("", "", "plainprogress", "Use (legacy) plain progress update", cxxopts::value(m_PlainProgress), "<plainprogress>");
+
 	m_Options.parse_positional({"project", "oplog", "gcpath"});
 	m_Options.positional_help("[<projectid> <oplogid> [<gcpath>]]");
 }
@@ -1332,6 +1378,10 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 			{
 				Writer.AddBool("clean"sv, true);
 			}
+			if (m_Force)
+			{
+				Writer.AddBool("force"sv, true);
+			}
 			if (!m_FileDirectoryPath.empty())
 			{
 				Writer.BeginObject("file"sv);
@@ -1421,7 +1471,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 	}
 	else
 	{
-		ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload));
+		ExecuteAsyncOperation(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload), m_PlainProgress);
 	}
 	return 0;
 }
@@ -1761,8 +1811,12 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 	size_t			 WorkerCount = Min(std::thread::hardware_concurrency(), 16u);
 	WorkerThreadPool WorkerPool(gsl::narrow<int>(WorkerCount));
 	Latch			 WorkRemaining(1);
+	size_t			 EmitCount = 0;
 
 	std::unordered_set<std::u8string> FileNames;
+	std::atomic<uint64_t>			  WrittenByteCount = 0;
+
+	Stopwatch WriteStopWatch;
 
 	auto EmitFilesForDataArray = [&](CbArrayView DataArray) {
 		for (auto DataIter : DataArray)
@@ -1783,36 +1837,43 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				{
 					continue;
 				}
+				EmitCount++;
 				WorkRemaining.AddCount(1);
-				WorkerPool.ScheduleWork([this, &RootPath, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining]() {
-					auto _ = MakeGuard([&WorkRemaining]() { WorkRemaining.CountDown(); });
-					if (HttpClient::Response ChunkResponse =
-							Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath))
-					{
-						auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer {
-							IoHash	 RawHash;
-							uint64_t RawSize;
-							if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize))
-							{
-								return Compressed.Decompress().AsIoBuffer();
+				WorkerPool.ScheduleWork(
+					[this, &RootPath, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining, &WrittenByteCount]() {
+						auto _ = MakeGuard([&WorkRemaining]() { WorkRemaining.CountDown(); });
+						if (HttpClient::Response ChunkResponse =
+								Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath))
+						{
+							auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer {
+								IoHash	 RawHash;
+								uint64_t RawSize;
+								if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize))
+								{
+									return Compressed.Decompress().AsIoBuffer();
+								};
+								return std::move(Buffer);
 							};
-							return std::move(Buffer);
-						};
 
-						IoBuffer ChunkData = m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload;
+							IoBuffer ChunkData =
+								m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload;
 
-						std::filesystem::path TargetPath = RootPath / FileName;
-						if (!MoveToFile(TargetPath, ChunkData))
+							std::filesystem::path TargetPath = RootPath / FileName;
+							if (!MoveToFile(TargetPath, ChunkData))
+							{
+								WriteFile(TargetPath, ChunkData);
+							}
+							WrittenByteCount.fetch_add(ChunkData.GetSize());
+							++FileCount;
+						}
+						else
 						{
-							WriteFile(TargetPath, ChunkData);
+							ZEN_CONSOLE("Unable to fetch '{}' (chunk {}). Reason: '{}'",
+										FileName,
+										ChunkId,
+										ChunkResponse.ErrorMessage(""sv));
 						}
-						++FileCount;
-					}
-					else
-					{
-						ZEN_CONSOLE("Unable to fetch '{}' (chunk {}). Reason: '{}'", FileName, ChunkId, ChunkResponse.ErrorMessage(""sv));
-					}
-				});
+					});
 			}
 		}
 	};
@@ -1823,21 +1884,66 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 	{
 		if (CbObject ResponseObject = Response.AsObject())
 		{
-			for (auto EntryIter : ResponseObject["entries"sv])
+			std::unique_ptr<ProgressBar> EmitProgressBar;
 			{
-				CbObjectView Entry = EntryIter.AsObjectView();
-				if (!m_KeyFilter.empty())
+				ProgressBar ParseProgressBar(false);
+				CbArrayView Entries	  = ResponseObject["entries"sv].AsArrayView();
+				uint64_t	Remaining = Entries.Num();
+				for (auto EntryIter : Entries)
 				{
-					// ZEN_CONSOLE("{}", Entry["key"].AsString());
-					if (Entry["key"].AsString().find(m_KeyFilter) == std::string_view::npos)
+					CbObjectView Entry = EntryIter.AsObjectView();
+					ParseProgressBar.UpdateState(
+						{.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining},
+						false);
+					Remaining--;
+					if (!m_KeyFilter.empty())
 					{
-						continue;
+						// ZEN_CONSOLE("{}", Entry["key"].AsString());
+						if (Entry["key"].AsString().find(m_KeyFilter) == std::string_view::npos)
+						{
+							continue;
+						}
+					}
+					if (!EmitProgressBar)
+					{
+						EmitProgressBar = std::make_unique<ProgressBar>(false);
+						WriteStopWatch.Reset();
 					}
+
+					EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView());
+					EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView());
+
+					++OplogEntryCount;
 				}
-				EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView());
-				EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView());
+				ParseProgressBar.Finish();
+			}
 
-				++OplogEntryCount;
+			WorkRemaining.CountDown();
+			if (EmitProgressBar)
+			{
+				while (!WorkRemaining.Wait(200))
+				{
+					uint64_t EmitRemaining = gsl::narrow<uint64_t>(WorkRemaining.Remaining());
+
+					uint64_t WrittenBytes	= WrittenByteCount.load();
+					uint64_t ElapsedTimeMS	= WriteStopWatch.GetElapsedTimeMs();
+					uint64_t BytesPerSecond = ElapsedTimeMS > 0 ? (1000 * WrittenBytes) / ElapsedTimeMS : WrittenBytes;
+
+					EmitProgressBar->UpdateState({.Task			  = "Writing files",
+												  .Details		  = fmt::format("{}/{} files, {}/sec ({})",
+																			EmitCount - EmitRemaining,
+																			EmitCount,
+																			NiceBytes(BytesPerSecond),
+																			NiceBytes(WrittenByteCount.load())),
+												  .TotalCount	  = EmitCount,
+												  .RemainingCount = EmitRemaining},
+												 false);
+				}
+				EmitProgressBar->Finish();
+			}
+			else
+			{
+				WorkRemaining.Wait();
 			}
 		}
 		else
@@ -1851,8 +1957,6 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 
 		return 1;
 	}
-	WorkRemaining.CountDown();
-	WorkRemaining.Wait();
 
 	std::filesystem::remove_all(TmpPath);
 
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index b7fc045f8..e70254ec5 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -125,6 +125,7 @@ private:
 	std::string m_FileName;
 	std::string m_BaseFileName;
 	bool		m_FileForceEnableTempBlocks = false;
+	bool		m_PlainProgress				= false;
 };
 
 class ImportOplogCommand : public ProjectStoreCommand
@@ -149,6 +150,7 @@ private:
 	bool			 m_Async					= false;
 	bool			 m_IgnoreMissingAttachments = false;
 	bool			 m_Clean					= false;
+	bool			 m_PlainProgress			= false;
 
 	std::string m_CloudUrl;
 	std::string m_CloudNamespace;
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 026e8c84f..eb67649b8 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -25,6 +25,7 @@
 #include "cmds/workspaces_cmd.h"
 
 #include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/string.h>
@@ -252,6 +253,106 @@ ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec)
 	return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy);
 }
 
+ProgressBar::ProgressBar(bool PlainProgress) : m_PlainProgress(PlainProgress), m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000)
+{
+}
+
+ProgressBar::~ProgressBar()
+{
+	try
+	{
+		Finish();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("ProgressBar::~ProgressBar() failed with {}", Ex.what());
+	}
+}
+
+void
+ProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
+{
+	if (DoLinebreak == false && m_State == NewState)
+	{
+		return;
+	}
+
+	uint64_t ElapsedTimeMS = m_SW.GetElapsedTimeMs();
+	if (!DoLinebreak && (NewState.Task == m_State.Task) && ((m_LastUpdateMS + 200) > ElapsedTimeMS))
+	{
+		return;
+	}
+
+	m_LastUpdateMS = ElapsedTimeMS;
+
+	size_t PercentDone =
+		NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u;
+
+	if (m_PlainProgress)
+	{
+		ZEN_CONSOLE("{} {}% ({})", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS));
+	}
+	else
+	{
+		size_t ProgressBarSize = 20;
+
+		size_t		ProgressBarCount = (ProgressBarSize * PercentDone) / 100;
+		uint64_t	Completed		 = NewState.TotalCount - NewState.RemainingCount;
+		uint64_t	ETAMS			 = (Completed > 0) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0;
+		std::string ETA				 = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : "";
+
+		std::string			   Output	   = fmt::format("\r{} {:#3}%: |{}{}|: {}{}{}",
+										 NewState.Task,
+										 PercentDone,
+										 std::string(ProgressBarCount, '#'),
+										 std::string(ProgressBarSize - ProgressBarCount, ' '),
+										 NiceTimeSpanMs(ElapsedTimeMS),
+										 ETA,
+										 NewState.Details.empty() ? "" : fmt::format(". {}", NewState.Details));
+		std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0;
+		printf("%s%s%s", Output.c_str(), std::string(EraseLength, ' ').c_str(), DoLinebreak ? "\n" : "");
+		m_LastOutputLength = DoLinebreak ? 0 : Output.length();
+		m_State			   = NewState;
+	}
+}
+
+void
+ProgressBar::ForceLinebreak()
+{
+	if (m_LastOutputLength > 0)
+	{
+		State NewState = m_State;
+		UpdateState(NewState, /*DoLinebreak*/ true);
+	}
+}
+
+void
+ProgressBar::Finish()
+{
+	if (m_LastOutputLength > 0 && m_State.RemainingCount > 0)
+	{
+		State NewState			= m_State;
+		NewState.RemainingCount = 0;
+		NewState.Details		= "";
+		UpdateState(NewState, /*DoLinebreak*/ true);
+	}
+	m_State			   = State{};
+	m_LastOutputLength = 0;
+	m_SW.Reset();
+}
+
+bool
+ProgressBar::IsSameTask(std::string_view Task) const
+{
+	return Task == m_State.Task;
+}
+
+bool
+ProgressBar::HasActiveTask() const
+{
+	return !m_State.Task.empty();
+}
+
 }  // namespace zen
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/src/zen/zen.h b/src/zen/zen.h
index c26e164f7..9c9586050 100644
--- a/src/zen/zen.h
+++ b/src/zen/zen.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <zencore/except.h>
+#include <zencore/timer.h>
 #include <zencore/zencore.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -71,4 +72,33 @@ class CacheStoreCommand : public ZenCmdBase
 	virtual ZenCmdCategory& CommandCategory() const override { return g_CacheStoreCategory; }
 };
 
+class ProgressBar
+{
+public:
+	struct State
+	{
+		bool		operator==(const State&) const = default;
+		std::string Task;
+		std::string Details;
+		uint64_t	TotalCount	   = 0;
+		uint64_t	RemainingCount = 0;
+	};
+
+	explicit ProgressBar(bool PlainProgress);
+	~ProgressBar();
+
+	void UpdateState(const State& NewState, bool DoLinebreak);
+	void ForceLinebreak();
+	void Finish();
+	bool IsSameTask(std::string_view Task) const;
+	bool HasActiveTask() const;
+
+private:
+	const bool m_PlainProgress;
+	Stopwatch  m_SW;
+	uint64_t   m_LastUpdateMS;
+	State	   m_State;
+	size_t	   m_LastOutputLength = 0;
+};
+
 }  // namespace zen
diff --git a/src/zencore/include/zencore/jobqueue.h b/src/zencore/include/zencore/jobqueue.h
index 06143791c..d5ec6255a 100644
--- a/src/zencore/include/zencore/jobqueue.h
+++ b/src/zencore/include/zencore/jobqueue.h
@@ -22,9 +22,10 @@ class JobQueue;
 class JobContext
 {
 public:
-	virtual bool IsCancelled() const														   = 0;
-	virtual void ReportMessage(std::string_view Message)									   = 0;
-	virtual void ReportProgress(std::string_view CurrentOp, uint32_t CurrentOpPercentComplete) = 0;
+	virtual bool IsCancelled() const					 = 0;
+	virtual void ReportMessage(std::string_view Message) = 0;
+	//	virtual void ReportProgress(std::string_view CurrentOp, uint32_t CurrentOpPercentComplete) = 0;
+	virtual void ReportProgress(std::string_view CurrentOp, std::string_view Details, ptrdiff_t TotalCount, ptrdiff_t RemainingCount) = 0;
 };
 
 class JobQueue
@@ -48,7 +49,9 @@ public:
 	struct State
 	{
 		std::string				 CurrentOp;
-		uint32_t				 CurrentOpPercentComplete = 0;
+		std::string				 CurrentOpDetails;
+		ptrdiff_t				 TotalCount;
+		ptrdiff_t				 RemainingCount;
 		std::vector<std::string> Messages;
 		std::string				 AbortReason;
 	};
diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp
index d26d0dd1e..b97484458 100644
--- a/src/zencore/jobqueue.cpp
+++ b/src/zencore/jobqueue.cpp
@@ -11,6 +11,10 @@
 #	include <zencore/testing.h>
 #endif	// ZEN_WITH_TESTS
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 #include <deque>
 #include <thread>
 #include <unordered_map>
@@ -49,9 +53,12 @@ public:
 
 		virtual bool IsCancelled() const override { return CancelFlag.load(); }
 		virtual void ReportMessage(std::string_view Message) override { Queue->ReportMessage(Id, Message); }
-		virtual void ReportProgress(std::string_view CurrentOp, uint32_t CurrentOpPercentComplete) override
+		virtual void ReportProgress(std::string_view CurrentOp,
+									std::string_view Details,
+									ptrdiff_t		 TotalCount,
+									ptrdiff_t		 RemainingCount) override
 		{
-			Queue->ReportProgress(Id, CurrentOp, CurrentOpPercentComplete);
+			Queue->ReportProgress(Id, CurrentOp, Details, TotalCount, RemainingCount);
 		}
 	};
 
@@ -254,16 +261,20 @@ public:
 	virtual std::optional<JobDetails> Get(JobId Id) override
 	{
 		auto Convert = [](Status Status, Job& Job) -> JobDetails {
-			return JobDetails{.Name			  = Job.Name,
-							  .Status		  = Status,
-							  .State		  = {.CurrentOp				   = Job.State.CurrentOp,
-												 .CurrentOpPercentComplete = Job.State.CurrentOpPercentComplete,
-												 .Messages				   = std::move(Job.State.Messages),
-												 .AbortReason			   = Job.State.AbortReason},
-							  .CreateTime	  = JobClock::TimePointFromTick(Job.CreateTick),
-							  .StartTime	  = JobClock::TimePointFromTick(Job.StartTick),
-							  .EndTime		  = JobClock::TimePointFromTick(Job.EndTick),
-							  .WorkerThreadId = Job.WorkerThreadId};
+			return JobDetails{
+				.Name			= Job.Name,
+				.Status			= Status,
+				.State			= {.CurrentOp		 = Job.State.CurrentOp,
+								   .CurrentOpDetails = Job.State.CurrentOpDetails,
+								   .TotalCount		 = Job.State.TotalCount,
+								   .RemainingCount	 = Job.State.RemainingCount,
+						   //												 .CurrentOpPercentComplete = Job.State.CurrentOpPercentComplete,
+								   .Messages	= std::move(Job.State.Messages),
+								   .AbortReason = Job.State.AbortReason},
+				.CreateTime		= JobClock::TimePointFromTick(Job.CreateTick),
+				.StartTime		= JobClock::TimePointFromTick(Job.StartTick),
+				.EndTime		= JobClock::TimePointFromTick(Job.EndTick),
+				.WorkerThreadId = Job.WorkerThreadId};
 		};
 
 		std::optional<JobDetails> Result;
@@ -304,13 +315,15 @@ public:
 		});
 	}
 
-	void ReportProgress(JobId Id, std::string_view CurrentOp, uint32_t CurrentOpPercentComplete)
+	void ReportProgress(JobId Id, std::string_view CurrentOp, std::string_view Details, ptrdiff_t TotalCount, ptrdiff_t RemainingCount)
 	{
 		QueueLock.WithExclusiveLock([&]() {
 			auto It = RunningJobs.find(Id.Id);
 			ZEN_ASSERT(It != RunningJobs.end());
-			It->second->State.CurrentOp				   = CurrentOp;
-			It->second->State.CurrentOpPercentComplete = CurrentOpPercentComplete;
+			It->second->State.CurrentOp		   = CurrentOp;
+			It->second->State.CurrentOpDetails = Details;
+			It->second->State.TotalCount	   = TotalCount;
+			It->second->State.RemainingCount   = RemainingCount;
 		});
 	}
 
@@ -442,13 +455,13 @@ TEST_CASE("JobQueue")
 				{
 					return;
 				}
-				Context.ReportProgress("going to sleep", 0);
+				Context.ReportProgress("going to sleep", "", 100, 100);
 				Sleep(10);
 				if (Context.IsCancelled())
 				{
 					return;
 				}
-				Context.ReportProgress("going to sleep again", 50);
+				Context.ReportProgress("going to sleep again", "", 100, 50);
 				if ((I & 0xFF) == 0x10)
 				{
 					zen::ThrowSystemError(8, fmt::format("Job {} forced to fail", I));
@@ -458,7 +471,7 @@ TEST_CASE("JobQueue")
 				{
 					return;
 				}
-				Context.ReportProgress("done", 100);
+				Context.ReportProgress("done", "", 100, 0);
 			});
 		});
 	}
@@ -507,11 +520,16 @@ TEST_CASE("JobQueue")
 					RemainingJobs.push_back(Id);
 					break;
 				case JobQueue::Status::Running:
-					ZEN_DEBUG("{} running. '{}' {}% '{}'",
-							  Id.Id,
-							  CurrentState->State.CurrentOp,
-							  CurrentState->State.CurrentOpPercentComplete,
-							  Join(CurrentState->State.Messages, " "sv));
+					ZEN_DEBUG(
+						"{} running. '{}{}' {}% '{}'",
+						Id.Id,
+						CurrentState->State.CurrentOp,
+						CurrentState->State.CurrentOpDetails.empty() ? ""sv : fmt::format(", {}", CurrentState->State.CurrentOpDetails),
+						CurrentState->State.TotalCount > 0
+							? gsl::narrow<uint32_t>((100 * (CurrentState->State.TotalCount - CurrentState->State.RemainingCount)) /
+													CurrentState->State.TotalCount)
+							: 0,
+						Join(CurrentState->State.Messages, " "sv));
 					RemainingJobs.push_back(Id);
 					break;
 				case JobQueue::Status::Aborted:
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index f6311cf3e..ea830923f 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -159,8 +159,20 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 			auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) {
 				if (!State.CurrentOp.empty())
 				{
-					Obj.AddString("CurrentOp"sv, State.CurrentOp);
-					Obj.AddInteger("CurrentOpPercentComplete"sv, State.CurrentOpPercentComplete);
+					Obj.AddString(
+						"CurrentOp"sv,
+						State.CurrentOpDetails.empty() ? State.CurrentOp : fmt::format("{}: {}", State.CurrentOp, State.CurrentOpDetails));
+					Obj.AddString("Op"sv, State.CurrentOp);
+					if (!State.CurrentOpDetails.empty())
+					{
+						Obj.AddString("Details"sv, State.CurrentOpDetails);
+					}
+					Obj.AddInteger("TotalCount"sv, gsl::narrow<uint64_t>(State.TotalCount));
+					Obj.AddInteger("RemainingCount"sv, gsl::narrow<uint64_t>(State.RemainingCount));
+					Obj.AddInteger("CurrentOpPercentComplete"sv,
+								   State.TotalCount > 0
+									   ? gsl::narrow<uint32_t>((100 * (State.TotalCount - State.RemainingCount)) / State.TotalCount)
+									   : 0);
 				}
 				if (!State.Messages.empty())
 				{
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 5f4f44e31..8dbd94d39 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -194,7 +194,7 @@ namespace {
 		{
 			for (const std::string& OpLogId : OpLogs)
 			{
-				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false);
+				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 				if (Oplog != nullptr)
 				{
 					CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
@@ -471,7 +471,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -948,7 +948,7 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1026,7 +1026,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1168,7 +1168,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1265,7 +1265,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 	{
 		case HttpVerb::kGet:
 			{
-				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 				if (!OplogIt)
 				{
 					return HttpReq.WriteResponse(HttpResponseCode::NotFound,
@@ -1299,7 +1299,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 					OplogMarkerPath = Params["gcpath"sv].AsString();
 				}
 
-				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false);
+				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 				if (!OplogIt)
 				{
 					if (!Project->NewOplog(OplogId, OplogMarkerPath))
@@ -1336,7 +1336,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 					OplogMarkerPath = Params["gcpath"sv].AsString();
 				}
 
-				ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false);
+				ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 				if (!FoundLog)
 				{
 					if (!Project->NewOplog(OplogId, OplogMarkerPath))
@@ -1426,7 +1426,7 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1917,7 +1917,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false);
+	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1972,7 +1972,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false);
+	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 0e6b2d3c9..25be159b9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1151,6 +1151,12 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath)
 	return std::filesystem::is_regular_file(StateFilePath);
 }
 
+bool
+ProjectStore::Oplog::Exists() const
+{
+	return ExistsAt(m_BasePath);
+}
+
 void
 ProjectStore::Oplog::Read()
 {
@@ -2925,22 +2931,33 @@ ProjectStore::Project::NewOplog(std::string_view OplogId, const std::filesystem:
 }
 
 ProjectStore::Oplog*
-ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact)
+ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bool VerifyPathOnDisk)
 {
 	ZEN_TRACE_CPU("Store::OpenOplog");
 
+	std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
 	{
-		RwLock::SharedLockScope _(m_ProjectLock);
+		RwLock::SharedLockScope ProjectLock(m_ProjectLock);
 
 		auto OplogIt = m_Oplogs.find(std::string(OplogId));
 
 		if (OplogIt != m_Oplogs.end())
 		{
-			return OplogIt->second.get();
+			if (!VerifyPathOnDisk || Oplog::ExistsAt(OplogBasePath))
+			{
+				return OplogIt->second.get();
+			}
+
+			// Somebody deleted the oplog on disk behind our back
+			ProjectLock.ReleaseNow();
+			std::filesystem::path DeletePath;
+			if (!RemoveOplog(OplogId, DeletePath))
+			{
+				ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+			}
 		}
 	}
 
-	std::filesystem::path	   OplogBasePath = BasePathForOplog(OplogId);
 	RwLock::ExclusiveLockScope Lock(m_ProjectLock);
 	if (auto It = m_Oplogs.find(std::string{OplogId}); It != m_Oplogs.end())
 	{
@@ -3114,7 +3131,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
 	std::vector<std::string> OpLogs = ScanForOplogs();
 	for (const std::string& OpLogId : OpLogs)
 	{
-		OpenOplog(OpLogId, /*AllowCompact*/ false);
+		OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 	}
 	IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) {
 		if (!IsExpired(GcClock::TimePoint::min(), Ops))
@@ -3711,7 +3728,7 @@ ProjectStore::GetProjectFiles(const std::string_view				 ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -3843,7 +3860,7 @@ ProjectStore::GetProjectChunkInfos(const std::string_view				  ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -3962,7 +3979,7 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk info request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4042,7 +4059,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4152,7 +4169,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4199,7 +4216,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk put request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4241,7 +4258,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!Oplog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4326,7 +4343,7 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!Oplog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4456,7 +4473,7 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!Oplog)
 	{
 		HttpReq.WriteResponse(HttpResponseCode::NotFound,
@@ -5685,7 +5702,7 @@ TEST_CASE("project.store.lifetimes")
 	std::filesystem::path DeletePath;
 	CHECK(Project->PrepareForDelete(DeletePath));
 	CHECK(!DeletePath.empty());
-	CHECK(Project->OpenOplog("oplog1", /*AllowCompact*/ false) == nullptr);
+	CHECK(Project->OpenOplog("oplog1", /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true) == nullptr);
 	// Oplog is now invalid, but pointer can still be accessed since we store old oplog pointers
 	CHECK(Oplog->OplogCount() == 0);
 	// Project is still valid since we have a Ref to it
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 10c0ed8da..49970b677 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -82,6 +82,7 @@ public:
 		~Oplog();
 
 		[[nodiscard]] static bool ExistsAt(const std::filesystem::path& BasePath);
+		bool					  Exists() const;
 
 		void Read();
 		void Write();
@@ -275,7 +276,7 @@ public:
 		std::filesystem::path ProjectFilePath;
 
 		Oplog*					 NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath);
-		Oplog*					 OpenOplog(std::string_view OplogId, bool AllowCompact);
+		Oplog*					 OpenOplog(std::string_view OplogId, bool AllowCompact, bool VerifyPathOnDisk);
 		bool					 DeleteOplog(std::string_view OplogId);
 		bool					 RemoveOplog(std::string_view OplogId, std::filesystem::path& OutDeletePath);
 		void					 IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const;
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index ab31d5ec5..3c44432b5 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -84,12 +84,12 @@ private:
 };
 
 void
-ReportProgress(JobContext* OptionalContext, std::string_view CurrentOp, ptrdiff_t Total, ptrdiff_t Remaining)
+ReportProgress(JobContext* OptionalContext, std::string_view CurrentOp, std::string_view Details, ptrdiff_t Total, ptrdiff_t Remaining)
 {
 	if (OptionalContext)
 	{
 		ZEN_ASSERT(Total > 0);
-		OptionalContext->ReportProgress(CurrentOp, gsl::narrow<uint32_t>((100 * (Total - Remaining)) / Total));
+		OptionalContext->ReportProgress(CurrentOp, Details, Total, Remaining);
 	}
 }
 
@@ -529,7 +529,8 @@ BuildContainer(CidStore&																	ChunkStore,
 					if (OpCount % 1000 == 0)
 					{
 						ReportProgress(OptionalContext,
-									   fmt::format("Building oplog: {} ops processed", OpCount),
+									   "Building oplog"sv,
+									   fmt::format("{} ops processed", OpCount),
 									   TotalOpCount,
 									   TotalOpCount - OpCount);
 					}
@@ -547,7 +548,7 @@ BuildContainer(CidStore&																	ChunkStore,
 				}
 				if (TotalOpCount > 0)
 				{
-					ReportProgress(OptionalContext, fmt::format("Building oplog: {} ops processed", OpCount), TotalOpCount, 0);
+					ReportProgress(OptionalContext, "Building oplog"sv, fmt::format("{} ops processed", OpCount), TotalOpCount, 0);
 				}
 			}
 			SectionOpsWriter.EndArray();  // "ops"
@@ -911,21 +912,23 @@ BuildContainer(CidStore&																	ChunkStore,
 				{
 					Remaining = ResolveAttachmentsLatch.Remaining();
 					ReportProgress(OptionalContext,
+								   "Resolving attachments"sv,
 								   fmt::format("Aborting, {} attachments remaining...", Remaining),
 								   UploadAttachments.size(),
 								   Remaining);
 				}
-				ReportProgress(OptionalContext, fmt::format("Resolving attachments, {} remaining...", 0), UploadAttachments.size(), 0);
+				ReportProgress(OptionalContext, "Resolving attachments"sv, "Aborted"sv, UploadAttachments.size(), 0);
 				return {};
 			}
 			ReportProgress(OptionalContext,
-						   fmt::format("Resolving attachments, {} remaining...", Remaining),
+						   "Resolving attachments"sv,
+						   fmt::format("{} remaining...", Remaining),
 						   UploadAttachments.size(),
 						   Remaining);
 		}
 		if (UploadAttachments.size() > 0)
 		{
-			ReportProgress(OptionalContext, fmt::format("Resolving attachments, {} remaining...", 0), UploadAttachments.size(), 0);
+			ReportProgress(OptionalContext, "Resolving attachments"sv, ""sv, UploadAttachments.size(), 0);
 		}
 
 		if (IsCancelled(OptionalContext))
@@ -1127,11 +1130,11 @@ BuildContainer(CidStore&																	ChunkStore,
 				}
 				if (ChunksAssembled % 1000 == 0)
 				{
-					ReportProgress(
-						OptionalContext,
-						fmt::format("Assembling blocks: {} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
-						ChunkAssembleCount,
-						ChunkAssembleCount - ChunksAssembled);
+					ReportProgress(OptionalContext,
+								   "Assembling blocks"sv,
+								   fmt::format("{} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
+								   ChunkAssembleCount,
+								   ChunkAssembleCount - ChunksAssembled);
 				}
 				const IoHash& RawHash(HashIt->first);
 				const Oid	  CurrentOpKey = HashIt->second;
@@ -1191,9 +1194,8 @@ BuildContainer(CidStore&																	ChunkStore,
 						if (ChunksAssembled % 1000 == 0)
 						{
 							ReportProgress(OptionalContext,
-										   fmt::format("Assembling blocks: {} attachments processed, {} blocks assembled",
-													   ChunksAssembled,
-													   ComposedBlocks),
+										   "Assembling blocks"sv,
+										   fmt::format("{} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
 										   ChunkAssembleCount,
 										   ChunkAssembleCount - ChunksAssembled);
 						}
@@ -1237,11 +1239,11 @@ BuildContainer(CidStore&																	ChunkStore,
 
 			if (ChunkAssembleCount > 0)
 			{
-				ReportProgress(
-					OptionalContext,
-					fmt::format("Assembling blocks: {} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
-					ChunkAssembleCount,
-					0);
+				ReportProgress(OptionalContext,
+							   "Assembling blocks"sv,
+							   fmt::format("{} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
+							   ChunkAssembleCount,
+							   0);
 			}
 
 			ReportMessage(OptionalContext,
@@ -1260,13 +1262,18 @@ BuildContainer(CidStore&																	ChunkStore,
 				{
 					ptrdiff_t Remaining = BlockCreateLatch.Remaining();
 					ReportProgress(OptionalContext,
+								   "Assembling blocks"sv,
 								   fmt::format("Aborting, {} blocks remaining...", Remaining),
 								   GeneratedBlockCount,
 								   Remaining);
 				}
 				if (GeneratedBlockCount > 0)
 				{
-					ReportProgress(OptionalContext, fmt::format("Aborting, {} blocks remaining...", 0), GeneratedBlockCount, 0);
+					ReportProgress(OptionalContext,
+								   "Assembling blocks"sv,
+								   fmt::format("Aborting, {} blocks remaining...", 0),
+								   GeneratedBlockCount,
+								   0);
 				}
 				return {};
 			}
@@ -1293,20 +1300,21 @@ BuildContainer(CidStore&																	ChunkStore,
 				{
 					Remaining = BlockCreateLatch.Remaining();
 					ReportProgress(OptionalContext,
+								   "Creating blocks"sv,
 								   fmt::format("Aborting, {} blocks remaining...", Remaining),
 								   GeneratedBlockCount,
 								   Remaining);
 				}
-				ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", 0), GeneratedBlockCount, 0);
+				ReportProgress(OptionalContext, "Creating blocks"sv, "Aborted"sv, GeneratedBlockCount, 0);
 				return {};
 			}
-			ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", Remaining), GeneratedBlockCount, Remaining);
+			ReportProgress(OptionalContext, "Creating blocks"sv, fmt::format("{} remaining...", Remaining), GeneratedBlockCount, Remaining);
 		}
 
 		if (GeneratedBlockCount > 0)
 		{
 			uint64_t NowMS = Timer.GetElapsedTimeMs();
-			ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", 0), GeneratedBlockCount, 0);
+			ReportProgress(OptionalContext, "Creating blocks"sv, ""sv, GeneratedBlockCount, 0);
 			ReportMessage(OptionalContext,
 						  fmt::format("Created {} blocks in {}", GeneratedBlockCount, NiceTimeSpanMs(NowMS - CreateBlocksStartMS)));
 		}
@@ -1734,17 +1742,18 @@ UploadAttachments(WorkerThreadPool&														  WorkerPool,
 			}
 		}
 		uint64_t PartialTransferWallTimeMS = Timer.GetElapsedTimeMs();
-		ReportProgress(
-			OptionalContext,
-			fmt::format("Saving attachments, {} remaining... {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
-			AttachmentsToSave,
-			Remaining);
+		ReportProgress(OptionalContext,
+					   "Saving attachments"sv,
+					   fmt::format("{} remaining... {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
+					   AttachmentsToSave,
+					   Remaining);
 	}
 	uint64_t ElapsedTimeMS = Timer.GetElapsedTimeMs();
 	if (AttachmentsToSave > 0)
 	{
 		ReportProgress(OptionalContext,
-					   fmt::format("Saving attachments, {} remaining. {}", 0, GetStats(RemoteStore.GetStats(), ElapsedTimeMS)),
+					   "Saving attachments"sv,
+					   fmt::format("{}", GetStats(RemoteStore.GetStats(), ElapsedTimeMS)),
 					   AttachmentsToSave,
 					   0);
 	}
@@ -2296,7 +2305,8 @@ WriteOplogSection(ProjectStore::Oplog& Oplog, const CbObjectView& SectionObject,
 		OpsData.clear();
 		OpDataOffsets.clear();
 		ReportProgress(OptionalContext,
-					   fmt::format("Writing oplog, {} remaining...", OpCount - OpsCompleteCount),
+					   "Writing oplog"sv,
+					   fmt::format("{} remaining...", OpCount - OpsCompleteCount),
 					   OpCount,
 					   OpCount - OpsCompleteCount);
 	};
@@ -2319,6 +2329,9 @@ WriteOplogSection(ProjectStore::Oplog& Oplog, const CbObjectView& SectionObject,
 	{
 		AppendBatch();
 	}
+
+	ReportProgress(OptionalContext, "Writing oplog"sv, ""sv, OpCount, 0);
+
 	return RemoteProjectStore::Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0};
 }
 
@@ -2808,11 +2821,11 @@ LoadOplog(CidStore&			   ChunkStore,
 		{
 			PartialTransferWallTimeMS += LoadAttachmentsTimer.GetElapsedTimeMs() - DownloadStartMS.load();
 		}
-		ReportProgress(
-			OptionalContext,
-			fmt::format("Loading attachments, {} remaining. {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
-			AttachmentCount.load(),
-			Remaining);
+		ReportProgress(OptionalContext,
+					   "Loading attachments"sv,
+					   fmt::format("{} remaining. {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
+					   AttachmentCount.load(),
+					   Remaining);
 	}
 	if (DownloadStartMS != (uint64_t)-1)
 	{
@@ -2822,7 +2835,8 @@ LoadOplog(CidStore&			   ChunkStore,
 	if (AttachmentCount.load() > 0)
 	{
 		ReportProgress(OptionalContext,
-					   fmt::format("Loading attachments, {} remaining. {}", 0, GetStats(RemoteStore.GetStats(), TransferWallTimeMS)),
+					   "Loading attachments"sv,
+					   fmt::format("{}", GetStats(RemoteStore.GetStats(), TransferWallTimeMS)),
 					   AttachmentCount.load(),
 					   0);
 	}
@@ -2838,12 +2852,16 @@ LoadOplog(CidStore&			   ChunkStore,
 				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", "");
 			}
 		}
-		ReportProgress(OptionalContext, fmt::format("Writing attachments, {} remaining.", Remaining), AttachmentCount.load(), Remaining);
+		ReportProgress(OptionalContext,
+					   "Writing attachments"sv,
+					   fmt::format("{} remaining.", Remaining),
+					   AttachmentCount.load(),
+					   Remaining);
 	}
 
 	if (AttachmentCount.load() > 0)
 	{
-		ReportProgress(OptionalContext, fmt::format("Writing attachments, {} remaining.", 0), AttachmentCount.load(), 0);
+		ReportProgress(OptionalContext, "Writing attachments", ""sv, AttachmentCount.load(), 0);
 	}
 
 	if (Result.ErrorCode == 0)
@@ -2959,11 +2977,12 @@ LoadOplog(CidStore&			   ChunkStore,
 					}
 				}
 				ReportProgress(OptionalContext,
-							   fmt::format("Dechunking attachments, {} remaining...", Remaining),
+							   "Dechunking attachments"sv,
+							   fmt::format("{} remaining...", Remaining),
 							   FilesToDechunk.size(),
 							   Remaining);
 			}
-			ReportProgress(OptionalContext, fmt::format("Dechunking attachments, {} remaining...", 0), FilesToDechunk.size(), 0);
+			ReportProgress(OptionalContext, "Dechunking attachments"sv, ""sv, FilesToDechunk.size(), 0);
 		}
 		Result = RemoteResult.ConvertResult();
 	}
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
index 7e4fbe83e..6e14b7632 100644
--- a/src/zenserver/vfs/vfsimpl.cpp
+++ b/src/zenserver/vfs/vfsimpl.cpp
@@ -362,7 +362,7 @@ VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNo
 
 			// Oplog contents enumeration
 
-			if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, false))
+			if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true))
 			{
 				Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId);