Improved oplog import/export progress indicator at commandline (#206)

Nicer progress bar during oplog import/export
Verify that oplog has not been deleted from disk behind our back

6 files changed, 120 insertions, 71 deletions

diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index f6311cf3e..ea830923f 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -159,8 +159,20 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 			auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) {
 				if (!State.CurrentOp.empty())
 				{
-					Obj.AddString("CurrentOp"sv, State.CurrentOp);
-					Obj.AddInteger("CurrentOpPercentComplete"sv, State.CurrentOpPercentComplete);
+					Obj.AddString(
+						"CurrentOp"sv,
+						State.CurrentOpDetails.empty() ? State.CurrentOp : fmt::format("{}: {}", State.CurrentOp, State.CurrentOpDetails));
+					Obj.AddString("Op"sv, State.CurrentOp);
+					if (!State.CurrentOpDetails.empty())
+					{
+						Obj.AddString("Details"sv, State.CurrentOpDetails);
+					}
+					Obj.AddInteger("TotalCount"sv, gsl::narrow<uint64_t>(State.TotalCount));
+					Obj.AddInteger("RemainingCount"sv, gsl::narrow<uint64_t>(State.RemainingCount));
+					Obj.AddInteger("CurrentOpPercentComplete"sv,
+								   State.TotalCount > 0
+									   ? gsl::narrow<uint32_t>((100 * (State.TotalCount - State.RemainingCount)) / State.TotalCount)
+									   : 0);
 				}
 				if (!State.Messages.empty())
 				{
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 5f4f44e31..8dbd94d39 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -194,7 +194,7 @@ namespace {
 		{
 			for (const std::string& OpLogId : OpLogs)
 			{
-				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false);
+				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 				if (Oplog != nullptr)
 				{
 					CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
@@ -471,7 +471,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -948,7 +948,7 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1026,7 +1026,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1168,7 +1168,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1265,7 +1265,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 	{
 		case HttpVerb::kGet:
 			{
-				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 				if (!OplogIt)
 				{
 					return HttpReq.WriteResponse(HttpResponseCode::NotFound,
@@ -1299,7 +1299,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 					OplogMarkerPath = Params["gcpath"sv].AsString();
 				}
 
-				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false);
+				ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 				if (!OplogIt)
 				{
 					if (!Project->NewOplog(OplogId, OplogMarkerPath))
@@ -1336,7 +1336,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
 					OplogMarkerPath = Params["gcpath"sv].AsString();
 				}
 
-				ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false);
+				ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 				if (!FoundLog)
 				{
 					if (!Project->NewOplog(OplogId, OplogMarkerPath))
@@ -1426,7 +1426,7 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1917,7 +1917,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false);
+	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1972,7 +1972,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false);
+	ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 0e6b2d3c9..25be159b9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1151,6 +1151,12 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath)
 	return std::filesystem::is_regular_file(StateFilePath);
 }
 
+bool
+ProjectStore::Oplog::Exists() const
+{
+	return ExistsAt(m_BasePath);
+}
+
 void
 ProjectStore::Oplog::Read()
 {
@@ -2925,22 +2931,33 @@ ProjectStore::Project::NewOplog(std::string_view OplogId, const std::filesystem:
 }
 
 ProjectStore::Oplog*
-ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact)
+ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bool VerifyPathOnDisk)
 {
 	ZEN_TRACE_CPU("Store::OpenOplog");
 
+	std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
 	{
-		RwLock::SharedLockScope _(m_ProjectLock);
+		RwLock::SharedLockScope ProjectLock(m_ProjectLock);
 
 		auto OplogIt = m_Oplogs.find(std::string(OplogId));
 
 		if (OplogIt != m_Oplogs.end())
 		{
-			return OplogIt->second.get();
+			if (!VerifyPathOnDisk || Oplog::ExistsAt(OplogBasePath))
+			{
+				return OplogIt->second.get();
+			}
+
+			// Somebody deleted the oplog on disk behind our back
+			ProjectLock.ReleaseNow();
+			std::filesystem::path DeletePath;
+			if (!RemoveOplog(OplogId, DeletePath))
+			{
+				ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+			}
 		}
 	}
 
-	std::filesystem::path	   OplogBasePath = BasePathForOplog(OplogId);
 	RwLock::ExclusiveLockScope Lock(m_ProjectLock);
 	if (auto It = m_Oplogs.find(std::string{OplogId}); It != m_Oplogs.end())
 	{
@@ -3114,7 +3131,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
 	std::vector<std::string> OpLogs = ScanForOplogs();
 	for (const std::string& OpLogId : OpLogs)
 	{
-		OpenOplog(OpLogId, /*AllowCompact*/ false);
+		OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 	}
 	IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) {
 		if (!IsExpired(GcClock::TimePoint::min(), Ops))
@@ -3711,7 +3728,7 @@ ProjectStore::GetProjectFiles(const std::string_view				 ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -3843,7 +3860,7 @@ ProjectStore::GetProjectChunkInfos(const std::string_view				  ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -3962,7 +3979,7 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk info request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4042,7 +4059,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4152,7 +4169,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4199,7 +4216,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk put request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4241,7 +4258,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!Oplog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4326,7 +4343,7 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!Oplog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4456,7 +4473,7 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!Oplog)
 	{
 		HttpReq.WriteResponse(HttpResponseCode::NotFound,
@@ -5685,7 +5702,7 @@ TEST_CASE("project.store.lifetimes")
 	std::filesystem::path DeletePath;
 	CHECK(Project->PrepareForDelete(DeletePath));
 	CHECK(!DeletePath.empty());
-	CHECK(Project->OpenOplog("oplog1", /*AllowCompact*/ false) == nullptr);
+	CHECK(Project->OpenOplog("oplog1", /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true) == nullptr);
 	// Oplog is now invalid, but pointer can still be accessed since we store old oplog pointers
 	CHECK(Oplog->OplogCount() == 0);
 	// Project is still valid since we have a Ref to it
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 10c0ed8da..49970b677 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -82,6 +82,7 @@ public:
 		~Oplog();
 
 		[[nodiscard]] static bool ExistsAt(const std::filesystem::path& BasePath);
+		bool					  Exists() const;
 
 		void Read();
 		void Write();
@@ -275,7 +276,7 @@ public:
 		std::filesystem::path ProjectFilePath;
 
 		Oplog*					 NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath);
-		Oplog*					 OpenOplog(std::string_view OplogId, bool AllowCompact);
+		Oplog*					 OpenOplog(std::string_view OplogId, bool AllowCompact, bool VerifyPathOnDisk);
 		bool					 DeleteOplog(std::string_view OplogId);
 		bool					 RemoveOplog(std::string_view OplogId, std::filesystem::path& OutDeletePath);
 		void					 IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const;
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index ab31d5ec5..3c44432b5 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -84,12 +84,12 @@ private:
 };
 
 void
-ReportProgress(JobContext* OptionalContext, std::string_view CurrentOp, ptrdiff_t Total, ptrdiff_t Remaining)
+ReportProgress(JobContext* OptionalContext, std::string_view CurrentOp, std::string_view Details, ptrdiff_t Total, ptrdiff_t Remaining)
 {
 	if (OptionalContext)
 	{
 		ZEN_ASSERT(Total > 0);
-		OptionalContext->ReportProgress(CurrentOp, gsl::narrow<uint32_t>((100 * (Total - Remaining)) / Total));
+		OptionalContext->ReportProgress(CurrentOp, Details, Total, Remaining);
 	}
 }
 
@@ -529,7 +529,8 @@ BuildContainer(CidStore&																	ChunkStore,
 					if (OpCount % 1000 == 0)
 					{
 						ReportProgress(OptionalContext,
-									   fmt::format("Building oplog: {} ops processed", OpCount),
+									   "Building oplog"sv,
+									   fmt::format("{} ops processed", OpCount),
 									   TotalOpCount,
 									   TotalOpCount - OpCount);
 					}
@@ -547,7 +548,7 @@ BuildContainer(CidStore&																	ChunkStore,
 				}
 				if (TotalOpCount > 0)
 				{
-					ReportProgress(OptionalContext, fmt::format("Building oplog: {} ops processed", OpCount), TotalOpCount, 0);
+					ReportProgress(OptionalContext, "Building oplog"sv, fmt::format("{} ops processed", OpCount), TotalOpCount, 0);
 				}
 			}
 			SectionOpsWriter.EndArray();  // "ops"
@@ -911,21 +912,23 @@ BuildContainer(CidStore&																	ChunkStore,
 				{
 					Remaining = ResolveAttachmentsLatch.Remaining();
 					ReportProgress(OptionalContext,
+								   "Resolving attachments"sv,
 								   fmt::format("Aborting, {} attachments remaining...", Remaining),
 								   UploadAttachments.size(),
 								   Remaining);
 				}
-				ReportProgress(OptionalContext, fmt::format("Resolving attachments, {} remaining...", 0), UploadAttachments.size(), 0);
+				ReportProgress(OptionalContext, "Resolving attachments"sv, "Aborted"sv, UploadAttachments.size(), 0);
 				return {};
 			}
 			ReportProgress(OptionalContext,
-						   fmt::format("Resolving attachments, {} remaining...", Remaining),
+						   "Resolving attachments"sv,
+						   fmt::format("{} remaining...", Remaining),
 						   UploadAttachments.size(),
 						   Remaining);
 		}
 		if (UploadAttachments.size() > 0)
 		{
-			ReportProgress(OptionalContext, fmt::format("Resolving attachments, {} remaining...", 0), UploadAttachments.size(), 0);
+			ReportProgress(OptionalContext, "Resolving attachments"sv, ""sv, UploadAttachments.size(), 0);
 		}
 
 		if (IsCancelled(OptionalContext))
@@ -1127,11 +1130,11 @@ BuildContainer(CidStore&																	ChunkStore,
 				}
 				if (ChunksAssembled % 1000 == 0)
 				{
-					ReportProgress(
-						OptionalContext,
-						fmt::format("Assembling blocks: {} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
-						ChunkAssembleCount,
-						ChunkAssembleCount - ChunksAssembled);
+					ReportProgress(OptionalContext,
+								   "Assembling blocks"sv,
+								   fmt::format("{} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
+								   ChunkAssembleCount,
+								   ChunkAssembleCount - ChunksAssembled);
 				}
 				const IoHash& RawHash(HashIt->first);
 				const Oid	  CurrentOpKey = HashIt->second;
@@ -1191,9 +1194,8 @@ BuildContainer(CidStore&																	ChunkStore,
 						if (ChunksAssembled % 1000 == 0)
 						{
 							ReportProgress(OptionalContext,
-										   fmt::format("Assembling blocks: {} attachments processed, {} blocks assembled",
-													   ChunksAssembled,
-													   ComposedBlocks),
+										   "Assembling blocks"sv,
+										   fmt::format("{} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
 										   ChunkAssembleCount,
 										   ChunkAssembleCount - ChunksAssembled);
 						}
@@ -1237,11 +1239,11 @@ BuildContainer(CidStore&																	ChunkStore,
 
 			if (ChunkAssembleCount > 0)
 			{
-				ReportProgress(
-					OptionalContext,
-					fmt::format("Assembling blocks: {} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
-					ChunkAssembleCount,
-					0);
+				ReportProgress(OptionalContext,
+							   "Assembling blocks"sv,
+							   fmt::format("{} attachments processed, {} blocks assembled", ChunksAssembled, ComposedBlocks),
+							   ChunkAssembleCount,
+							   0);
 			}
 
 			ReportMessage(OptionalContext,
@@ -1260,13 +1262,18 @@ BuildContainer(CidStore&																	ChunkStore,
 				{
 					ptrdiff_t Remaining = BlockCreateLatch.Remaining();
 					ReportProgress(OptionalContext,
+								   "Assembling blocks"sv,
 								   fmt::format("Aborting, {} blocks remaining...", Remaining),
 								   GeneratedBlockCount,
 								   Remaining);
 				}
 				if (GeneratedBlockCount > 0)
 				{
-					ReportProgress(OptionalContext, fmt::format("Aborting, {} blocks remaining...", 0), GeneratedBlockCount, 0);
+					ReportProgress(OptionalContext,
+								   "Assembling blocks"sv,
+								   fmt::format("Aborting, {} blocks remaining...", 0),
+								   GeneratedBlockCount,
+								   0);
 				}
 				return {};
 			}
@@ -1293,20 +1300,21 @@ BuildContainer(CidStore&																	ChunkStore,
 				{
 					Remaining = BlockCreateLatch.Remaining();
 					ReportProgress(OptionalContext,
+								   "Creating blocks"sv,
 								   fmt::format("Aborting, {} blocks remaining...", Remaining),
 								   GeneratedBlockCount,
 								   Remaining);
 				}
-				ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", 0), GeneratedBlockCount, 0);
+				ReportProgress(OptionalContext, "Creating blocks"sv, "Aborted"sv, GeneratedBlockCount, 0);
 				return {};
 			}
-			ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", Remaining), GeneratedBlockCount, Remaining);
+			ReportProgress(OptionalContext, "Creating blocks"sv, fmt::format("{} remaining...", Remaining), GeneratedBlockCount, Remaining);
 		}
 
 		if (GeneratedBlockCount > 0)
 		{
 			uint64_t NowMS = Timer.GetElapsedTimeMs();
-			ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", 0), GeneratedBlockCount, 0);
+			ReportProgress(OptionalContext, "Creating blocks"sv, ""sv, GeneratedBlockCount, 0);
 			ReportMessage(OptionalContext,
 						  fmt::format("Created {} blocks in {}", GeneratedBlockCount, NiceTimeSpanMs(NowMS - CreateBlocksStartMS)));
 		}
@@ -1734,17 +1742,18 @@ UploadAttachments(WorkerThreadPool&														  WorkerPool,
 			}
 		}
 		uint64_t PartialTransferWallTimeMS = Timer.GetElapsedTimeMs();
-		ReportProgress(
-			OptionalContext,
-			fmt::format("Saving attachments, {} remaining... {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
-			AttachmentsToSave,
-			Remaining);
+		ReportProgress(OptionalContext,
+					   "Saving attachments"sv,
+					   fmt::format("{} remaining... {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
+					   AttachmentsToSave,
+					   Remaining);
 	}
 	uint64_t ElapsedTimeMS = Timer.GetElapsedTimeMs();
 	if (AttachmentsToSave > 0)
 	{
 		ReportProgress(OptionalContext,
-					   fmt::format("Saving attachments, {} remaining. {}", 0, GetStats(RemoteStore.GetStats(), ElapsedTimeMS)),
+					   "Saving attachments"sv,
+					   fmt::format("{}", GetStats(RemoteStore.GetStats(), ElapsedTimeMS)),
 					   AttachmentsToSave,
 					   0);
 	}
@@ -2296,7 +2305,8 @@ WriteOplogSection(ProjectStore::Oplog& Oplog, const CbObjectView& SectionObject,
 		OpsData.clear();
 		OpDataOffsets.clear();
 		ReportProgress(OptionalContext,
-					   fmt::format("Writing oplog, {} remaining...", OpCount - OpsCompleteCount),
+					   "Writing oplog"sv,
+					   fmt::format("{} remaining...", OpCount - OpsCompleteCount),
 					   OpCount,
 					   OpCount - OpsCompleteCount);
 	};
@@ -2319,6 +2329,9 @@ WriteOplogSection(ProjectStore::Oplog& Oplog, const CbObjectView& SectionObject,
 	{
 		AppendBatch();
 	}
+
+	ReportProgress(OptionalContext, "Writing oplog"sv, ""sv, OpCount, 0);
+
 	return RemoteProjectStore::Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0};
 }
 
@@ -2808,11 +2821,11 @@ LoadOplog(CidStore&			   ChunkStore,
 		{
 			PartialTransferWallTimeMS += LoadAttachmentsTimer.GetElapsedTimeMs() - DownloadStartMS.load();
 		}
-		ReportProgress(
-			OptionalContext,
-			fmt::format("Loading attachments, {} remaining. {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
-			AttachmentCount.load(),
-			Remaining);
+		ReportProgress(OptionalContext,
+					   "Loading attachments"sv,
+					   fmt::format("{} remaining. {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)),
+					   AttachmentCount.load(),
+					   Remaining);
 	}
 	if (DownloadStartMS != (uint64_t)-1)
 	{
@@ -2822,7 +2835,8 @@ LoadOplog(CidStore&			   ChunkStore,
 	if (AttachmentCount.load() > 0)
 	{
 		ReportProgress(OptionalContext,
-					   fmt::format("Loading attachments, {} remaining. {}", 0, GetStats(RemoteStore.GetStats(), TransferWallTimeMS)),
+					   "Loading attachments"sv,
+					   fmt::format("{}", GetStats(RemoteStore.GetStats(), TransferWallTimeMS)),
 					   AttachmentCount.load(),
 					   0);
 	}
@@ -2838,12 +2852,16 @@ LoadOplog(CidStore&			   ChunkStore,
 				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", "");
 			}
 		}
-		ReportProgress(OptionalContext, fmt::format("Writing attachments, {} remaining.", Remaining), AttachmentCount.load(), Remaining);
+		ReportProgress(OptionalContext,
+					   "Writing attachments"sv,
+					   fmt::format("{} remaining.", Remaining),
+					   AttachmentCount.load(),
+					   Remaining);
 	}
 
 	if (AttachmentCount.load() > 0)
 	{
-		ReportProgress(OptionalContext, fmt::format("Writing attachments, {} remaining.", 0), AttachmentCount.load(), 0);
+		ReportProgress(OptionalContext, "Writing attachments", ""sv, AttachmentCount.load(), 0);
 	}
 
 	if (Result.ErrorCode == 0)
@@ -2959,11 +2977,12 @@ LoadOplog(CidStore&			   ChunkStore,
 					}
 				}
 				ReportProgress(OptionalContext,
-							   fmt::format("Dechunking attachments, {} remaining...", Remaining),
+							   "Dechunking attachments"sv,
+							   fmt::format("{} remaining...", Remaining),
 							   FilesToDechunk.size(),
 							   Remaining);
 			}
-			ReportProgress(OptionalContext, fmt::format("Dechunking attachments, {} remaining...", 0), FilesToDechunk.size(), 0);
+			ReportProgress(OptionalContext, "Dechunking attachments"sv, ""sv, FilesToDechunk.size(), 0);
 		}
 		Result = RemoteResult.ConvertResult();
 	}
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
index 7e4fbe83e..6e14b7632 100644
--- a/src/zenserver/vfs/vfsimpl.cpp
+++ b/src/zenserver/vfs/vfsimpl.cpp
@@ -362,7 +362,7 @@ VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNo
 
 			// Oplog contents enumeration
 
-			if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, false))
+			if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true))
 			{
 				Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId);