Improved oplog import/export progress indicator at commandline (#206)

Nicer progress bar during oplog import/export
Verify that oplog has not been deleted from disk behind our back

1 files changed, 32 insertions, 15 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 0e6b2d3c9..25be159b9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1151,6 +1151,12 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath)
 	return std::filesystem::is_regular_file(StateFilePath);
 }
 
+bool
+ProjectStore::Oplog::Exists() const
+{
+	return ExistsAt(m_BasePath);
+}
+
 void
 ProjectStore::Oplog::Read()
 {
@@ -2925,22 +2931,33 @@ ProjectStore::Project::NewOplog(std::string_view OplogId, const std::filesystem:
 }
 
 ProjectStore::Oplog*
-ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact)
+ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bool VerifyPathOnDisk)
 {
 	ZEN_TRACE_CPU("Store::OpenOplog");
 
+	std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
 	{
-		RwLock::SharedLockScope _(m_ProjectLock);
+		RwLock::SharedLockScope ProjectLock(m_ProjectLock);
 
 		auto OplogIt = m_Oplogs.find(std::string(OplogId));
 
 		if (OplogIt != m_Oplogs.end())
 		{
-			return OplogIt->second.get();
+			if (!VerifyPathOnDisk || Oplog::ExistsAt(OplogBasePath))
+			{
+				return OplogIt->second.get();
+			}
+
+			// Somebody deleted the oplog on disk behind our back
+			ProjectLock.ReleaseNow();
+			std::filesystem::path DeletePath;
+			if (!RemoveOplog(OplogId, DeletePath))
+			{
+				ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+			}
 		}
 	}
 
-	std::filesystem::path	   OplogBasePath = BasePathForOplog(OplogId);
 	RwLock::ExclusiveLockScope Lock(m_ProjectLock);
 	if (auto It = m_Oplogs.find(std::string{OplogId}); It != m_Oplogs.end())
 	{
@@ -3114,7 +3131,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
 	std::vector<std::string> OpLogs = ScanForOplogs();
 	for (const std::string& OpLogId : OpLogs)
 	{
-		OpenOplog(OpLogId, /*AllowCompact*/ false);
+		OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
 	}
 	IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) {
 		if (!IsExpired(GcClock::TimePoint::min(), Ops))
@@ -3711,7 +3728,7 @@ ProjectStore::GetProjectFiles(const std::string_view				 ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -3843,7 +3860,7 @@ ProjectStore::GetProjectChunkInfos(const std::string_view				  ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -3962,7 +3979,7 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk info request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4042,7 +4059,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4152,7 +4169,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4199,7 +4216,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!FoundLog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Chunk put request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4241,7 +4258,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
 	if (!Oplog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4326,7 +4343,7 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!Oplog)
 	{
 		return {HttpResponseCode::NotFound, fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
@@ -4456,7 +4473,7 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 	}
 	Project->TouchProject();
 
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true);
+	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
 	if (!Oplog)
 	{
 		HttpReq.WriteResponse(HttpResponseCode::NotFound,
@@ -5685,7 +5702,7 @@ TEST_CASE("project.store.lifetimes")
 	std::filesystem::path DeletePath;
 	CHECK(Project->PrepareForDelete(DeletePath));
 	CHECK(!DeletePath.empty());
-	CHECK(Project->OpenOplog("oplog1", /*AllowCompact*/ false) == nullptr);
+	CHECK(Project->OpenOplog("oplog1", /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true) == nullptr);
 	// Oplog is now invalid, but pointer can still be accessed since we store old oplog pointers
 	CHECK(Oplog->OplogCount() == 0);
 	// Project is still valid since we have a Ref to it