oplog level GC (#209)

Adds check for marker file supplied by UE to see if an oplog is expired (user has deleted the corresponding cooked folder).
Fixed concurrency vulnerabilities is project store related to oplogs.
* fix concurrency vulnerabilities
* propagate lifetime file path
* oplog level gc
* changelog

1 files changed, 26 insertions, 14 deletions

diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index 9d909de8c..8cebc5f6c 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -70,11 +70,18 @@ public:
 
 	struct Oplog
 	{
-		Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath);
+		Oplog(std::string_view			   Id,
+			  Project*					   Project,
+			  CidStore&					   Store,
+			  std::filesystem::path		   BasePath,
+			  const std::filesystem::path& MarkerPath);
 		~Oplog();
 
 		[[nodiscard]] static bool ExistsAt(std::filesystem::path BasePath);
 
+		void Read();
+		void Write();
+
 		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void IterateOplog(std::function<void(CbObject)>&& Fn);
 		std::optional<CbObject> GetOpByKey(const Oid& Key);
@@ -98,18 +105,6 @@ public:
 			kUpdateReplay
 		};
 
-		/** Update tracking metadata for a new oplog entry
-		 *
-		 * This is used during replay (and gets called as part of new op append)
-		 *
-		 * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected
-		 */
-		uint32_t RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate);
-
-		/** Scan oplog and register each entry, thus updating the in-memory tracking tables
-		 */
-		void ReplayLog();
-
 		const std::string& OplogId() const { return m_OplogId; }
 
 		const std::filesystem::path& TempPath() const { return m_TempPath; }
@@ -126,6 +121,7 @@ public:
 			return m_LatestOpMap.size();
 		}
 
+		bool				  IsExpired() const;
 		std::filesystem::path PrepareForDelete(bool MoveFolder);
 
 	private:
@@ -141,6 +137,7 @@ public:
 		Project*			  m_OuterProject = nullptr;
 		CidStore&			  m_CidStore;
 		std::filesystem::path m_BasePath;
+		std::filesystem::path m_MarkerPath;
 		std::filesystem::path m_TempPath;
 
 		mutable RwLock					 m_OplogLock;
@@ -154,6 +151,21 @@ public:
 		RefPtr<OplogStorage> m_Storage;
 		std::string			 m_OplogId;
 
+		/** Scan oplog and register each entry, thus updating the in-memory tracking tables
+		 */
+		void ReplayLog();
+
+		/** Update tracking metadata for a new oplog entry
+		 *
+		 * This is used during replay (and gets called as part of new op append)
+		 *
+		 * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected
+		 */
+		uint32_t RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock,
+									CbObject					Core,
+									const OplogEntry&			OpEntry,
+									UpdateType					TypeOfUpdate);
+
 		bool AddFileMapping(const RwLock::ExclusiveLockScope& OplogLock,
 							Oid								  FileId,
 							IoHash							  Hash,
@@ -171,7 +183,7 @@ public:
 		std::string			  ProjectRootDir;
 		std::string			  ProjectFilePath;
 
-		Oplog*					 NewOplog(std::string_view OplogId);
+		Oplog*					 NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath);
 		Oplog*					 OpenOplog(std::string_view OplogId);
 		void					 DeleteOplog(std::string_view OplogId);
 		void					 IterateOplogs(std::function<void(const Oplog&)>&& Fn) const;