fix gc infinite loop (#453)

* make sure we update last gc time even if gc fails
* If we can't check if an oplog/project markerfile exists, assume it is not expired

3 files changed, 26 insertions, 10 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 4402e4486..e000976ea 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1580,12 +1580,20 @@ bool
 ProjectStore::Project::IsExpired(const RwLock::SharedLockScope&,
 								 const std::string&			  EntryName,
 								 const std::filesystem::path& MarkerPath,
-								 const GcClock::TimePoint	  ExpireTime) const
+								 const GcClock::TimePoint	  ExpireTime)
 {
 	if (!MarkerPath.empty())
 	{
-		if (std::filesystem::exists(MarkerPath))
+		std::error_code Ec;
+		if (std::filesystem::exists(MarkerPath, Ec))
 		{
+			if (Ec)
+			{
+				ZEN_WARN("Failed to check expiry via marker file '{}', assuming {} is not expired",
+						 EntryName.empty() ? "project" : EntryName,
+						 MarkerPath.string());
+				return false;
+			}
 			return false;
 		}
 	}
@@ -1603,7 +1611,7 @@ ProjectStore::Project::IsExpired(const RwLock::SharedLockScope&,
 }
 
 bool
-ProjectStore::Project::IsExpired(const RwLock::SharedLockScope& ProjectLock, const GcClock::TimePoint ExpireTime) const
+ProjectStore::Project::IsExpired(const RwLock::SharedLockScope& ProjectLock, const GcClock::TimePoint ExpireTime)
 {
 	return IsExpired(ProjectLock, std::string(), ProjectFilePath, ExpireTime);
 }
@@ -1611,13 +1619,13 @@ ProjectStore::Project::IsExpired(const RwLock::SharedLockScope& ProjectLock, con
 bool
 ProjectStore::Project::IsExpired(const RwLock::SharedLockScope& ProjectLock,
 								 const GcClock::TimePoint		ExpireTime,
-								 const ProjectStore::Oplog&		Oplog) const
+								 const ProjectStore::Oplog&		Oplog)
 {
 	return IsExpired(ProjectLock, Oplog.OplogId(), Oplog.MarkerPath(), ExpireTime);
 }
 
 bool
-ProjectStore::Project::IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog) const
+ProjectStore::Project::IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog)
 {
 	RwLock::SharedLockScope Lock(m_ProjectLock);
 	return IsExpired(Lock, Oplog.OplogId(), Oplog.MarkerPath(), ExpireTime);
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 9c0f8790d..0d55e37e9 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -224,9 +224,9 @@ public:
 		void					 IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const;
 		void					 IterateOplogs(std::function<void(const RwLock::SharedLockScope&, Oplog&)>&& Fn);
 		std::vector<std::string> ScanForOplogs() const;
-		bool					 IsExpired(const RwLock::SharedLockScope&, const GcClock::TimePoint ExpireTime) const;
-		bool IsExpired(const RwLock::SharedLockScope&, const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog) const;
-		bool IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog) const;
+		bool					 IsExpired(const RwLock::SharedLockScope&, const GcClock::TimePoint ExpireTime);
+		bool IsExpired(const RwLock::SharedLockScope&, const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog);
+		bool IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog);
 		void TouchProject() const;
 		void TouchOplog(std::string_view Oplog) const;
 
@@ -256,7 +256,7 @@ public:
 		bool				  IsExpired(const RwLock::SharedLockScope&,
 										const std::string&			 EntryName,
 										const std::filesystem::path& MarkerPath,
-										const GcClock::TimePoint	 ExpireTime) const;
+										const GcClock::TimePoint	 ExpireTime);
 		void				  WriteAccessTimes();
 		void				  ReadAccessTimes();
 	};
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index d86bfd0fa..ef8945d91 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1046,15 +1046,23 @@ GcScheduler::SchedulerThread()
 			{
 				ZEN_ERROR("scheduling garbage collection failed with system error exception: '{}'", SystemError.what());
 			}
+			m_LastGcTime			= GcClock::Now();
+			m_LastLightweightGcTime = m_LastGcTime;
+			WaitTime				= m_Config.MonitorInterval;
 		}
 		catch (std::bad_alloc& BadAlloc)
 		{
 			ZEN_WARN("scheduling garbage collection ran out of memory: '{}'", BadAlloc.what());
+			m_LastGcTime			= GcClock::Now();
+			m_LastLightweightGcTime = m_LastGcTime;
+			WaitTime				= m_Config.MonitorInterval;
 		}
 		catch (std::exception& Ex)
 		{
 			ZEN_ERROR("scheduling garbage collection failed with: '{}'", Ex.what());
-			WaitTime = m_Config.MonitorInterval;
+			m_LastGcTime			= GcClock::Now();
+			m_LastLightweightGcTime = m_LastGcTime;
+			WaitTime				= m_Config.MonitorInterval;
 		}
 	}
 }