Add `--gc-projectstore-duration-seconds` option (#281)

* Add `--gc-projectstore-duration-seconds` option
* Cleanup lua gc options parsing
* Remove dead configuration values
* changelog

6 files changed, 57 insertions, 52 deletions

diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 3a93c9aec..c37622cb6 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -54,6 +54,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler) : m_GcScheduler(Sched
 				}
 			}
 
+			if (auto Param = Params.GetValue("maxprojectstoreduration"); Param.empty() == false)
+			{
+				if (auto Value = ParseInt<uint64_t>(Param))
+				{
+					GcParams.MaxProjectStoreDuration = std::chrono::seconds(Value.value());
+				}
+			}
+
 			if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false)
 			{
 				if (auto Value = ParseInt<uint64_t>(Param))
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index d56b3cfe2..3a6e5cbc3 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -1006,7 +1006,7 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 		{
 			LogEntryCount = ReadLog(LogPath, m_LogFlushPosition);
 		}
-		else
+		else if (fs::is_regular_file(LogPath))
 		{
 			ZEN_WARN("removing invalid cas log at '{}'", LogPath);
 			std::filesystem::remove(LogPath);
@@ -1512,7 +1512,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 				  NiceLatencyNs(ReadBlockLongestTimeUs));
 	});
 
-	const GcClock::TimePoint ExpireTime = GcCtx.ExpireTime();
+	const GcClock::TimePoint ExpireTime = GcCtx.CacheExpireTime();
 
 	const GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
 
@@ -2976,7 +2976,7 @@ TEST_CASE("z$.gc")
 										 GcClock::Duration		 MaxDuration,
 										 std::span<const IoHash> Cids,
 										 std::vector<IoHash>&	 OutKeep) {
-			GcContext GcCtx(Time - MaxDuration);
+			GcContext GcCtx(Time - MaxDuration, Time - MaxDuration);
 			Gc.CollectGarbage(GcCtx);
 			OutKeep.clear();
 			GcCtx.FilterCids(Cids, [&OutKeep](const IoHash& Hash) { OutKeep.push_back(Hash); });
@@ -3055,7 +3055,7 @@ TEST_CASE("z$.gc")
 		}
 
 		{
-			GcContext GcCtx(CurrentTime - std::chrono::hours(46));
+			GcContext GcCtx(CurrentTime - std::chrono::hours(46), CurrentTime - std::chrono::hours(46));
 
 			Gc.CollectGarbage(GcCtx);
 
@@ -3069,7 +3069,7 @@ TEST_CASE("z$.gc")
 
 		// Move forward in time and collect again
 		{
-			GcContext GcCtx(CurrentTime + std::chrono::minutes(2));
+			GcContext GcCtx(CurrentTime + std::chrono::minutes(2), CurrentTime + std::chrono::minutes(2));
 			Gc.CollectGarbage(GcCtx);
 
 			for (const auto& Key : Keys)
@@ -3099,7 +3099,7 @@ TEST_CASE("z$.gc")
 		}
 
 		{
-			GcContext GcCtx(GcClock::Now() - std::chrono::hours(2));
+			GcContext GcCtx(GcClock::Now() - std::chrono::hours(2), GcClock::Now() - std::chrono::hours(2));
 			GcCtx.CollectSmallObjects(true);
 
 			Gc.CollectGarbage(GcCtx);
@@ -3114,7 +3114,7 @@ TEST_CASE("z$.gc")
 
 		// Move forward in time and collect again
 		{
-			GcContext GcCtx(GcClock::Now() + std::chrono::minutes(2));
+			GcContext GcCtx(GcClock::Now() + std::chrono::minutes(2), GcClock::Now() + std::chrono::minutes(2));
 			GcCtx.CollectSmallObjects(true);
 
 			Zcs.Flush();
@@ -3303,7 +3303,7 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 					C++;
 				}
 
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 				GcCtx.CollectSmallObjects(true);
 				GcCtx.AddRetainedCids(KeepHashes);
 				Zcs.CollectGarbage(GcCtx);
@@ -3351,7 +3351,7 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 					C++;
 				}
 
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 				GcCtx.CollectSmallObjects(true);
 				GcCtx.AddRetainedCids(KeepHashes);
 				Zcs.CollectGarbage(GcCtx);
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 5b635b89d..0e7aac079 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -506,6 +506,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 	options.add_option("gc",
 					   "",
+					   "gc-projectstore-duration-seconds",
+					   "Max duration in seconds before project store entries get evicted. Default set to 1209600 (2 weeks)",
+					   cxxopts::value<int32_t>(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"),
+					   "");
+
+	options.add_option("gc",
+					   "",
 					   "disk-reserve-size",
 					   "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.",
 					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"),
@@ -522,14 +529,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   "",
 					   "gc-low-diskspace-threshold",
 					   "Minimum free space on disk to allow writes to disk. Default set to 268435456 (256 Mb). Set to zero to disable.",
-					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.Cache.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"),
+					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"),
 					   "");
 
 	options.add_option("gc",
 					   "",
 					   "gc-disksize-softlimit",
 					   "Garbage collection disk usage soft limit. Default set to 0 (Off).",
-					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.Cache.DiskSizeSoftLimit)->default_value("0"),
+					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"),
 					   "");
 
 	options.add_option("objectstore",
@@ -878,21 +885,16 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio
 			ServerOptions.GcConfig.MonitorIntervalSeconds			 = GcConfig.value().get_or("monitorintervalseconds", 30);
 			ServerOptions.GcConfig.IntervalSeconds					 = GcConfig.value().get_or("intervalseconds", 0);
 			ServerOptions.GcConfig.DiskReserveSize					 = GcConfig.value().get_or("diskreservesize", uint64_t(1u << 28));
+			ServerOptions.GcConfig.DiskSizeSoftLimit				 = GcConfig.value().get_or("disksizesoftlimit", 0);
 			ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites = GcConfig.value().get_or("lowdiskspacethreshold", 0);
 
 			if (sol::optional<sol::table> CacheGcConfig = GcConfig.value()["cache"])
 			{
 				ServerOptions.GcConfig.Cache.MaxDurationSeconds = CacheGcConfig.value().get_or("maxdurationseconds", int32_t(0));
-				ServerOptions.GcConfig.Cache.DiskSizeLimit		= CacheGcConfig.value().get_or("disksizelimit", ~uint64_t(0));
-				ServerOptions.GcConfig.Cache.MemorySizeLimit	= CacheGcConfig.value().get_or("memorysizelimit", ~uint64_t(0));
-				ServerOptions.GcConfig.Cache.DiskSizeSoftLimit	= CacheGcConfig.value().get_or("disksizesoftlimit", 0);
 			}
-
-			if (sol::optional<sol::table> CasGcConfig = GcConfig.value()["cas"])
+			if (sol::optional<sol::table> CacheGcConfig = GcConfig.value()["projectstore"])
 			{
-				ServerOptions.GcConfig.Cas.LargeStrategySizeLimit = CasGcConfig.value().get_or("largestrategysizelimit", ~uint64_t(0));
-				ServerOptions.GcConfig.Cas.SmallStrategySizeLimit = CasGcConfig.value().get_or("smallstrategysizelimit", ~uint64_t(0));
-				ServerOptions.GcConfig.Cas.TinyStrategySizeLimit  = CasGcConfig.value().get_or("tinystrategysizelimit", ~uint64_t(0));
+				ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds = CacheGcConfig.value().get_or("maxdurationseconds", int32_t(0));
 			}
 		}
 
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index caf4adce7..4f63e2644 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -69,32 +69,26 @@ struct ZenUpstreamCacheConfig
 
 struct ZenCacheEvictionPolicy
 {
-	uint64_t DiskSizeLimit					   = ~uint64_t(0);
-	uint64_t MemorySizeLimit				   = 1024 * 1024 * 1024;
-	int32_t	 MaxDurationSeconds				   = 24 * 60 * 60;
-	uint64_t DiskSizeSoftLimit				   = 0;
-	uint64_t MinimumFreeDiskSpaceToAllowWrites = 256u * 1024u * 1024u;
-	bool	 Enabled						   = true;
+	int32_t MaxDurationSeconds = 24 * 60 * 60;
 };
 
-struct ZenCasEvictionPolicy
+struct ZenProjectStoreEvictionPolicy
 {
-	uint64_t LargeStrategySizeLimit = ~uint64_t(0);
-	uint64_t SmallStrategySizeLimit = ~uint64_t(0);
-	uint64_t TinyStrategySizeLimit	= ~uint64_t(0);
-	bool	 Enabled				= true;
+	int32_t MaxDurationSeconds = 7 * 24 * 60 * 60;
 };
 
 struct ZenGcConfig
 {
-	ZenCasEvictionPolicy   Cas;
-	ZenCacheEvictionPolicy Cache;
-	int32_t				   MonitorIntervalSeconds			 = 30;
-	int32_t				   IntervalSeconds					 = 0;
-	bool				   CollectSmallObjects				 = true;
-	bool				   Enabled							 = true;
-	uint64_t			   DiskReserveSize					 = 1ul << 28;
-	uint64_t			   MinimumFreeDiskSpaceToAllowWrites = 1ul << 28;
+	//	ZenCasEvictionPolicy   Cas;
+	ZenCacheEvictionPolicy		  Cache;
+	ZenProjectStoreEvictionPolicy ProjectStore;
+	int32_t						  MonitorIntervalSeconds			= 30;
+	int32_t						  IntervalSeconds					= 0;
+	bool						  CollectSmallObjects				= true;
+	bool						  Enabled							= true;
+	uint64_t					  DiskReserveSize					= 1ul << 28;
+	uint64_t					  DiskSizeSoftLimit					= 0;
+	uint64_t					  MinimumFreeDiskSpaceToAllowWrites = 1ul << 28;
 };
 
 struct ZenOpenIdProviderConfig
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 03c1e8927..0beb207ae 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1581,7 +1581,7 @@ ProjectStore::Project::GatherReferences(GcContext& GcCtx)
 	}
 
 	IterateOplogs([&](Oplog& Ops) {
-		if (!IsExpired(GcCtx.ExpireTime(), Ops))
+		if (!IsExpired(GcCtx.ProjectStoreExpireTime(), Ops))
 		{
 			Ops.GatherReferences(GcCtx);
 		}
@@ -1804,7 +1804,7 @@ ProjectStore::GatherReferences(GcContext& GcCtx)
 
 		for (auto& Kv : m_Projects)
 		{
-			if (Kv.second->IsExpired(GcCtx.ExpireTime()))
+			if (Kv.second->IsExpired(GcCtx.ProjectStoreExpireTime()))
 			{
 				ExpiredProjectCount++;
 				continue;
@@ -1842,7 +1842,7 @@ ProjectStore::CollectGarbage(GcContext& GcCtx)
 		RwLock::SharedLockScope _(m_ProjectsLock);
 		for (auto& Kv : m_Projects)
 		{
-			if (Kv.second->IsExpired(GcCtx.ExpireTime()))
+			if (Kv.second->IsExpired(GcCtx.ProjectStoreExpireTime()))
 			{
 				ExpiredProjects.push_back(Kv.second);
 				ExpiredProjectCount++;
@@ -1865,7 +1865,7 @@ ProjectStore::CollectGarbage(GcContext& GcCtx)
 		{
 			RwLock::ExclusiveLockScope _(m_ProjectsLock);
 			Project->IterateOplogs([&GcCtx, &Project, &ExpiredOplogs](ProjectStore::Oplog& Oplog) {
-				if (Project->IsExpired(GcCtx.ExpireTime(), Oplog))
+				if (Project->IsExpired(GcCtx.ProjectStoreExpireTime(), Oplog))
 				{
 					ExpiredOplogs.push_back(Oplog.OplogId());
 				}
@@ -1893,7 +1893,7 @@ ProjectStore::CollectGarbage(GcContext& GcCtx)
 		std::string			  ProjectId;
 		{
 			RwLock::ExclusiveLockScope _(m_ProjectsLock);
-			if (!Project->IsExpired(GcCtx.ExpireTime()))
+			if (!Project->IsExpired(GcCtx.ProjectStoreExpireTime()))
 			{
 				ZEN_DEBUG("ProjectStore::CollectGarbage skipped garbage collect of project '{}'. Project no longer expired.", ProjectId);
 				continue;
@@ -4173,7 +4173,7 @@ TEST_CASE("project.store.gc")
 	}
 
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 		ProjectStore.GatherReferences(GcCtx);
 		size_t RefCount = 0;
 		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
@@ -4184,7 +4184,7 @@ TEST_CASE("project.store.gc")
 	}
 
 	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
 		ProjectStore.GatherReferences(GcCtx);
 		size_t RefCount = 0;
 		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
@@ -4197,7 +4197,7 @@ TEST_CASE("project.store.gc")
 	std::filesystem::remove(Project1FilePath);
 
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 		ProjectStore.GatherReferences(GcCtx);
 		size_t RefCount = 0;
 		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
@@ -4208,7 +4208,7 @@ TEST_CASE("project.store.gc")
 	}
 
 	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
 		ProjectStore.GatherReferences(GcCtx);
 		size_t RefCount = 0;
 		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
@@ -4220,7 +4220,7 @@ TEST_CASE("project.store.gc")
 
 	std::filesystem::remove(Project2OplogPath);
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 		ProjectStore.GatherReferences(GcCtx);
 		size_t RefCount = 0;
 		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
@@ -4231,7 +4231,7 @@ TEST_CASE("project.store.gc")
 	}
 
 	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
 		ProjectStore.GatherReferences(GcCtx);
 		size_t RefCount = 0;
 		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
@@ -4243,7 +4243,7 @@ TEST_CASE("project.store.gc")
 
 	std::filesystem::remove(Project2FilePath);
 	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
 		ProjectStore.GatherReferences(GcCtx);
 		size_t RefCount = 0;
 		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index b97cbbc26..789b6f4a6 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -409,11 +409,12 @@ public:
 			.MonitorInterval				   = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds),
 			.Interval						   = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds),
 			.MaxCacheDuration				   = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds),
+			.MaxProjectStoreDuration		   = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds),
 			.CollectSmallObjects			   = ServerOptions.GcConfig.CollectSmallObjects,
 			.Enabled						   = ServerOptions.GcConfig.Enabled,
 			.DiskReserveSize				   = ServerOptions.GcConfig.DiskReserveSize,
-			.DiskSizeSoftLimit				   = ServerOptions.GcConfig.Cache.DiskSizeSoftLimit,
-			.MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.Cache.MinimumFreeDiskSpaceToAllowWrites};
+			.DiskSizeSoftLimit				   = ServerOptions.GcConfig.DiskSizeSoftLimit,
+			.MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites};
 		m_GcScheduler.Initialize(GcConfig);
 
 		return EffectiveBasePort;