Make gc reserve a global resource

6 files changed, 180 insertions, 114 deletions

diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index b7fc18b4e..ac0f863cc 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -428,6 +428,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   "Max duration in seconds before Z$ entries get evicted.",
 					   cxxopts::value<int32_t>(ServerOptions.GcConfig.Cache.MaxDurationSeconds)->default_value("86400"),
 					   "");
+
+	options.add_option("gc",
+					   "",
+					   "disk-reserve-size",
+					   "Size of gc disk reserve in bytes.",
+					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"),
+					   "");
 	try
 	{
 		auto result			  = options.parse(argc, argv);
@@ -699,6 +706,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio
 		if (sol::optional<sol::table> GcConfig = lua["gc"])
 		{
 			ServerOptions.GcConfig.IntervalSeconds = GcConfig.value().get_or("intervalseconds", 0);
+			ServerOptions.GcConfig.DiskReserveSize = GcConfig.value().get_or("diskreservesize", uint64_t(1u << 28));
 
 			if (sol::optional<sol::table> CacheGcConfig = GcConfig.value()["cache"])
 			{
diff --git a/zenserver/config.h b/zenserver/config.h
index a61a7f89f..9f1b3645c 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -91,6 +91,7 @@ struct ZenGcConfig
 	int32_t				   IntervalSeconds	   = 0;
 	bool				   CollectSmallObjects = true;
 	bool				   Enabled			   = true;
+	uint64_t			   DiskReserveSize	   = 1ul << 28;
 };
 
 struct ZenServerOptions
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 667bcd317..f81deb167 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -369,6 +369,7 @@ public:
 			.MaxCacheDuration	 = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds),
 			.CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects,
 			.Enabled			 = ServerOptions.GcConfig.Enabled,
+			.DiskReserveSize	 = ServerOptions.GcConfig.DiskReserveSize,
 		};
 		m_GcScheduler.Initialize(GcConfig);
 
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 2a4f3c012..c424cde74 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -112,11 +112,6 @@ namespace {
 		return Path.ToPath();
 	}
 
-	std::filesystem::path GetGCReservePath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
-	{
-		return GetBasePath(RootPath, ContainerBaseName) / (ContainerBaseName + ".gc.reserve" + DataExtension);
-	}
-
 	std::filesystem::path GetLegacyLogPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
 	{
 		return RootPath / (ContainerBaseName + LogExtension);
@@ -352,7 +347,6 @@ namespace {
 
 		BasicFile BlockFile;
 		BlockFile.Open(LegacySobsPath, BasicFile::EMode::kRead);
-		uint64_t FileSize = BlockFile.FileSize();
 
 		std::unordered_map<IoHash, LegacyCasDiskIndexEntry, IoHash::Hasher> LegacyDiskIndex;
 
@@ -1223,8 +1217,8 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 				}
 				if (Space.Free < m_MaxBlockSize)
 				{
-					std::filesystem::path GCReservePath = GetGCReservePath(m_Config.RootDirectory, m_ContainerBaseName);
-					if (!std::filesystem::is_regular_file(GCReservePath))
+					uint64_t ReclaimedSpace = GcCtx.ClaimGCReserve();
+					if (ReclaimedSpace == 0)
 					{
 						ZEN_INFO("garbage collect for '{}' FAILED, required disk space {}, free {}",
 								 m_Config.RootDirectory / m_ContainerBaseName,
@@ -1244,14 +1238,8 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 					ZEN_INFO("using gc reserve for '{}', disk free {}",
 							 m_Config.RootDirectory / m_ContainerBaseName,
 							 NiceBytes(Space.Free));
-					std::filesystem::path NewBlockPath = GetBlockPath(m_BlocksBasePath, NextBlockIndex);
-					std::filesystem::rename(GCReservePath, NewBlockPath);
-					NewBlockFile->Open();
-				}
-				else
-				{
-					NewBlockFile->Create(m_MaxBlockSize);
 				}
+				NewBlockFile->Create(m_MaxBlockSize);
 				NewBlockIndex = NextBlockIndex;
 				WriteOffset	  = 0;
 			}
@@ -1305,34 +1293,6 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 	}
 
 	GcCtx.DeletedCas(DeletedChunks);
-
-	std::filesystem::path GCReservePath = GetGCReservePath(m_Config.RootDirectory, m_ContainerBaseName);
-	if (std::filesystem::is_regular_file(GCReservePath))
-	{
-		return;
-	}
-
-	std::error_code Error;
-	DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Error);
-	if (Error)
-	{
-		ZEN_ERROR("get disk space in {} FAILED, reason '{}'", m_ContainerBaseName, Error.message());
-		return;
-	}
-	if (Space.Free < m_MaxBlockSize)
-	{
-		ZEN_INFO("not enough space for garbage collect reserve for '{}' FAILED, required disk space {}, free {}",
-				 m_Config.RootDirectory / m_ContainerBaseName,
-				 m_MaxBlockSize,
-				 NiceBytes(Space.Free));
-		return;
-	}
-	BasicFile GCReserveFile;
-	CreateDirectories(GCReservePath.parent_path());
-	GCReserveFile.Open(GCReservePath, BasicFile::EMode::kTruncate);
-	GCReserveFile.SetFileSize(m_MaxBlockSize);
-
-	ZEN_DEBUG("recreated garbage collect reserve for '{}', {} bytes", m_Config.RootDirectory / m_ContainerBaseName, NiceBytes(Space.Free));
 }
 
 void
@@ -1608,59 +1568,6 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		CreateDirectories(m_BlocksBasePath);
 	}
 
-	// Create GC reserve file if possible
-	std::filesystem::path GCReservePath = GetGCReservePath(m_Config.RootDirectory, m_ContainerBaseName);
-
-	std::error_code Error;
-	DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Error);
-	if (Error)
-	{
-		ZEN_ERROR("get disk space in {} FAILED, reason '{}'", m_Config.RootDirectory, Error.message());
-		return;
-	}
-
-	BasicFile GCReserveFile;
-	if (std::filesystem::is_regular_file(GCReservePath))
-	{
-		GCReserveFile.Open(GCReservePath, BasicFile::EMode::kWrite);
-		std::uint64_t CurrentSize = GCReserveFile.FileSize();
-		if ((Space.Free - CurrentSize) >= m_MaxBlockSize)
-		{
-			GCReserveFile.SetFileSize(m_MaxBlockSize);
-		}
-		else
-		{
-			// We need it to be the proper size if we are to use it
-			ZEN_WARN("removing gc reserve for '{}', not enough space free on drive, need {}, have {} ",
-					 m_Config.RootDirectory / m_ContainerBaseName,
-					 NiceBytes(m_MaxBlockSize),
-					 NiceBytes(Space.Free));
-
-			std::error_code Ec;
-			std::filesystem::remove(GCReservePath, Ec);
-			if (Ec)
-			{
-				ZEN_WARN("Failed to delete gc reserve file '{}' reason: '{}'", GCReservePath, Ec.message());
-			}
-		}
-	}
-	else
-	{
-		if (Space.Free >= m_MaxBlockSize)
-		{
-			CreateDirectories(GCReservePath.parent_path());
-			GCReserveFile.Open(GCReservePath, BasicFile::EMode::kTruncate);
-			GCReserveFile.SetFileSize(m_MaxBlockSize);
-		}
-		else
-		{
-			ZEN_WARN("can't create gc reserve for '{}', not enough space free on drive, need {}, have {} ",
-					 m_Config.RootDirectory / m_ContainerBaseName,
-					 NiceBytes(m_MaxBlockSize),
-					 NiceBytes(Space.Free));
-		}
-	}
-
 	if (MakeSnapshot)
 	{
 		MakeIndexSnapshot();
@@ -2342,10 +2249,6 @@ TEST_CASE("compactcas.legacyconversion")
 	std::filesystem::path				 SLegacylogPath = GetLegacyLogPath(CasConfig.RootDirectory, "test");
 	LegacyCasLog.Open(SLegacylogPath, CasLogFile::EMode::kTruncate);
 
-	BasicFile UCasFile;
-	UCasFile.Open(LegacyCasPath, BasicFile::EMode::kRead);
-	uint64_t FileSize = UCasFile.FileSize();
-
 	for (const CasDiskIndexEntry& Entry : LogEntries)
 	{
 		BlockStoreLocation		Location = Entry.Location.Get(16);
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index dee8c209f..ab7eece20 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -5,6 +5,7 @@
 #include <zencore/compactbinary.h>
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinaryvalidation.h>
+#include <zencore/except.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -19,6 +20,15 @@
 #include <fmt/format.h>
 #include <filesystem>
 
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#else
+#	include <fcntl.h>
+#	include <sys/file.h>
+#	include <sys/stat.h>
+#	include <unistd.h>
+#endif
+
 #if ZEN_WITH_TESTS
 #	include <zencore/compress.h>
 #	include <algorithm>
@@ -32,6 +42,102 @@ namespace fs = std::filesystem;
 
 //////////////////////////////////////////////////////////////////////////
 
+namespace {
+	std::error_code CreateGCReserve(const std::filesystem::path& Path, uint64_t Size)
+	{
+		if (Size == 0)
+		{
+			std::filesystem::remove(Path);
+			return std::error_code{};
+		}
+		CreateDirectories(Path.parent_path());
+		if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size)
+		{
+			return std::error_code();
+		}
+#if ZEN_PLATFORM_WINDOWS
+		DWORD dwCreationDisposition = CREATE_ALWAYS;
+		DWORD dwDesiredAccess		= GENERIC_READ | GENERIC_WRITE;
+
+		const DWORD dwShareMode			 = 0;
+		const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+		HANDLE		hTemplateFile		 = nullptr;
+
+		HANDLE FileHandle = CreateFile(Path.c_str(),
+									   dwDesiredAccess,
+									   dwShareMode,
+									   /* lpSecurityAttributes */ nullptr,
+									   dwCreationDisposition,
+									   dwFlagsAndAttributes,
+									   hTemplateFile);
+
+		if (FileHandle == INVALID_HANDLE_VALUE)
+		{
+			return zen::MakeErrorCodeFromLastError();
+		}
+		bool		  Keep = true;
+		auto		  _	   = MakeGuard([FileHandle, &Keep, Path]() {
+			::CloseHandle(FileHandle);
+			if (!Keep)
+			{
+				::DeleteFile(Path.c_str());
+			}
+		});
+		LARGE_INTEGER liFileSize;
+		liFileSize.QuadPart = Size;
+		BOOL OK				= ::SetFilePointerEx(FileHandle, liFileSize, 0, FILE_BEGIN);
+		if (!OK)
+		{
+			return zen::MakeErrorCodeFromLastError();
+		}
+		OK = ::SetEndOfFile(FileHandle);
+		if (!OK)
+		{
+			return zen::MakeErrorCodeFromLastError();
+		}
+		Keep = true;
+#else
+		int OpenFlags = O_CLOEXEC | O_RDWR | O_CREAT;
+		int Fd		  = open(Path.c_str(), OpenFlags, 0666);
+		if (Fd < 0)
+		{
+			return zen::MakeErrorCodeFromLastError();
+		}
+
+		bool Keep = true;
+		auto _	  = MakeGuard([Fd, &Keep, Path]() {
+			   close(Fd);
+			   if (!Keep)
+			   {
+				   unlink(Path.c_str());
+			   }
+		   });
+
+		if (fchmod(Fd, 0666) < 0)
+		{
+			return zen::MakeErrorCodeFromLastError();
+		}
+
+#	if ZEN_PLATFORM_MAC
+		if (ftruncate(Fd, (off_t)FileSize) < 0)
+		{
+			return zen::MakeErrorCodeFromLastError();
+		}
+#	else
+		if (ftruncate64(Fd, (off64_t)FileSize) < 0)
+		{
+			return zen::MakeErrorCodeFromLastError();
+		}
+#	endif
+		Keep = true;
+#endif
+		return std::error_code{};
+	}
+
+}  // namespace
+
+//////////////////////////////////////////////////////////////////////////
+
 CbObject
 LoadCompactBinaryObject(const fs::path& Path)
 {
@@ -75,6 +181,8 @@ struct GcContext::GcState
 	GcClock::Duration  m_MaxCacheDuration	 = std::chrono::hours(24);
 	bool			   m_DeletionMode		 = true;
 	bool			   m_CollectSmallObjects = false;
+
+	std::filesystem::path DiskReservePath;
 };
 
 GcContext::GcContext(GcClock::TimePoint Time) : m_State(std::make_unique<GcState>())
@@ -195,6 +303,27 @@ GcContext::MaxCacheDuration(GcClock::Duration Duration)
 	m_State->m_MaxCacheDuration = Duration;
 }
 
+void
+GcContext::DiskReservePath(const std::filesystem::path& Path)
+{
+	m_State->DiskReservePath = Path;
+}
+
+uint64_t
+GcContext::ClaimGCReserve()
+{
+	if (!std::filesystem::is_regular_file(m_State->DiskReservePath))
+	{
+		return 0;
+	}
+	uint64_t ReclaimedSize = std::filesystem::file_size(m_State->DiskReservePath);
+	if (std::filesystem::remove(m_State->DiskReservePath))
+	{
+		return ReclaimedSize;
+	}
+	return 0;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 GcContributor::GcContributor(CasGc& Gc) : m_Gc(Gc)
@@ -390,6 +519,15 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 
 	std::filesystem::create_directories(Config.RootDirectory);
 
+	std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
+	if (Ec)
+	{
+		ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'",
+				 m_Config.RootDirectory / "reserve.gc",
+				 NiceBytes(m_Config.DiskReserveSize),
+				 Ec.message());
+	}
+
 	m_LastGcTime = GcClock::Now();
 
 	if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state"))
@@ -517,6 +655,7 @@ GcScheduler::SchedulerThread()
 		GcCtx.SetDeletionMode(true);
 		GcCtx.CollectSmallObjects(m_Config.CollectSmallObjects);
 		GcCtx.MaxCacheDuration(m_Config.MaxCacheDuration);
+		GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc");
 
 		if (m_TriggerParams)
 		{
@@ -530,27 +669,37 @@ GcScheduler::SchedulerThread()
 			}
 		}
 
-		Stopwatch Timer;
-
 		ZEN_INFO("garbage collection STARTING, small objects gc {}, max cache duration {}",
 				 GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
 				 NiceTimeSpanMs(uint64_t(std::chrono::duration_cast<std::chrono::milliseconds>(GcCtx.MaxCacheDuration()).count())));
+		{
+			Stopwatch  Timer;
+			const auto __ =
+				MakeGuard([this, &Timer] { ZEN_INFO("garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
 
-		m_CasGc.CollectGarbage(GcCtx);
+			m_CasGc.CollectGarbage(GcCtx);
 
-		m_LastGcTime = GcClock::Now();
-		m_NextGcTime = NextGcTime(m_LastGcTime);
-		WaitTime	 = m_Config.MonitorInterval;
+			m_LastGcTime = GcClock::Now();
+			m_NextGcTime = NextGcTime(m_LastGcTime);
+			WaitTime	 = m_Config.MonitorInterval;
 
-		{
-			const fs::path Path = m_Config.RootDirectory / "gc_state";
-			ZEN_DEBUG("saving scheduler state to '{}'", Path);
-			CbObjectWriter SchedulderState;
-			SchedulderState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
-			SaveCompactBinaryObject(Path, SchedulderState.Save());
-		}
+			{
+				const fs::path Path = m_Config.RootDirectory / "gc_state";
+				ZEN_DEBUG("saving scheduler state to '{}'", Path);
+				CbObjectWriter SchedulderState;
+				SchedulderState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
+				SaveCompactBinaryObject(Path, SchedulderState.Save());
+			}
 
-		ZEN_INFO("garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+			std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
+			if (Ec)
+			{
+				ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'",
+						 m_Config.RootDirectory / "reserve.gc",
+						 NiceBytes(m_Config.DiskReserveSize),
+						 Ec.message());
+			}
+		}
 
 		uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
 		if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
diff --git a/zenstore/include/zenstore/gc.h b/zenstore/include/zenstore/gc.h
index b8ba338f0..bc8dee9a3 100644
--- a/zenstore/include/zenstore/gc.h
+++ b/zenstore/include/zenstore/gc.h
@@ -78,6 +78,9 @@ public:
 	GcClock::Duration MaxCacheDuration() const;
 	void			  MaxCacheDuration(GcClock::Duration Duration);
 
+	void	 DiskReservePath(const std::filesystem::path& Path);
+	uint64_t ClaimGCReserve();
+
 	inline bool Expired(GcClock::Tick TickCount) { return Time() - GcClock::TimePointFromTick(TickCount) > MaxCacheDuration(); }
 
 private:
@@ -170,6 +173,7 @@ struct GcSchedulerConfig
 	std::chrono::seconds  MaxCacheDuration{86400};
 	bool				  CollectSmallObjects = true;
 	bool				  Enabled			  = true;
+	uint64_t			  DiskReserveSize	  = 1ul << 28;
 };
 
 /**