New rotating file logger that keeps on running regardless of errors (#495)

* New rotating file logger that keeps on running regardless of errors

3 files changed, 307 insertions, 14 deletions

diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index 07e34305c..a8e8fa38e 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -12,7 +12,6 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <spdlog/sinks/basic_file_sink.h>
 #include <spdlog/sinks/daily_file_sink.h>
 #include <spdlog/sinks/msvc_sink.h>
-#include <spdlog/sinks/rotating_file_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
@@ -21,6 +20,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #include <zencore/fmtutils.h>
 #include <zencore/session.h>
 #include <zencore/string.h>
+#include <zenutil/basicfile.h>
 
 #include <chrono>
 #include <memory>
@@ -317,6 +317,265 @@ private:
 	std::string			 m_LogId;
 };
 
+// Basically the same functionality as spdlog::sinks::rotating_file_sink with the biggest difference
+// being that it just ignores any errors when writing/rotating files and keeps chugging on.
+// It will keep trying to log, and if it starts to work it will continue to log.
+class RotatingFileSink : public spdlog::sinks::sink
+{
+public:
+	RotatingFileSink(const std::filesystem::path& BaseFilename, std::size_t MaxSize, std::size_t MaxFiles, bool RotateOnOpen = false)
+	: m_BasePath(BaseFilename.parent_path())
+	, m_Stem(BaseFilename.stem().string())
+	, m_Extension(BaseFilename.extension().string())
+	, m_MaxSize(MaxSize)
+	, m_MaxFiles(MaxFiles)
+	{
+		std::filesystem::path RootFileName = GetFileName(0);
+		std::error_code		  Ec;
+		if (RotateOnOpen)
+		{
+			RwLock::ExclusiveLockScope RotateLock(m_Lock);
+			Rotate(RotateLock, Ec);
+		}
+		else
+		{
+			m_CurrentFile.Open(RootFileName, BasicFile::Mode::kWrite, Ec);
+			if (!Ec)
+			{
+				m_CurrentSize = m_CurrentFile.FileSize(Ec);
+			}
+			if (!Ec)
+			{
+				if (m_CurrentSize > m_MaxSize)
+				{
+					RwLock::ExclusiveLockScope RotateLock(m_Lock);
+					Rotate(RotateLock, Ec);
+				}
+			}
+		}
+
+		if (Ec)
+		{
+			throw std::system_error(Ec, fmt::format("Failed to open log file '{}'", RootFileName.string()));
+		}
+	}
+
+	virtual ~RotatingFileSink()
+	{
+		try
+		{
+			RwLock::ExclusiveLockScope RotateLock(m_Lock);
+			m_CurrentFile.Close();
+		}
+		catch (std::exception&)
+		{
+		}
+	}
+
+	RotatingFileSink(const RotatingFileSink&) = delete;
+	RotatingFileSink(RotatingFileSink&&)	  = delete;
+
+	RotatingFileSink& operator=(const RotatingFileSink&) = delete;
+	RotatingFileSink& operator=(RotatingFileSink&&) = delete;
+
+	virtual void log(const details::log_msg& msg) override
+	{
+		try
+		{
+			memory_buf_t Formatted;
+			if (TrySinkIt(msg, Formatted))
+			{
+				return;
+			}
+			while (true)
+			{
+				{
+					RwLock::ExclusiveLockScope RotateLock(m_Lock);
+					// Only rotate if no-one else has rotated before us
+					if (m_CurrentSize > m_MaxSize || !m_CurrentFile.IsOpen())
+					{
+						std::error_code Ec;
+						Rotate(RotateLock, Ec);
+						if (Ec)
+						{
+							return;
+						}
+					}
+				}
+				if (TrySinkIt(Formatted))
+				{
+					return;
+				}
+			}
+		}
+		catch (std::exception&)
+		{
+			// Silently eat errors
+		}
+	}
+	virtual void flush() override
+	{
+		try
+		{
+			RwLock::SharedLockScope Lock(m_Lock);
+			if (m_CurrentFile.IsOpen())
+			{
+				m_CurrentFile.Flush();
+			}
+		}
+		catch (std::exception&)
+		{
+			// Silently eat errors
+		}
+	}
+
+	virtual void set_pattern(const std::string& pattern) override
+	{
+		try
+		{
+			RwLock::ExclusiveLockScope _(m_Lock);
+			m_Formatter = details::make_unique<spdlog::pattern_formatter>(pattern);
+		}
+		catch (std::exception&)
+		{
+			// Silently eat errors
+		}
+	}
+	virtual void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override
+	{
+		try
+		{
+			RwLock::ExclusiveLockScope _(m_Lock);
+			m_Formatter = std::move(sink_formatter);
+		}
+		catch (std::exception&)
+		{
+			// Silently eat errors
+		}
+	}
+
+private:
+	static bool IsEmpty(const std::filesystem::path& Path, std::error_code& Ec)
+	{
+		bool Exists = std::filesystem::exists(Path, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		if (!Exists)
+		{
+			return true;
+		}
+		uintmax_t Size = std::filesystem::file_size(Path, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		return Size == 0;
+	}
+
+	void Rotate(RwLock::ExclusiveLockScope&, std::error_code& OutEc)
+	{
+		m_CurrentFile.Close();
+		bool BaseIsEmpty = IsEmpty(GetFileName(0), OutEc);
+		if (OutEc)
+		{
+			return;
+		}
+		if (!BaseIsEmpty)
+		{
+			// We try our best to rotate the logs, if we fail we fail and will try to open the base log file anyway
+			for (auto i = m_MaxFiles; i > 0; i--)
+			{
+				std::filesystem::path src = GetFileName(i - 1);
+				if (!std::filesystem::exists(src))
+				{
+					continue;
+				}
+				std::error_code		  DummyEc;
+				std::filesystem::path target = GetFileName(i);
+				if (std::filesystem::exists(target, DummyEc))
+				{
+					std::filesystem::remove(target, DummyEc);
+				}
+				std::filesystem::rename(src, target, DummyEc);
+			}
+		}
+		m_CurrentFile.Open(GetFileName(0), BasicFile::Mode::kWrite, OutEc);
+		if (OutEc)
+		{
+			return;
+		}
+		// If we fail to rotate, try extending the current log file
+		m_CurrentSize = m_CurrentFile.FileSize(OutEc);
+	}
+
+	bool TrySinkIt(const details::log_msg& msg, memory_buf_t& OutFormatted)
+	{
+		RwLock::SharedLockScope Lock(m_Lock);
+		if (!m_CurrentFile.IsOpen())
+		{
+			return false;
+		}
+		m_Formatter->format(msg, OutFormatted);
+		size_t add_size	 = OutFormatted.size();
+		size_t write_pos = m_CurrentSize.fetch_add(add_size);
+		if (write_pos + add_size > m_MaxSize)
+		{
+			return false;
+		}
+		std::error_code Ec;
+		m_CurrentFile.Write(OutFormatted.data(), OutFormatted.size(), write_pos, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		return true;
+	}
+
+	bool TrySinkIt(const memory_buf_t& Formatted)
+	{
+		RwLock::SharedLockScope Lock(m_Lock);
+		if (!m_CurrentFile.IsOpen())
+		{
+			return false;
+		}
+		size_t add_size	 = Formatted.size();
+		size_t write_pos = m_CurrentSize.fetch_add(add_size);
+		if (write_pos + add_size > m_MaxSize)
+		{
+			return false;
+		}
+
+		std::error_code Ec;
+		m_CurrentFile.Write(Formatted.data(), Formatted.size(), write_pos, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		return true;
+	}
+
+	std::filesystem::path GetFileName(size_t Index) const
+	{
+		if (Index == 0)
+		{
+			return m_BasePath / (m_Stem + m_Extension);
+		}
+		return m_BasePath / fmt::format("{}.{}{}", m_Stem, Index, m_Extension);
+	}
+
+	RwLock							   m_Lock;
+	const std::filesystem::path		   m_BasePath;
+	const std::string				   m_Stem;
+	const std::string				   m_Extension;
+	std::unique_ptr<spdlog::formatter> m_Formatter;
+	std::atomic_size_t				   m_CurrentSize;
+	const std::size_t				   m_MaxSize;
+	const std::size_t				   m_MaxFiles;
+	BasicFile						   m_CurrentFile;
+};
+
 }  // namespace zen::logging
 
 namespace zen {
@@ -364,10 +623,10 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
 																		   /* truncate */ false,
 																		   uint16_t(/* max files */ 14));
 #else
-	auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(GlobalOptions.AbsLogFile),
-																		   /* max size */ 128 * 1024 * 1024,
-																		   /* max files */ 16,
-																		   /* rotate on open */ true);
+	auto FileSink = std::make_shared<zen::logging::RotatingFileSink>(zen::PathToUtf8(GlobalOptions.AbsLogFile),
+																	 /* max size */ 128 * 1024 * 1024,
+																	 /* max files */ 16,
+																	 /* rotate on open */ true);
 #endif
 
 	std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); });
@@ -411,10 +670,10 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
 	std::filesystem::path HttpLogPath = GlobalOptions.DataDir / "logs" / "http.log";
 	// spdlog can't create directories that starts with `\\?\` so we make sure the folder exists before creating the logger instance
 	zen::CreateDirectories(HttpLogPath.parent_path());
-	auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(HttpLogPath),
-																		   /* max size */ 128 * 1024 * 1024,
-																		   /* max files */ 16,
-																		   /* rotate on open */ true);
+	auto HttpSink = std::make_shared<zen::logging::RotatingFileSink>(zen::PathToUtf8(HttpLogPath),
+																	 /* max size */ 128 * 1024 * 1024,
+																	 /* max files */ 16,
+																	 /* rotate on open */ true);
 
 	auto HttpLogger = std::make_shared<spdlog::logger>("http_requests", HttpSink);
 	RegisterLogger(HttpLogger);
@@ -422,10 +681,10 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
 	// Cache request logging
 	std::filesystem::path CacheLogPath = GlobalOptions.DataDir / "logs" / "z$.log";
 	zen::CreateDirectories(CacheLogPath.parent_path());
-	auto CacheSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(CacheLogPath),
-																			/* max size */ 128 * 1024 * 1024,
-																			/* max files */ 16,
-																			/* rotate on open */ false);
+	auto CacheSink = std::make_shared<zen::logging::RotatingFileSink>(zen::PathToUtf8(CacheLogPath),
+																	  /* max size */ 128 * 1024 * 1024,
+																	  /* max files */ 16,
+																	  /* rotate on open */ false);
 
 	auto CacheLogger = std::make_shared<spdlog::logger>("z$", CacheSink);
 	RegisterLogger(CacheLogger);
diff --git a/src/zenutil/basicfile.cpp b/src/zenutil/basicfile.cpp
index 259d622c4..1dce71e60 100644
--- a/src/zenutil/basicfile.cpp
+++ b/src/zenutil/basicfile.cpp
@@ -365,7 +365,39 @@ BasicFile::FileSize()
 	int Fd = int(uintptr_t(m_FileHandle));
 	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
 	struct stat Stat;
-	fstat(Fd, &Stat);
+	if (fstat(Fd, &Stat) == -1)
+	{
+		ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle)));
+	}
+	return uint64_t(Stat.st_size);
+#endif
+}
+
+uint64_t
+BasicFile::FileSize(std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	ULARGE_INTEGER liFileSize;
+	liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
+	if (liFileSize.LowPart == INVALID_FILE_SIZE)
+	{
+		int Error = zen::GetLastError();
+		if (Error)
+		{
+			Ec = MakeErrorCode(Error);
+			return 0;
+		}
+	}
+	return uint64_t(liFileSize.QuadPart);
+#else
+	int Fd = int(uintptr_t(m_FileHandle));
+	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+	struct stat Stat;
+	if (fstat(Fd, &Stat) == -1)
+	{
+		Ec = MakeErrorCodeFromLastError();
+		return 0;
+	}
 	return uint64_t(Stat.st_size);
 #endif
 }
diff --git a/src/zenutil/include/zenutil/basicfile.h b/src/zenutil/include/zenutil/basicfile.h
index eb97c885c..7797258e8 100644
--- a/src/zenutil/include/zenutil/basicfile.h
+++ b/src/zenutil/include/zenutil/basicfile.h
@@ -62,12 +62,14 @@ public:
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
 	void	 Flush();
 	[[nodiscard]] uint64_t FileSize();
+	[[nodiscard]] uint64_t FileSize(std::error_code& Ec);
 	void				   SetFileSize(uint64_t FileSize);
 	IoBuffer			   ReadAll();
 	void				   WriteAll(IoBuffer Data, std::error_code& Ec);
 	void*				   Detach();
 
 	inline void* Handle() { return m_FileHandle; }
+	bool		 IsOpen() const { return m_FileHandle != nullptr; }
 
 protected:
 	void* m_FileHandle = nullptr;  // This is either null or valid