Merge pull request #72 from EpicGames/de/set-ulimit

attempt to change the maximum number of files open at startup

5 files changed, 66 insertions, 13 deletions

diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index e2778089b..7ff0dc45c 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -23,6 +23,7 @@
 #if ZEN_PLATFORM_LINUX
 #	include <dirent.h>
 #	include <fcntl.h>
+#	include <sys/resource.h>
 #	include <sys/stat.h>
 #	include <unistd.h>
 #endif
@@ -31,6 +32,7 @@
 #	include <dirent.h>
 #	include <fcntl.h>
 #	include <libproc.h>
+#	include <sys/resource.h>
 #	include <sys/stat.h>
 #	include <sys/syslimits.h>
 #	include <unistd.h>
@@ -986,6 +988,31 @@ GetRunningExecutablePath()
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
+void
+MaximizeOpenFileCount()
+{
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	struct rlimit Limit;
+	int Error = getrlimit(RLIMIT_NOFILE, &Limit);
+	if (Error)
+	{
+		ZEN_WARN("failed getting rlimit RLIMIT_NOFILE, reason '{}'", zen::MakeErrorCode(Error).message());
+	}
+	else
+	{
+		struct rlimit NewLimit = Limit;
+		NewLimit.rlim_cur = NewLimit.rlim_max;
+		ZEN_INFO("changing RLIMIT_NOFILE from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}", Limit.rlim_cur, Limit.rlim_max, NewLimit.rlim_cur, NewLimit.rlim_max);
+
+		Error = setrlimit(RLIMIT_NOFILE, &NewLimit);
+		if (Error != 0)
+		{
+			ZEN_WARN("failed to set RLIMIT_NOFILE limits from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}, reason '{}'", Limit.rlim_cur, Limit.rlim_max, NewLimit.rlim_cur, NewLimit.rlim_max, zen::MakeErrorCode(Error).message());
+		}
+	}
+#endif
+}
+
 //////////////////////////////////////////////////////////////////////////
 //
 // Testing related code follows...
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index d1a5f3e0c..a6e76eaa0 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -34,6 +34,10 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle);
 
 ZENCORE_API std::filesystem::path GetRunningExecutablePath();
 
+/** Set the max open file handle count to max allowed for the current process on Linux and MacOS
+ */
+ZENCORE_API void MaximizeOpenFileCount();
+
 struct FileContents
 {
 	std::vector<IoBuffer> Data;
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index f81deb167..3ac5b9992 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -1018,6 +1018,8 @@ ZenEntryPoint::Run()
 
 		InitializeLogging(ServerOptions);
 
+		MaximizeOpenFileCount();
+
 		ZEN_INFO(ZEN_APP_NAME " - using lock file at '{}'", LockFilePath);
 
 		ZEN_INFO(ZEN_APP_NAME " - starting on port {}, version '{}'", ServerOptions.BasePort, ZEN_CFG_VERSION_BUILD_STRING_FULL);
@@ -1156,9 +1158,10 @@ test_main(int argc, char** argv)
 	zen::z$_forcelink();
 
 	zen::logging::InitializeLogging();
-
 	spdlog::set_level(spdlog::level::debug);
 
+	zen::MaximizeOpenFileCount();
+
 	return doctest::Context(argc, argv).run();
 }
 #endif
diff --git a/zenstore-test/zenstore-test.cpp b/zenstore-test/zenstore-test.cpp
index e6bd92ab9..030b1159d 100644
--- a/zenstore-test/zenstore-test.cpp
+++ b/zenstore-test/zenstore-test.cpp
@@ -1,9 +1,16 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #include <zencore/logging.h>
+#include <zencore/filesystem.h>
 #include <zencore/zencore.h>
 #include <zenstore/zenstore.h>
 
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+#	include <sys/time.h>
+#	include <sys/resource.h>
+#	include <zencore/except.h>
+#endif
+
 #if ZEN_WITH_TESTS
 #	define DOCTEST_CONFIG_IMPLEMENT
 #	include <zencore/testing.h>
@@ -17,6 +24,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 	zen::zenstore_forcelinktests();
 
 	zen::logging::InitializeLogging();
+	zen::MaximizeOpenFileCount();
 
 	return doctest::Context(argc, argv).run();
 #else
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index f89108dc4..920ed965f 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -2272,8 +2272,9 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 
 		CreateDirectories(CasConfig.RootDirectory);
 
-		const uint64_t kChunkSize  = 1048;
-		const int32_t  kChunkCount = 4096;
+		const uint64_t kChunkSize	= 1048;
+		const int32_t  kChunkCount	= 4096;
+		uint64_t	   ExpectedSize = 0;
 
 		std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> Chunks;
 		Chunks.reserve(kChunkCount);
@@ -2289,10 +2290,12 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 					continue;
 				}
 				Chunks[Hash] = Chunk;
+				ExpectedSize += Chunk.Size();
 				break;
 			}
 		}
 
+		std::atomic<size_t>	 WorkCompleted = 0;
 		WorkerThreadPool	 ThreadPool(4);
 		CasGc				 Gc;
 		CasContainerStrategy Cas(CasConfig, Gc);
@@ -2302,31 +2305,34 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 			{
 				const IoHash&	Hash   = Chunk.first;
 				const IoBuffer& Buffer = Chunk.second;
-				ThreadPool.ScheduleWork([&Cas, Buffer, Hash]() {
+				ThreadPool.ScheduleWork([&Cas, &WorkCompleted, Buffer, Hash]() {
 					CasStore::InsertResult InsertResult = Cas.InsertChunk(Buffer, Hash);
 					ZEN_ASSERT(InsertResult.New);
+					WorkCompleted.fetch_add(1);
 				});
 			}
-			while (ThreadPool.PendingWork() > 0)
+			while (WorkCompleted < Chunks.size())
 			{
 				Sleep(1);
 			}
 		}
 
+		WorkCompleted			 = 0;
 		const uint64_t TotalSize = Cas.StorageSize().DiskSize;
-		CHECK_EQ(kChunkSize * kChunkCount, TotalSize);
+		CHECK_EQ(ExpectedSize, TotalSize);
 
 		{
 			for (const auto& Chunk : Chunks)
 			{
-				ThreadPool.ScheduleWork([&Cas, &Chunk]() {
+				ThreadPool.ScheduleWork([&Cas, &WorkCompleted, &Chunk]() {
 					IoHash	 ChunkHash = Chunk.first;
 					IoBuffer Buffer	   = Cas.FindChunk(ChunkHash);
 					IoHash	 Hash	   = IoHash::HashBuffer(Buffer);
 					CHECK(ChunkHash == Hash);
+					WorkCompleted.fetch_add(1);
 				});
 			}
-			while (ThreadPool.PendingWork() > 0)
+			while (WorkCompleted < Chunks.size())
 			{
 				Sleep(1);
 			}
@@ -2339,6 +2345,7 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 			GcChunkHashes.insert(Chunk.first);
 		}
 		{
+			WorkCompleted = 0;
 			std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> NewChunks;
 			NewChunks.reserve(kChunkCount);
 
@@ -2353,20 +2360,22 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 
 			for (const auto& Chunk : NewChunks)
 			{
-				ThreadPool.ScheduleWork([&Cas, Chunk, &AddedChunkCount]() {
+				ThreadPool.ScheduleWork([&Cas, &WorkCompleted, Chunk, &AddedChunkCount]() {
 					Cas.InsertChunk(Chunk.second, Chunk.first);
 					AddedChunkCount.fetch_add(1);
+					WorkCompleted.fetch_add(1);
 				});
 			}
 			for (const auto& Chunk : Chunks)
 			{
-				ThreadPool.ScheduleWork([&Cas, Chunk]() {
+				ThreadPool.ScheduleWork([&Cas, &WorkCompleted, Chunk]() {
 					IoHash	 ChunkHash = Chunk.first;
 					IoBuffer Buffer	   = Cas.FindChunk(ChunkHash);
 					if (Buffer)
 					{
 						CHECK(ChunkHash == IoHash::HashBuffer(Buffer));
 					}
+					WorkCompleted.fetch_add(1);
 				});
 			}
 
@@ -2408,7 +2417,7 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 				Deleted.IterateChunks([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
 			}
 
-			while (ThreadPool.PendingWork() > 0)
+			while (WorkCompleted < NewChunks.size() + Chunks.size())
 			{
 				Sleep(1);
 			}
@@ -2449,14 +2458,16 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 			Deleted.IterateChunks([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
 		}
 		{
+			WorkCompleted = 0;
 			for (const IoHash& ChunkHash : GcChunkHashes)
 			{
-				ThreadPool.ScheduleWork([&Cas, ChunkHash]() {
+				ThreadPool.ScheduleWork([&Cas, &WorkCompleted, ChunkHash]() {
 					CHECK(Cas.HaveChunk(ChunkHash));
 					CHECK(ChunkHash == IoHash::HashBuffer(Cas.FindChunk(ChunkHash)));
+					WorkCompleted.fetch_add(1);
 				});
 			}
-			while (ThreadPool.PendingWork() > 0)
+			while (WorkCompleted < GcChunkHashes.size())
 			{
 				Sleep(1);
 			}