speed up tests (#555)

* faster FileSystemTraversal test
* faster jobqueue test
* faster NamedEvent test
* faster cache tests
* faster basic http tests
* faster blockstore test
* faster cache store tests
* faster compactcas tests
* more responsive zenserver launch
* tweak worker pool sizes in tests

1 files changed, 72 insertions, 58 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 77d21834a..6e51247f1 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -1686,7 +1686,7 @@ TEST_CASE("blockstore.iterate.chunks")
 												.Size		= DefaultIterateSmallChunkWindowSize * 2};
 	BlockStoreLocation BadBlockIndex		 = {.BlockIndex = 0xfffff, .Offset = 1024, .Size = 1024};
 
-	WorkerThreadPool WorkerPool(4);
+	WorkerThreadPool WorkerPool(Max(std::thread::hardware_concurrency() - 1u, 4u));
 
 	std::vector<BlockStoreLocation> Locations{FirstChunkLocation,
 											  SecondChunkLocation,
@@ -1772,7 +1772,7 @@ TEST_CASE("blockstore.iterate.chunks")
 					0);
 				CHECK(Continue);
 			},
-			WorkerThreadPool::EMode::EnableBacklog);
+			WorkerThreadPool::EMode::DisableBacklog);
 		return true;
 	});
 	WorkLatch.CountDown();
@@ -1789,7 +1789,7 @@ TEST_CASE("blockstore.thread.read.write")
 	BlockStore Store;
 	Store.Initialize(RootDirectory / "store", 1088, 1024);
 
-	constexpr size_t	  ChunkCount = 1000;
+	constexpr size_t	  ChunkCount = 500;
 	constexpr size_t	  Alignment	 = 8;
 	std::vector<IoBuffer> Chunks;
 	std::vector<IoHash>	  ChunkHashes;
@@ -1805,70 +1805,84 @@ TEST_CASE("blockstore.thread.read.write")
 	std::vector<BlockStoreLocation> ChunkLocations;
 	ChunkLocations.resize(ChunkCount);
 
-	WorkerThreadPool	WorkerPool(8);
-	std::atomic<size_t> WorkCompleted = 0;
-	for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
-	{
-		WorkerPool.ScheduleWork(
-			[&Store, ChunkIndex, &Chunks, &ChunkLocations, &WorkCompleted]() {
-				IoBuffer& Chunk = Chunks[ChunkIndex];
-				Store.WriteChunk(Chunk.Data(), Chunk.Size(), Alignment, [&](const BlockStoreLocation& L) {
-					ChunkLocations[ChunkIndex] = L;
-				});
-				WorkCompleted.fetch_add(1);
-			},
-			WorkerThreadPool::EMode::DisableBacklog);
-	}
-	while (WorkCompleted < Chunks.size())
+	WorkerThreadPool WorkerPool(Max(std::thread::hardware_concurrency() - 1u, 8u));
 	{
-		Sleep(1);
+		std::atomic<size_t> WorkCompleted = 0;
+		Latch				L(1);
+		for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
+		{
+			L.AddCount(1);
+			WorkerPool.ScheduleWork(
+				[&Store, ChunkIndex, &Chunks, &ChunkLocations, &WorkCompleted, &L]() {
+					auto	  _		= MakeGuard([&L]() { L.CountDown(); });
+					IoBuffer& Chunk = Chunks[ChunkIndex];
+					Store.WriteChunk(Chunk.Data(), Chunk.Size(), Alignment, [&](const BlockStoreLocation& L) {
+						ChunkLocations[ChunkIndex] = L;
+					});
+					WorkCompleted.fetch_add(1);
+				},
+				WorkerThreadPool::EMode::DisableBacklog);
+		}
+		L.CountDown();
+		L.Wait();
+		CHECK(WorkCompleted == Chunks.size());
 	}
 
-	WorkCompleted = 0;
-	for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
-	{
-		WorkerPool.ScheduleWork(
-			[&Store, ChunkIndex, &ChunkLocations, &ChunkHashes, &WorkCompleted]() {
-				IoBuffer VerifyChunk = Store.TryGetChunk(ChunkLocations[ChunkIndex]);
-				CHECK(VerifyChunk);
-				IoHash VerifyHash = IoHash::HashBuffer(VerifyChunk.Data(), VerifyChunk.Size());
-				CHECK(VerifyHash == ChunkHashes[ChunkIndex]);
-				WorkCompleted.fetch_add(1);
-			},
-			WorkerThreadPool::EMode::DisableBacklog);
-	}
-	while (WorkCompleted < Chunks.size())
 	{
-		Sleep(1);
+		std::atomic<size_t> WorkCompleted = 0;
+		Latch				L(1);
+		for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
+		{
+			L.AddCount(1);
+			WorkerPool.ScheduleWork(
+				[&Store, ChunkIndex, &ChunkLocations, &ChunkHashes, &WorkCompleted, &L]() {
+					auto	 _			 = MakeGuard([&L]() { L.CountDown(); });
+					IoBuffer VerifyChunk = Store.TryGetChunk(ChunkLocations[ChunkIndex]);
+					CHECK(VerifyChunk);
+					IoHash VerifyHash = IoHash::HashBuffer(VerifyChunk.Data(), VerifyChunk.Size());
+					CHECK(VerifyHash == ChunkHashes[ChunkIndex]);
+					WorkCompleted.fetch_add(1);
+				},
+				WorkerThreadPool::EMode::DisableBacklog);
+		}
+		L.CountDown();
+		L.Wait();
+		CHECK(WorkCompleted == Chunks.size());
 	}
 
 	std::vector<BlockStoreLocation> SecondChunkLocations;
 	SecondChunkLocations.resize(ChunkCount);
-	WorkCompleted = 0;
-	for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
-	{
-		WorkerPool.ScheduleWork(
-			[&Store, ChunkIndex, &Chunks, &SecondChunkLocations, &WorkCompleted]() {
-				IoBuffer& Chunk = Chunks[ChunkIndex];
-				Store.WriteChunk(Chunk.Data(), Chunk.Size(), Alignment, [&](const BlockStoreLocation& L) {
-					SecondChunkLocations[ChunkIndex] = L;
-				});
-				WorkCompleted.fetch_add(1);
-			},
-			WorkerThreadPool::EMode::DisableBacklog);
-		WorkerPool.ScheduleWork(
-			[&Store, ChunkIndex, &ChunkLocations, &ChunkHashes, &WorkCompleted]() {
-				IoBuffer VerifyChunk = Store.TryGetChunk(ChunkLocations[ChunkIndex]);
-				CHECK(VerifyChunk);
-				IoHash VerifyHash = IoHash::HashBuffer(VerifyChunk.Data(), VerifyChunk.Size());
-				CHECK(VerifyHash == ChunkHashes[ChunkIndex]);
-				WorkCompleted.fetch_add(1);
-			},
-			WorkerThreadPool::EMode::DisableBacklog);
-	}
-	while (WorkCompleted < Chunks.size() * 2)
 	{
-		Sleep(1);
+		std::atomic<size_t> WorkCompleted = 0;
+		Latch				L(1);
+		for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
+		{
+			L.AddCount(1);
+			WorkerPool.ScheduleWork(
+				[&Store, ChunkIndex, &Chunks, &SecondChunkLocations, &WorkCompleted, &L]() {
+					auto	  _		= MakeGuard([&L]() { L.CountDown(); });
+					IoBuffer& Chunk = Chunks[ChunkIndex];
+					Store.WriteChunk(Chunk.Data(), Chunk.Size(), Alignment, [&](const BlockStoreLocation& L) {
+						SecondChunkLocations[ChunkIndex] = L;
+					});
+					WorkCompleted.fetch_add(1);
+				},
+				WorkerThreadPool::EMode::DisableBacklog);
+			L.AddCount(1);
+			WorkerPool.ScheduleWork(
+				[&Store, ChunkIndex, &ChunkLocations, &ChunkHashes, &WorkCompleted, &L]() {
+					auto	 _			 = MakeGuard([&L]() { L.CountDown(); });
+					IoBuffer VerifyChunk = Store.TryGetChunk(ChunkLocations[ChunkIndex]);
+					CHECK(VerifyChunk);
+					IoHash VerifyHash = IoHash::HashBuffer(VerifyChunk.Data(), VerifyChunk.Size());
+					CHECK(VerifyHash == ChunkHashes[ChunkIndex]);
+					WorkCompleted.fetch_add(1);
+				},
+				WorkerThreadPool::EMode::DisableBacklog);
+		}
+		L.CountDown();
+		L.Wait();
+		CHECK(WorkCompleted == Chunks.size() * 2);
 	}
 }