stop exceptions from leaking on threaded work (#102)

* catch exceptions in threaded work
* don't abort all project file/chunk info fetch for single failure

2 files changed, 69 insertions, 39 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index a542a6581..0f9481210 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -965,7 +965,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid>													 ChunkIds,
 				break;
 			}
 			WorkLatch.AddCount(1);
-			OptionalWorkerPool->ScheduleWork([&WorkLatch, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() {
+			OptionalWorkerPool->ScheduleWork([this, &WorkLatch, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() {
 				auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
 				if (Result.load() == false)
 				{
@@ -973,14 +973,21 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid>													 ChunkIds,
 				}
 				size_t						 FileChunkIndex = FileChunkIndexes[ChunkIndex];
 				const std::filesystem::path& FilePath		= FileChunkPaths[ChunkIndex];
-				IoBuffer					 Payload		= IoBufferBuilder::MakeFromFile(FilePath);
-				if (Payload)
+				try
 				{
-					if (!AsyncCallback(FileChunkIndex, Payload))
+					IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
+					if (Payload)
 					{
-						Result.store(false);
+						if (!AsyncCallback(FileChunkIndex, Payload))
+						{
+							Result.store(false);
+						}
 					}
 				}
+				catch (const std::exception& Ex)
+				{
+					ZEN_WARN("Exception caught when iterating file chunk {}, path '{}'. Reason: '{}'", FileChunkIndex, FilePath, Ex.what());
+				}
 			});
 		}
 
@@ -2838,20 +2845,27 @@ ProjectStore::GetProjectFiles(const std::string_view				 ProjectId,
 		FoundLog->IterateChunks(
 			Ids,
 			[&](size_t Index, const IoBuffer& Payload) {
-				uint64_t Size = Payload.GetSize();
-				if (WantsRawSizeField)
+				try
 				{
-					uint64_t RawSize = Size;
-					if (Payload.GetContentType() == ZenContentType::kCompressedBinary)
+					uint64_t Size = Payload.GetSize();
+					if (WantsRawSizeField)
+					{
+						uint64_t RawSize = Size;
+						if (Payload.GetContentType() == ZenContentType::kCompressedBinary)
+						{
+							IoHash __;
+							(void)CompressedBuffer::FromCompressed(SharedBuffer(Payload), __, RawSize);
+						}
+						RawSizes[Index] = RawSize;
+					}
+					if (WantsSizeField)
 					{
-						IoHash __;
-						(void)CompressedBuffer::FromCompressed(SharedBuffer(Payload), __, RawSize);
+						Sizes[Index] = Size;
 					}
-					RawSizes[Index] = RawSize;
 				}
-				if (WantsSizeField)
+				catch (const std::exception& Ex)
 				{
-					Sizes[Index] = Size;
+					ZEN_WARN("Failed getting project file info for id {}. Reason: '{}'", Ids[Index], Ex.what());
 				}
 				return true;
 			},
@@ -2955,20 +2969,27 @@ ProjectStore::GetProjectChunkInfos(const std::string_view				  ProjectId,
 		(void)FoundLog->IterateChunks(
 			Hashes,
 			[&](size_t Index, const IoBuffer& Chunk) -> bool {
-				uint64_t Size = Chunk.GetSize();
-				if (WantsRawSizeField)
+				try
 				{
-					uint64_t RawSize = Size;
-					if (Chunk.GetContentType() == ZenContentType::kCompressedBinary)
+					uint64_t Size = Chunk.GetSize();
+					if (WantsRawSizeField)
+					{
+						uint64_t RawSize = Size;
+						if (Chunk.GetContentType() == ZenContentType::kCompressedBinary)
+						{
+							IoHash __;
+							(void)CompressedBuffer::FromCompressed(SharedBuffer(Chunk), __, RawSize);
+						}
+						RawSizes[Index] = RawSize;
+					}
+					if (WantsSizeField)
 					{
-						IoHash __;
-						(void)CompressedBuffer::FromCompressed(SharedBuffer(Chunk), __, RawSize);
+						Sizes[Index] = Size;
 					}
-					RawSizes[Index] = RawSize;
 				}
-				if (WantsSizeField)
+				catch (const std::exception& Ex)
 				{
-					Sizes[Index] = Size;
+					ZEN_WARN("Failed getting chunk info for id {}. Reason: '{}'", Ids[Index], Ex.what());
 				}
 				return true;
 			},
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index de6753a6b..15d329442 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -237,22 +237,31 @@ CreateBlock(WorkerThreadPool&											  WorkerPool,
 			{
 				return;
 			}
-			ZEN_ASSERT(!Chunks.empty());
-			size_t			 ChunkCount = Chunks.size();
-			Stopwatch		 Timer;
-			CompressedBuffer CompressedBlock = GenerateBlock(std::move(Chunks));
-			IoHash			 BlockHash		 = CompressedBlock.DecodeRawHash();
-			{
-				// We can share the lock as we are not resizing the vector and only touch BlockHash at our own index
-				RwLock::SharedLockScope __(SectionsLock);
-				Blocks[BlockIndex].BlockHash = BlockHash;
-			}
-			uint64_t BlockSize = CompressedBlock.GetCompressedSize();
-			AsyncOnBlock(std::move(CompressedBlock), BlockHash);
-			ZEN_INFO("Generated block with {} attachments in {} ({})",
-					 ChunkCount,
-					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
-					 NiceBytes(BlockSize));
+			size_t ChunkCount = Chunks.size();
+			try
+			{
+				ZEN_ASSERT(ChunkCount > 0);
+				Stopwatch		 Timer;
+				CompressedBuffer CompressedBlock = GenerateBlock(std::move(Chunks));
+				IoHash			 BlockHash		 = CompressedBlock.DecodeRawHash();
+				{
+					// We can share the lock as we are not resizing the vector and only touch BlockHash at our own index
+					RwLock::SharedLockScope __(SectionsLock);
+					Blocks[BlockIndex].BlockHash = BlockHash;
+				}
+				uint64_t BlockSize = CompressedBlock.GetCompressedSize();
+				AsyncOnBlock(std::move(CompressedBlock), BlockHash);
+				ZEN_INFO("Generated block with {} attachments in {} ({})",
+						 ChunkCount,
+						 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+						 NiceBytes(BlockSize));
+			}
+			catch (const std::exception& Ex)
+			{
+				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::InternalServerError),
+									  fmt::format("Failed creating block {} with {} chunks", BlockIndex, ChunkCount),
+									  Ex.what());
+			}
 		});
 }