graceful wait in parallelwork destructor (#438)

* exception safety when issuing ParallelWork
* add asserts to Latch usage to catch usage errors
* extended error messaging and recovery handling in ParallelWork destructor to help find issues

2 files changed, 30 insertions, 2 deletions

diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h
index d7e986551..639c6968c 100644
--- a/src/zenutil/include/zenutil/parallelwork.h
+++ b/src/zenutil/include/zenutil/parallelwork.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zencore/scopeguard.h>
 #include <zencore/thread.h>
 #include <zencore/workthreadpool.h>
 
@@ -26,6 +27,7 @@ public:
 		try
 		{
 			WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] {
+				auto _ = MakeGuard([this]() { m_PendingWork.CountDown(); });
 				try
 				{
 					while (m_PauseFlag && !m_AbortFlag)
@@ -38,7 +40,6 @@ public:
 				{
 					OnError(std::current_exception(), m_AbortFlag);
 				}
-				m_PendingWork.CountDown();
 			});
 		}
 		catch (const std::exception&)
diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp
index 67fc03c04..aa806438b 100644
--- a/src/zenutil/parallelwork.cpp
+++ b/src/zenutil/parallelwork.cpp
@@ -2,6 +2,7 @@
 
 #include <zenutil/parallelwork.h>
 
+#include <zencore/callstack.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -34,7 +35,33 @@ ParallelWork::~ParallelWork()
 			m_PendingWork.CountDown();
 		}
 		m_PendingWork.Wait();
-		ZEN_ASSERT(m_PendingWork.Remaining() == 0);
+		ptrdiff_t RemainingWork = m_PendingWork.Remaining();
+		if (RemainingWork != 0)
+		{
+			void*			 Frames[8];
+			uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+			CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+			ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}",
+					  RemainingWork,
+					  CallstackToString(Callstack, "  "));
+			FreeCallstack(Callstack);
+
+			uint32_t WaitedMs = 0;
+			while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000)
+			{
+				Sleep(50);
+				WaitedMs += 50;
+			}
+			RemainingWork = m_PendingWork.Remaining();
+			if (RemainingWork != 0)
+			{
+				ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork)
+			}
+			else
+			{
+				ZEN_INFO("ParallelWork destructor safety wait succeeded");
+			}
+		}
 	}
 	catch (const std::exception& Ex)
 	{