graceful wait in parallelwork destructor (#438)

* exception safety when issuing ParallelWork
* add asserts to Latch usage to catch usage errors
* extended error messaging and recovery handling in ParallelWork destructor to help find issues

1 files changed, 28 insertions, 1 deletions

diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp
index 67fc03c04..aa806438b 100644
--- a/src/zenutil/parallelwork.cpp
+++ b/src/zenutil/parallelwork.cpp
@@ -2,6 +2,7 @@
 
 #include <zenutil/parallelwork.h>
 
+#include <zencore/callstack.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -34,7 +35,33 @@ ParallelWork::~ParallelWork()
 			m_PendingWork.CountDown();
 		}
 		m_PendingWork.Wait();
-		ZEN_ASSERT(m_PendingWork.Remaining() == 0);
+		ptrdiff_t RemainingWork = m_PendingWork.Remaining();
+		if (RemainingWork != 0)
+		{
+			void*			 Frames[8];
+			uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+			CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+			ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}",
+					  RemainingWork,
+					  CallstackToString(Callstack, "  "));
+			FreeCallstack(Callstack);
+
+			uint32_t WaitedMs = 0;
+			while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000)
+			{
+				Sleep(50);
+				WaitedMs += 50;
+			}
+			RemainingWork = m_PendingWork.Remaining();
+			if (RemainingWork != 0)
+			{
+				ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork)
+			}
+			else
+			{
+				ZEN_INFO("ParallelWork destructor safety wait succeeded");
+			}
+		}
 	}
 	catch (const std::exception& Ex)
 	{