Make sure we wait for all scheduled tasks to complete before throwing exceptions further (#662)

Bugfix: We must not throw exceptions to calling function until all async work we spawned has returned

1 files changed, 31 insertions, 9 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 1ffc959e7..b9cb89fc9 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -2647,7 +2647,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 std::string
 ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
 {
-	return fmt::format("cachebucket:'{}'", m_BucketDir.string());
+	return fmt::format("cachebucket: '{}'", m_BucketDir.string());
 }
 
 class DiskBucketStoreCompactor : public GcStoreCompactor
@@ -3039,6 +3039,8 @@ public:
 		}
 	}
 
+	virtual std::string GetGcName(GcCtx& Ctx) override { return m_CacheBucket.GetGcName(Ctx); }
+
 	virtual void PreCache(GcCtx& Ctx) override
 	{
 		ZEN_TRACE_CPU("Z$::Bucket::PreCache");
@@ -3510,8 +3512,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 	{
 		WorkLatch.AddCount(1);
 		Pool.ScheduleWork([&]() {
-			auto _ = MakeGuard([&]() { WorkLatch.CountDown(); });
-
+			auto			  _			 = MakeGuard([&]() { WorkLatch.CountDown(); });
 			const std::string BucketName = PathToUtf8(BucketPath.stem());
 			try
 			{
@@ -3621,13 +3622,27 @@ ZenCacheDiskLayer::Flush()
 	{
 		WorkerThreadPool& Pool = GetSmallWorkerPool();
 		Latch			  WorkLatch(1);
-		for (auto& Bucket : Buckets)
+		try
 		{
-			WorkLatch.AddCount(1);
-			Pool.ScheduleWork([&]() {
-				auto _ = MakeGuard([&]() { WorkLatch.CountDown(); });
-				Bucket->Flush();
-			});
+			for (auto& Bucket : Buckets)
+			{
+				WorkLatch.AddCount(1);
+				Pool.ScheduleWork([&]() {
+					auto _ = MakeGuard([&]() { WorkLatch.CountDown(); });
+					try
+					{
+						Bucket->Flush();
+					}
+					catch (std::exception& Ex)
+					{
+						ZEN_ERROR("Failed flushing bucket. Reason: '{}'", Ex.what());
+					}
+				});
+			}
+		}
+		catch (std::exception& Ex)
+		{
+			ZEN_ERROR("Failed to flush buckets at '{}'. Reason: '{}'", m_RootDir, Ex.what());
 		}
 		WorkLatch.CountDown();
 		while (!WorkLatch.Wait(1000))
@@ -3660,6 +3675,13 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
 
 		for (auto& Result : Results)
 		{
+			if (Result.valid())
+			{
+				Result.wait();
+			}
+		}
+		for (auto& Result : Results)
+		{
 			Result.get();
 		}
 	}