clean up any incomplete blocks if ReclaimSpace fails

1 files changed, 153 insertions, 136 deletions

diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index b82b93823..2573863a8 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -404,177 +404,194 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 	}
 
 	Ref<BlockStoreFile> NewBlockFile;
-	uint64_t			WriteOffset	  = 0;
-	uint32_t			NewBlockIndex = 0;
-
-	for (uint32_t BlockIndex : BlocksToReWrite)
+	try
 	{
-		const size_t ChunkMapIndex = BlockIndexToChunkMapIndex[BlockIndex];
-
-		Ref<BlockStoreFile> OldBlockFile;
+		uint64_t WriteOffset   = 0;
+		uint32_t NewBlockIndex = 0;
+		for (uint32_t BlockIndex : BlocksToReWrite)
 		{
-			RwLock::SharedLockScope _i(m_InsertLock);
-			Stopwatch				Timer;
-			const auto				__ = MakeGuard([&] {
-				 uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-				 WriteBlockTimeUs += ElapsedUs;
-				 WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
-			 });
-			OldBlockFile			   = m_ChunkBlocks[BlockIndex];
-			ZEN_ASSERT(OldBlockFile);
-		}
+			const size_t ChunkMapIndex = BlockIndexToChunkMapIndex[BlockIndex];
 
-		const ChunkIndexArray& KeepMap = BlockKeepChunks[ChunkMapIndex];
-		if (KeepMap.empty())
-		{
-			const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
-			for (size_t DeleteIndex : DeleteMap)
+			Ref<BlockStoreFile> OldBlockFile;
 			{
-				DeletedSize += ChunkLocations[DeleteIndex].Size;
+				RwLock::SharedLockScope _i(m_InsertLock);
+				Stopwatch				Timer;
+				const auto				__ = MakeGuard([&] {
+					 uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
+					 WriteBlockTimeUs += ElapsedUs;
+					 WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
+				 });
+				OldBlockFile			   = m_ChunkBlocks[BlockIndex];
+				ZEN_ASSERT(OldBlockFile);
 			}
-			ChangeCallback({}, DeleteMap);
-			DeletedCount += DeleteMap.size();
-			{
-				RwLock::ExclusiveLockScope _i(m_InsertLock);
-				Stopwatch				   Timer;
-				const auto				   __ = MakeGuard([&] {
-					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-					ReadBlockTimeUs += ElapsedUs;
-					ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
-				});
-				m_ChunkBlocks[BlockIndex]	  = nullptr;
-			}
-			ZEN_DEBUG("marking cas store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
-			std::error_code Ec;
-			OldBlockFile->MarkAsDeleteOnClose(Ec);
-			if (Ec)
-			{
-				ZEN_WARN("Failed to flag file '{}' for deletion: '{}'", OldBlockFile->GetPath(), Ec.message());
-			}
-			continue;
-		}
 
-		MovedChunksArray	 MovedChunks;
-		std::vector<uint8_t> Chunk;
-		for (const size_t& ChunkIndex : KeepMap)
-		{
-			const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
-			Chunk.resize(ChunkLocation.Size);
-			OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
-
-			if (!NewBlockFile || (WriteOffset + Chunk.size() > m_MaxBlockSize))
+			const ChunkIndexArray& KeepMap = BlockKeepChunks[ChunkMapIndex];
+			if (KeepMap.empty())
 			{
-				uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
-
-				if (NewBlockFile)
+				const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
+				for (size_t DeleteIndex : DeleteMap)
 				{
-					NewBlockFile->Truncate(WriteOffset);
-					NewBlockFile->Flush();
+					DeletedSize += ChunkLocations[DeleteIndex].Size;
 				}
+				ChangeCallback({}, DeleteMap);
+				DeletedCount += DeleteMap.size();
 				{
-					ChangeCallback(MovedChunks, {});
-					MovedCount += KeepMap.size();
-					MovedChunks.clear();
-					RwLock::ExclusiveLockScope __(m_InsertLock);
+					RwLock::ExclusiveLockScope _i(m_InsertLock);
 					Stopwatch				   Timer;
-					const auto				   ___ = MakeGuard([&] {
+					const auto				   __ = MakeGuard([&] {
 						uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
 						ReadBlockTimeUs += ElapsedUs;
 						ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
 					});
-					if (m_ChunkBlocks.size() == m_MaxBlockCount)
-					{
-						ZEN_ERROR("unable to allocate a new block in '{}', count limit {} exeeded",
-								  m_BlocksBasePath,
-								  static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
-						return;
-					}
-					while (m_ChunkBlocks.contains(NextBlockIndex))
-					{
-						NextBlockIndex = (NextBlockIndex + 1) & (m_MaxBlockCount - 1);
-					}
-					std::filesystem::path NewBlockPath = GetBlockPath(m_BlocksBasePath, NextBlockIndex);
-					NewBlockFile					   = new BlockStoreFile(NewBlockPath);
-					m_ChunkBlocks[NextBlockIndex]	   = NewBlockFile;
+					m_ChunkBlocks[BlockIndex]	  = nullptr;
 				}
-
-				std::error_code Error;
-				DiskSpace		Space = DiskSpaceInfo(m_BlocksBasePath, Error);
-				if (Error)
+				ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+				std::error_code Ec;
+				OldBlockFile->MarkAsDeleteOnClose(Ec);
+				if (Ec)
 				{
-					ZEN_ERROR("get disk space in '{}' FAILED, reason: '{}'", m_BlocksBasePath, Error.message());
-					return;
+					ZEN_WARN("Failed to flag file '{}' for deletion: '{}'", OldBlockFile->GetPath(), Ec.message());
 				}
-				if (Space.Free < m_MaxBlockSize)
+				continue;
+			}
+
+			MovedChunksArray	 MovedChunks;
+			std::vector<uint8_t> Chunk;
+			for (const size_t& ChunkIndex : KeepMap)
+			{
+				const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
+				Chunk.resize(ChunkLocation.Size);
+				OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
+
+				if (!NewBlockFile || (WriteOffset + Chunk.size() > m_MaxBlockSize))
 				{
-					uint64_t ReclaimedSpace = DiskReserveCallback();
-					if (Space.Free + ReclaimedSpace < m_MaxBlockSize)
+					uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
+
+					if (NewBlockFile)
 					{
-						ZEN_WARN("garbage collect for '{}' FAILED, required disk space {}, free {}",
-								 m_BlocksBasePath,
-								 m_MaxBlockSize,
-								 NiceBytes(Space.Free + ReclaimedSpace));
-						RwLock::ExclusiveLockScope _l(m_InsertLock);
+						NewBlockFile->Truncate(WriteOffset);
+						NewBlockFile->Flush();
+						NewBlockFile = nullptr;
+					}
+					{
+						ChangeCallback(MovedChunks, {});
+						MovedCount += KeepMap.size();
+						MovedChunks.clear();
+						RwLock::ExclusiveLockScope __(m_InsertLock);
 						Stopwatch				   Timer;
-						const auto				   __ = MakeGuard([&] {
+						const auto				   ___ = MakeGuard([&] {
 							uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
 							ReadBlockTimeUs += ElapsedUs;
 							ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
 						});
-						m_ChunkBlocks.erase(NextBlockIndex);
+						if (m_ChunkBlocks.size() == m_MaxBlockCount)
+						{
+							ZEN_ERROR("unable to allocate a new block in '{}', count limit {} exeeded",
+									  m_BlocksBasePath,
+									  static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
+							return;
+						}
+						while (m_ChunkBlocks.contains(NextBlockIndex))
+						{
+							NextBlockIndex = (NextBlockIndex + 1) & (m_MaxBlockCount - 1);
+						}
+						std::filesystem::path NewBlockPath = GetBlockPath(m_BlocksBasePath, NextBlockIndex);
+						NewBlockFile					   = new BlockStoreFile(NewBlockPath);
+						m_ChunkBlocks[NextBlockIndex]	   = NewBlockFile;
+					}
+
+					std::error_code Error;
+					DiskSpace		Space = DiskSpaceInfo(m_BlocksBasePath, Error);
+					if (Error)
+					{
+						ZEN_ERROR("get disk space in '{}' FAILED, reason: '{}'", m_BlocksBasePath, Error.message());
 						return;
 					}
+					if (Space.Free < m_MaxBlockSize)
+					{
+						uint64_t ReclaimedSpace = DiskReserveCallback();
+						if (Space.Free + ReclaimedSpace < m_MaxBlockSize)
+						{
+							ZEN_WARN("garbage collect for '{}' FAILED, required disk space {}, free {}",
+									 m_BlocksBasePath,
+									 m_MaxBlockSize,
+									 NiceBytes(Space.Free + ReclaimedSpace));
+							RwLock::ExclusiveLockScope _l(m_InsertLock);
+							Stopwatch				   Timer;
+							const auto				   __ = MakeGuard([&] {
+								uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
+								ReadBlockTimeUs += ElapsedUs;
+								ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
+							});
+							m_ChunkBlocks.erase(NextBlockIndex);
+							return;
+						}
 
-					ZEN_INFO("using gc reserve for '{}', reclaimed {}, disk free {}",
-							 m_BlocksBasePath,
-							 ReclaimedSpace,
-							 NiceBytes(Space.Free + ReclaimedSpace));
+						ZEN_INFO("using gc reserve for '{}', reclaimed {}, disk free {}",
+								 m_BlocksBasePath,
+								 ReclaimedSpace,
+								 NiceBytes(Space.Free + ReclaimedSpace));
+					}
+					NewBlockFile->Create(m_MaxBlockSize);
+					NewBlockIndex = NextBlockIndex;
+					WriteOffset	  = 0;
 				}
-				NewBlockFile->Create(m_MaxBlockSize);
-				NewBlockIndex = NextBlockIndex;
-				WriteOffset	  = 0;
-			}
 
-			NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
-			MovedChunks.push_back({ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = WriteOffset, .Size = Chunk.size()}});
-			WriteOffset = RoundUp(WriteOffset + Chunk.size(), PayloadAlignment);
-		}
-		Chunk.clear();
-		if (NewBlockFile)
-		{
-			NewBlockFile->Truncate(WriteOffset);
-			NewBlockFile->Flush();
-			NewBlockFile = {};
-		}
+				NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
+				MovedChunks.push_back({ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = WriteOffset, .Size = Chunk.size()}});
+				WriteOffset = RoundUp(WriteOffset + Chunk.size(), PayloadAlignment);
+			}
+			Chunk.clear();
+			if (NewBlockFile)
+			{
+				NewBlockFile->Truncate(WriteOffset);
+				NewBlockFile->Flush();
+				NewBlockFile = nullptr;
+			}
 
-		const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
-		for (size_t DeleteIndex : DeleteMap)
-		{
-			DeletedSize += ChunkLocations[DeleteIndex].Size;
-		}
+			const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
+			for (size_t DeleteIndex : DeleteMap)
+			{
+				DeletedSize += ChunkLocations[DeleteIndex].Size;
+			}
 
-		ChangeCallback(MovedChunks, DeleteMap);
-		MovedCount += KeepMap.size();
-		DeletedCount += DeleteMap.size();
-		MovedChunks.clear();
-		{
-			RwLock::ExclusiveLockScope __(m_InsertLock);
-			Stopwatch				   Timer;
-			const auto				   ___ = MakeGuard([&] {
-				uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-				ReadBlockTimeUs += ElapsedUs;
-				ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
-			});
-			m_ChunkBlocks[BlockIndex]	   = nullptr;
+			ChangeCallback(MovedChunks, DeleteMap);
+			MovedCount += KeepMap.size();
+			DeletedCount += DeleteMap.size();
+			MovedChunks.clear();
+			{
+				RwLock::ExclusiveLockScope __(m_InsertLock);
+				Stopwatch				   Timer;
+				const auto				   ___ = MakeGuard([&] {
+					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
+					ReadBlockTimeUs += ElapsedUs;
+					ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
+				});
+				m_ChunkBlocks[BlockIndex]	   = nullptr;
+			}
+			ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+			std::error_code Ec;
+			OldBlockFile->MarkAsDeleteOnClose(Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to flag file '{}' for deletion: '{}'", OldBlockFile->GetPath(), Ec.message());
+			}
+			OldBlockFile = nullptr;
 		}
-		ZEN_DEBUG("marking cas store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
-		std::error_code Ec;
-		OldBlockFile->MarkAsDeleteOnClose(Ec);
-		if (Ec)
+	}
+	catch (std::exception& ex)
+	{
+		ZEN_ERROR("reclaiming space for '{}' failed with: '{}'", m_BlocksBasePath, ex.what());
+		if (NewBlockFile)
 		{
-			ZEN_WARN("Failed to flag file '{}' for deletion: '{}'", OldBlockFile->GetPath(), Ec.message());
+			ZEN_DEBUG("dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
+			std::error_code Ec;
+			NewBlockFile->MarkAsDeleteOnClose(Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to flag file '{}' for deletion: '{}'", NewBlockFile->GetPath(), Ec.message());
+			}
 		}
-		OldBlockFile = nullptr;
 	}
 }