faster accesstime save restore (#439)

- Improvement: Reduce time a cache bucket is locked for write when flushing/garbage collecting
  - Change format for faster read/write and reduced size on disk
  - Don't lock index while writing manifest to disk
  - Skip garbage collect if we are currently in a Flush operation
  - BlockStore::Flush no longer terminates currently writing block
  - Garbage collect references to currently writing block but keep the block as new data may be added
  - Fix BlockStore::Prune used disk space calculation
  - Don't materialize data in filecas when we just need the size

5 files changed, 131 insertions, 124 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index b5ed17fc6..f99b0bc4a 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -233,7 +233,6 @@ BlockStore::Prune(const std::vector<BlockStoreLocation>& KnownLocations)
 		if (!KnownBlocks.contains(BlockIndex))
 		{
 			Ref<BlockStoreFile> BlockFile = m_ChunkBlocks[BlockIndex];
-			m_TotalSize.fetch_add(BlockFile->FileSize(), std::memory_order::relaxed);
 			BlocksToDelete.push_back(BlockIndex);
 		}
 	}
@@ -242,6 +241,7 @@ BlockStore::Prune(const std::vector<BlockStoreLocation>& KnownLocations)
 	{
 		// Clear out unused blocks
 		Ref<BlockStoreFile> BlockFile = m_ChunkBlocks[BlockIndex];
+		m_TotalSize.fetch_sub(BlockFile->FileSize(), std::memory_order::relaxed);
 		m_ChunkBlocks.erase(BlockIndex);
 		ZEN_DEBUG("marking block store file '{}' for delete, block #{}", BlockFile->GetPath(), BlockIndex);
 		BlockFile->MarkAsDeleteOnClose();
@@ -354,22 +354,31 @@ BlockStore::TryGetChunk(const BlockStoreLocation& Location) const
 }
 
 void
-BlockStore::Flush()
+BlockStore::Flush(bool ForceNewBlock)
 {
 	ZEN_TRACE_CPU("BlockStore::Flush");
 
-	RwLock::ExclusiveLockScope _(m_InsertLock);
-	if (m_CurrentInsertOffset > 0)
+	if (ForceNewBlock)
 	{
-		uint32_t WriteBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
-		WriteBlockIndex			 = (WriteBlockIndex + 1) & (m_MaxBlockCount - 1);
-		if (m_WriteBlock)
+		RwLock::ExclusiveLockScope _(m_InsertLock);
+		if (m_CurrentInsertOffset > 0)
 		{
-			m_WriteBlock->Flush();
+			uint32_t WriteBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
+			WriteBlockIndex			 = (WriteBlockIndex + 1) & (m_MaxBlockCount - 1);
+			if (m_WriteBlock)
+			{
+				m_WriteBlock->Flush();
+			}
+			m_WriteBlock = nullptr;
+			m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release);
+			m_CurrentInsertOffset = 0;
 		}
-		m_WriteBlock = nullptr;
-		m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release);
-		m_CurrentInsertOffset = 0;
+		return;
+	}
+	RwLock::SharedLockScope _(m_InsertLock);
+	if (m_WriteBlock)
+	{
+		m_WriteBlock->Flush();
 	}
 }
 
@@ -449,11 +458,6 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 	{
 		const BlockStoreLocation& Location = ChunkLocations[Index];
 		OldTotalSize += Location.Size;
-		if (Snapshot.m_ActiveWriteBlocks.contains(Location.BlockIndex))
-		{
-			continue;
-		}
-
 		auto   BlockIndexPtr = BlockIndexToChunkMapIndex.find(Location.BlockIndex);
 		size_t ChunkMapIndex = 0;
 		if (BlockIndexPtr == BlockIndexToChunkMapIndex.end())
@@ -524,9 +528,12 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 		uint32_t NewBlockIndex = 0;
 		for (uint32_t BlockIndex : BlocksToReWrite)
 		{
+			bool IsActiveWriteBlock = Snapshot.m_ActiveWriteBlocks.contains(BlockIndex);
+
 			const size_t ChunkMapIndex = BlockIndexToChunkMapIndex[BlockIndex];
 
 			Ref<BlockStoreFile> OldBlockFile;
+			if (!IsActiveWriteBlock)
 			{
 				RwLock::SharedLockScope _i(m_InsertLock);
 				Stopwatch				Timer;
@@ -553,6 +560,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 				}
 				ChangeCallback({}, DeleteMap);
 				DeletedCount += DeleteMap.size();
+				if (OldBlockFile)
 				{
 					RwLock::ExclusiveLockScope _i(m_InsertLock);
 					Stopwatch				   Timer;
@@ -561,18 +569,15 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 						ReadBlockTimeUs += ElapsedUs;
 						ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
 					});
-					if (OldBlockFile)
-					{
-						ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
-						ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
-						m_ChunkBlocks.erase(BlockIndex);
-						m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
-						OldBlockFile->MarkAsDeleteOnClose();
-					}
+					ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+					ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
+					m_ChunkBlocks.erase(BlockIndex);
+					m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
+					OldBlockFile->MarkAsDeleteOnClose();
 				}
 				continue;
 			}
-			else if (!OldBlockFile)
+			else if (!OldBlockFile && !IsActiveWriteBlock)
 			{
 				// If the block file pointed to does not exist, move any keep chunk them to deleted list
 				ZEN_ERROR("Expected to find block {} in {} - this should never happen, marking {} entries as deleted.",
@@ -585,6 +590,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 			}
 
 			MovedChunksArray MovedChunks;
+			if (OldBlockFile)
 			{
 				ZEN_TRACE_CPU("BlockStore::ReclaimSpace::MoveBlock");
 				std::vector<uint8_t> Chunk;
@@ -689,9 +695,11 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 			}
 
 			ChangeCallback(MovedChunks, DeleteMap);
-			MovedCount += KeepMap.size();
+			MovedCount += MovedChunks.size();
 			DeletedCount += DeleteMap.size();
 			MovedChunks.clear();
+
+			if (OldBlockFile)
 			{
 				RwLock::ExclusiveLockScope __(m_InsertLock);
 				Stopwatch				   Timer;
@@ -700,14 +708,12 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 					ReadBlockTimeUs += ElapsedUs;
 					ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
 				});
-				if (OldBlockFile)
-				{
-					ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
-					ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
-					m_ChunkBlocks.erase(BlockIndex);
-					m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
-					OldBlockFile->MarkAsDeleteOnClose();
-				}
+
+				ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+				ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
+				m_ChunkBlocks.erase(BlockIndex);
+				m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
+				OldBlockFile->MarkAsDeleteOnClose();
 			}
 		}
 		if (NewBlockFile)
@@ -1117,7 +1123,7 @@ TEST_CASE("blockstore.clean.stray.blocks")
 	CHECK(GetDirectoryContent(RootDirectory / "store", true, false).size() == 1);
 }
 
-TEST_CASE("blockstore.flush.forces.new.block")
+TEST_CASE("blockstore.flush.force.new.block")
 {
 	using namespace blockstore::impl;
 
@@ -1129,10 +1135,10 @@ TEST_CASE("blockstore.flush.forces.new.block")
 
 	std::string FirstChunkData = "This is the data of the first chunk that we will write";
 	WriteStringAsChunk(Store, FirstChunkData, 4);
-	Store.Flush();
+	Store.Flush(/*ForceNewBlock*/ true);
 	std::string SecondChunkData = "This is the data for the second chunk that we will write";
 	WriteStringAsChunk(Store, SecondChunkData, 4);
-	Store.Flush();
+	Store.Flush(/*ForceNewBlock*/ true);
 	std::string ThirdChunkData =
 		"This is a much longer string that will not fit in the first block so it should be placed in the second block";
 	WriteStringAsChunk(Store, ThirdChunkData, 4);
@@ -1157,7 +1163,7 @@ TEST_CASE("blockstore.iterate.chunks")
 
 	std::string		   SecondChunkData	   = "This is the data for the second chunk that we will write";
 	BlockStoreLocation SecondChunkLocation = WriteStringAsChunk(Store, SecondChunkData, 4);
-	Store.Flush();
+	Store.Flush(/*ForceNewBlock*/ false);
 
 	std::string		   VeryLargeChunk(ScrubSmallChunkWindowSize * 2, 'L');
 	BlockStoreLocation VeryLargeChunkLocation = WriteStringAsChunk(Store, VeryLargeChunk, 4);
@@ -1267,7 +1273,7 @@ TEST_CASE("blockstore.reclaim.space")
 		ChunksToKeep.push_back(ChunkIndex);
 	}
 
-	Store.Flush();
+	Store.Flush(/*ForceNewBlock*/ false);
 	BlockStore::ReclaimSnapshotState State1 = Store.GetReclaimSnapshotState();
 	Store.ReclaimSpace(State1, ChunkLocations, ChunksToKeep, Alignment, true);
 
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 1d1797597..ce2e53527 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -230,7 +230,7 @@ CasContainerStrategy::FilterChunks(HashKeySet& InOutChunks)
 void
 CasContainerStrategy::Flush()
 {
-	m_BlockStore.Flush();
+	m_BlockStore.Flush(/*ForceNewBlock*/ false);
 	m_CasLog.Flush();
 	MakeIndexSnapshot();
 }
@@ -801,7 +801,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		auto						  BlockIt	   = BlockSizes.find(DiskLocation.GetBlockIndex());
 		if (BlockIt == BlockSizes.end())
 		{
-			ZEN_WARN("Unknown block {} for entry {}", DiskLocation.GetBlockIndex(), Entry.first.ToHexString());
+			ZEN_WARN("Unknown block {} for entry {} in '{}'", DiskLocation.GetBlockIndex(), Entry.first.ToHexString(), BasePath);
 		}
 		else
 		{
@@ -810,7 +810,10 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			uint64_t BlockSize = BlockIt->second;
 			if (BlockLocation.Offset + BlockLocation.Size > BlockSize)
 			{
-				ZEN_WARN("Range is outside of block {} for entry {}", BlockLocation.BlockIndex, Entry.first.ToHexString());
+				ZEN_WARN("Range is outside of block {} for entry {} in '{}'",
+						 BlockLocation.BlockIndex,
+						 Entry.first.ToHexString(),
+						 BasePath);
 			}
 			else
 			{
@@ -1068,7 +1071,6 @@ TEST_CASE("compactcas.gc.removefile")
 
 TEST_CASE("compactcas.gc.compact")
 {
-	//	for (uint32_t i = 0; i < 100; ++i)
 	{
 		ScopedTemporaryDirectory TempDir;
 
@@ -1111,6 +1113,17 @@ TEST_CASE("compactcas.gc.compact")
 		CHECK(Cas.HaveChunk(ChunkHashes[7]));
 		CHECK(Cas.HaveChunk(ChunkHashes[8]));
 
+		auto ValidateChunkExists = [&](size_t Index) {
+			IoBuffer Chunk	= Cas.FindChunk(ChunkHashes[Index]);
+			bool	 Exists = !!Chunk;
+			CHECK(Exists);
+			IoHash Hash = IoHash::HashBuffer(Chunk);
+			if (ChunkHashes[Index] != Hash)
+			{
+				CHECK(fmt::format("{}", ChunkHashes[Index]) == fmt::format("{}", Hash));
+			}
+		};
+
 		// Keep first and last
 		{
 			GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
@@ -1134,8 +1147,8 @@ TEST_CASE("compactcas.gc.compact")
 			CHECK(!Cas.HaveChunk(ChunkHashes[7]));
 			CHECK(Cas.HaveChunk(ChunkHashes[8]));
 
-			CHECK(ChunkHashes[0] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[0])));
-			CHECK(ChunkHashes[8] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[8])));
+			ValidateChunkExists(0);
+			ValidateChunkExists(8);
 
 			Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
 			Cas.InsertChunk(Chunks[2], ChunkHashes[2]);
@@ -1167,7 +1180,7 @@ TEST_CASE("compactcas.gc.compact")
 			CHECK(!Cas.HaveChunk(ChunkHashes[7]));
 			CHECK(Cas.HaveChunk(ChunkHashes[8]));
 
-			CHECK(ChunkHashes[8] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[8])));
+			ValidateChunkExists(8);
 
 			Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
 			Cas.InsertChunk(Chunks[2], ChunkHashes[2]);
@@ -1201,9 +1214,9 @@ TEST_CASE("compactcas.gc.compact")
 			CHECK(Cas.HaveChunk(ChunkHashes[7]));
 			CHECK(!Cas.HaveChunk(ChunkHashes[8]));
 
-			CHECK(ChunkHashes[1] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[1])));
-			CHECK(ChunkHashes[4] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[4])));
-			CHECK(ChunkHashes[7] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[7])));
+			ValidateChunkExists(1);
+			ValidateChunkExists(4);
+			ValidateChunkExists(7);
 
 			Cas.InsertChunk(Chunks[0], ChunkHashes[0]);
 			Cas.InsertChunk(Chunks[2], ChunkHashes[2]);
@@ -1236,9 +1249,9 @@ TEST_CASE("compactcas.gc.compact")
 			CHECK(Cas.HaveChunk(ChunkHashes[7]));
 			CHECK(Cas.HaveChunk(ChunkHashes[8]));
 
-			CHECK(ChunkHashes[6] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[6])));
-			CHECK(ChunkHashes[7] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[7])));
-			CHECK(ChunkHashes[8] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[8])));
+			ValidateChunkExists(6);
+			ValidateChunkExists(7);
+			ValidateChunkExists(8);
 
 			Cas.InsertChunk(Chunks[0], ChunkHashes[0]);
 			Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
@@ -1273,11 +1286,11 @@ TEST_CASE("compactcas.gc.compact")
 			CHECK(!Cas.HaveChunk(ChunkHashes[7]));
 			CHECK(Cas.HaveChunk(ChunkHashes[8]));
 
-			CHECK(ChunkHashes[0] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[0])));
-			CHECK(ChunkHashes[2] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[2])));
-			CHECK(ChunkHashes[4] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[4])));
-			CHECK(ChunkHashes[6] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[6])));
-			CHECK(ChunkHashes[8] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[8])));
+			ValidateChunkExists(0);
+			ValidateChunkExists(2);
+			ValidateChunkExists(4);
+			ValidateChunkExists(6);
+			ValidateChunkExists(8);
 
 			Cas.InsertChunk(Chunks[1], ChunkHashes[1]);
 			Cas.InsertChunk(Chunks[3], ChunkHashes[3]);
@@ -1286,15 +1299,15 @@ TEST_CASE("compactcas.gc.compact")
 		}
 
 		// Verify that we nicely appended blocks even after all GC operations
-		CHECK(ChunkHashes[0] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[0])));
-		CHECK(ChunkHashes[1] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[1])));
-		CHECK(ChunkHashes[2] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[2])));
-		CHECK(ChunkHashes[3] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[3])));
-		CHECK(ChunkHashes[4] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[4])));
-		CHECK(ChunkHashes[5] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[5])));
-		CHECK(ChunkHashes[6] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[6])));
-		CHECK(ChunkHashes[7] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[7])));
-		CHECK(ChunkHashes[8] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[8])));
+		ValidateChunkExists(0);
+		ValidateChunkExists(1);
+		ValidateChunkExists(2);
+		ValidateChunkExists(3);
+		ValidateChunkExists(4);
+		ValidateChunkExists(5);
+		ValidateChunkExists(6);
+		ValidateChunkExists(7);
+		ValidateChunkExists(8);
 	}
 }
 
@@ -1497,6 +1510,7 @@ TEST_CASE("compactcas.threadedinsert")
 				IoBuffer Chunk	= CreateRandomChunk(kChunkSize);
 				IoHash	 Hash	= HashBuffer(Chunk);
 				NewChunks[Hash] = Chunk;
+				GcChunkHashes.insert(Hash);
 			}
 
 			std::atomic_uint32_t AddedChunkCount;
@@ -1522,42 +1536,40 @@ TEST_CASE("compactcas.threadedinsert")
 				});
 			}
 
-			while (AddedChunkCount.load() < NewChunks.size())
+			std::unordered_set<IoHash, IoHash::Hasher> ChunksToDelete;
+			std::vector<IoHash>						   KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end());
+			size_t									   C = 0;
+			while (C < KeepHashes.size())
 			{
-				// Need to be careful since we might GC blocks we don't know outside of RwLock::ExclusiveLockScope
-				for (const auto& Chunk : NewChunks)
+				if (C % 155 == 0)
 				{
-					if (Cas.HaveChunk(Chunk.first))
+					if (C < KeepHashes.size() - 1)
 					{
-						GcChunkHashes.emplace(Chunk.first);
+						ChunksToDelete.insert(KeepHashes[C]);
+						KeepHashes[C] = KeepHashes[KeepHashes.size() - 1];
+						KeepHashes.pop_back();
 					}
-				}
-				std::vector<IoHash> KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end());
-				size_t				C = 0;
-				while (C < KeepHashes.size())
-				{
-					if (C % 155 == 0)
+					if (C + 3 < KeepHashes.size() - 1)
 					{
-						if (C < KeepHashes.size() - 1)
-						{
-							KeepHashes[C] = KeepHashes[KeepHashes.size() - 1];
-							KeepHashes.pop_back();
-						}
-						if (C + 3 < KeepHashes.size() - 1)
-						{
-							KeepHashes[C + 3] = KeepHashes[KeepHashes.size() - 1];
-							KeepHashes.pop_back();
-						}
+						ChunksToDelete.insert(KeepHashes[C + 3]);
+						KeepHashes[C + 3] = KeepHashes[KeepHashes.size() - 1];
+						KeepHashes.pop_back();
 					}
-					C++;
 				}
+				C++;
+			}
 
+			while (AddedChunkCount.load() < NewChunks.size())
+			{
 				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 				GcCtx.CollectSmallObjects(true);
 				GcCtx.AddRetainedCids(KeepHashes);
 				Cas.CollectGarbage(GcCtx);
 				const HashKeySet& Deleted = GcCtx.DeletedCids();
-				Deleted.IterateHashes([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
+				Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
+					CHECK(ChunksToDelete.contains(ChunkHash));
+					GcChunkHashes.erase(ChunkHash);
+				});
 			}
 
 			while (WorkCompleted < NewChunks.size() + Chunks.size())
@@ -1565,40 +1577,15 @@ TEST_CASE("compactcas.threadedinsert")
 				Sleep(1);
 			}
 
-			// Need to be careful since we might GC blocks we don't know outside of RwLock::ExclusiveLockScope
-			for (const auto& Chunk : NewChunks)
-			{
-				if (Cas.HaveChunk(Chunk.first))
-				{
-					GcChunkHashes.emplace(Chunk.first);
-				}
-			}
-			std::vector<IoHash> KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end());
-			size_t				C = 0;
-			while (C < KeepHashes.size())
-			{
-				if (C % 155 == 0)
-				{
-					if (C < KeepHashes.size() - 1)
-					{
-						KeepHashes[C] = KeepHashes[KeepHashes.size() - 1];
-						KeepHashes.pop_back();
-					}
-					if (C + 3 < KeepHashes.size() - 1)
-					{
-						KeepHashes[C + 3] = KeepHashes[KeepHashes.size() - 1];
-						KeepHashes.pop_back();
-					}
-				}
-				C++;
-			}
-
 			GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 			GcCtx.CollectSmallObjects(true);
 			GcCtx.AddRetainedCids(KeepHashes);
 			Cas.CollectGarbage(GcCtx);
 			const HashKeySet& Deleted = GcCtx.DeletedCids();
-			Deleted.IterateHashes([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
+			Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
+				CHECK(ChunksToDelete.contains(ChunkHash));
+				GcChunkHashes.erase(ChunkHash);
+			});
 		}
 		{
 			WorkCompleted = 0;
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 0d742d7e1..fe568a487 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -795,6 +795,21 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, IoBuffer&&
 }
 
 void
+FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, uint64_t Size)>&& Callback)
+{
+	ZEN_TRACE_CPU("FileCas::IterateChunks");
+
+	ZEN_ASSERT(m_IsInitialized);
+
+	RwLock::SharedLockScope _(m_Lock);
+	for (const auto& It : m_Index)
+	{
+		const IoHash& NameHash = It.first;
+		Callback(NameHash, It.second.Size);
+	}
+}
+
+void
 FileCasStrategy::Flush()
 {
 	ZEN_TRACE_CPU("FileCas::Flush");
@@ -927,7 +942,7 @@ FileCasStrategy::CollectGarbage(GcContext& GcCtx)
 
 	{
 		ZEN_TRACE_CPU("FileCas::CollectGarbage::Filter");
-		IterateChunks([&](const IoHash& Hash, IoBuffer&& Payload) {
+		IterateChunks([&](const IoHash& Hash, uint64_t Size) {
 			bool KeepThis	= false;
 			CandidateCas[0] = Hash;
 			GcCtx.FilterCids(CandidateCas, [&](const IoHash& Hash) {
@@ -935,16 +950,14 @@ FileCasStrategy::CollectGarbage(GcContext& GcCtx)
 				KeepThis = true;
 			});
 
-			const uint64_t FileSize = Payload.GetSize();
-
 			if (!KeepThis)
 			{
 				ChunksToDelete.push_back(Hash);
-				ChunksToDeleteBytes.fetch_add(FileSize);
+				ChunksToDeleteBytes.fetch_add(Size);
 			}
 
 			++ChunkCount;
-			ChunkBytes.fetch_add(FileSize);
+			ChunkBytes.fetch_add(Size);
 		});
 	}
 
diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h
index d9186f05b..10c181c0b 100644
--- a/src/zenstore/filecas.h
+++ b/src/zenstore/filecas.h
@@ -86,6 +86,7 @@ private:
 
 	inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardLocks[Hash.Hash[19]]; }
 	void		   IterateChunks(std::function<void(const IoHash& Hash, IoBuffer&& Payload)>&& Callback);
+	void		   IterateChunks(std::function<void(const IoHash& Hash, uint64_t PayloadSize)>&& Callback);
 	void		   DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec);
 
 	struct ShardingHelper
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index edd6df5a2..6fc0652f2 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -139,7 +139,7 @@ public:
 	void WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, const WriteChunkCallback& Callback);
 
 	IoBuffer TryGetChunk(const BlockStoreLocation& Location) const;
-	void	 Flush();
+	void	 Flush(bool ForceNewBlock);
 
 	ReclaimSnapshotState GetReclaimSnapshotState();
 	void				 ReclaimSpace(