improved block store logging and more gcv2 tests (#659)

* improved gc/blockstore logging
* more gcv2 tests

7 files changed, 215 insertions, 114 deletions

diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h
index f5526945a..215fb71a8 100644
--- a/src/zencore/include/zencore/testutils.h
+++ b/src/zencore/include/zencore/testutils.h
@@ -34,6 +34,15 @@ struct ScopedCurrentDirectoryChange
 
 IoBuffer CreateRandomBlob(uint64_t Size);
 
+struct FalseType
+{
+	static const bool Enabled = false;
+};
+struct TrueType
+{
+	static const bool Enabled = true;
+};
+
 }  // namespace zen
 
 #endif	// ZEN_WITH_TESTS
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 5bcb7f5b4..830ad9e55 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -1069,7 +1069,8 @@ void
 BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 						  uint32_t						  PayloadAlignment,
 						  const CompactCallback&		  ChangeCallback,
-						  const ClaimDiskReserveCallback& DiskReserveCallback)
+						  const ClaimDiskReserveCallback& DiskReserveCallback,
+						  std::string_view				  LogPrefix)
 {
 	ZEN_TRACE_CPU("BlockStore::CompactBlocks");
 
@@ -1079,7 +1080,8 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 
 	Stopwatch  TotalTimer;
 	const auto _ = MakeGuard([&] {
-		ZEN_DEBUG("Compact blocks for '{}' DONE after {}, deleted {} and moved {} chunks ({}) ",
+		ZEN_DEBUG("{}Compact blocks for '{}' DONE after {}, deleted {} and moved {} chunks ({}) ",
+				  LogPrefix,
 				  m_BlocksBasePath,
 				  NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()),
 				  NiceBytes(DeletedSize),
@@ -1107,7 +1109,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 			   }
 			   if (NewBlockFile->IsOpen())
 			   {
-				   ZEN_DEBUG("Dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
+				   ZEN_DEBUG("{}Dropping incomplete cas block store file '{}'", LogPrefix, NewBlockFile->GetPath());
 				   NewBlockFile->MarkAsDeleteOnClose();
 			   }
 		   }
@@ -1138,7 +1140,8 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 			RwLock::SharedLockScope _(m_InsertLock);
 			if ((BlockIndex == m_WriteBlockIndex.load()) && m_WriteBlock)
 			{
-				ZEN_ERROR("Compact Block was requested to rewrite the currently active write block in '{}', Block index {}",
+				ZEN_ERROR("{}compact Block was requested to rewrite the currently active write block in '{}', Block index {}",
+						  LogPrefix,
 						  m_BlocksBasePath,
 						  BlockIndex);
 				return false;
@@ -1146,12 +1149,18 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 			auto It = m_ChunkBlocks.find(BlockIndex);
 			if (It == m_ChunkBlocks.end())
 			{
-				ZEN_WARN("Compact Block was requested to rewrite an unknown block in '{}', Block index {}", m_BlocksBasePath, BlockIndex);
+				ZEN_WARN("{}compact Block was requested to rewrite an unknown block in '{}', Block index {}",
+						 LogPrefix,
+						 m_BlocksBasePath,
+						 BlockIndex);
 				return true;
 			}
 			if (!It->second)
 			{
-				ZEN_WARN("Compact Block was requested to rewrite a deleted block in '{}', Block index {}", m_BlocksBasePath, BlockIndex);
+				ZEN_WARN("{}compact Block was requested to rewrite a deleted block in '{}', Block index {}",
+						 LogPrefix,
+						 m_BlocksBasePath,
+						 BlockIndex);
 				return true;
 			}
 			OldBlockFile = It->second;
@@ -1162,10 +1171,14 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 
 		if (KeepChunkIndexes.empty())
 		{
-			ZEN_INFO("Dropping all chunks from '{}'", GetBlockPath(m_BlocksBasePath, BlockIndex));
+			ZEN_INFO("{}dropped all chunks from '{}', freeing {}",
+					 LogPrefix,
+					 GetBlockPath(m_BlocksBasePath, BlockIndex).filename(),
+					 NiceBytes(OldBlockSize));
 		}
 		else
 		{
+			uint64_t			 MovedFromBlock = 0;
 			std::vector<uint8_t> Chunk;
 			for (const size_t& ChunkIndex : KeepChunkIndexes)
 			{
@@ -1173,8 +1186,9 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 				if (ChunkLocation.Offset + ChunkLocation.Size > OldBlockSize)
 				{
 					ZEN_WARN(
-						"Compact Block skipping chunk outside of block range in '{}', Chunk start {}, Chunk size {} in Block {}, Block "
+						"{}compact Block skipping chunk outside of block range in '{}', Chunk start {}, Chunk size {} in Block {}, Block "
 						"size {}",
+						LogPrefix,
 						m_BlocksBasePath,
 						ChunkLocation.Offset,
 						ChunkLocation.Size,
@@ -1192,11 +1206,17 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 					{
 						ZEN_ASSERT_SLOW(NewBlockFile->IsOpen());
 						NewBlockFile->Flush();
-						MovedSize += NewBlockFile->FileSize();
+						uint64_t NewBlockSize = NewBlockFile->FileSize();
+						MovedSize += NewBlockSize;
 						NewBlockFile = nullptr;
 
 						ZEN_ASSERT(!MovedChunks.empty() || RemovedSize > 0);  // We should not have a new block if we haven't moved anything
 
+						ZEN_INFO("{}wrote block {} ({})",
+								 LogPrefix,
+								 GetBlockPath(m_BlocksBasePath, NewBlockIndex).filename(),
+								 NiceBytes(NewBlockSize));
+
 						if (!ReportChanges())
 						{
 							return false;
@@ -1210,7 +1230,8 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 						NextBlockIndex = GetFreeBlockIndex(NextBlockIndex, InsertLock, NewBlockPath);
 						if (NextBlockIndex == (uint32_t)m_MaxBlockCount)
 						{
-							ZEN_ERROR("unable to allocate a new block in '{}', count limit {} exeeded",
+							ZEN_ERROR("{}unable to allocate a new block in '{}', count limit {} exeeded",
+									  LogPrefix,
 									  m_BlocksBasePath,
 									  static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
 							return false;
@@ -1224,7 +1245,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 					DiskSpace		Space = DiskSpaceInfo(m_BlocksBasePath, Error);
 					if (Error)
 					{
-						ZEN_ERROR("get disk space in '{}' FAILED, reason: '{}'", m_BlocksBasePath, Error.message());
+						ZEN_ERROR("{}get disk space in '{}' FAILED, reason: '{}'", LogPrefix, m_BlocksBasePath, Error.message());
 						{
 							RwLock::ExclusiveLockScope _l(m_InsertLock);
 							ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
@@ -1240,7 +1261,8 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 						uint64_t ReclaimedSpace = DiskReserveCallback();
 						if (Space.Free + ReclaimedSpace < m_MaxBlockSize)
 						{
-							ZEN_WARN("garbage collect for '{}' FAILED, required disk space {}, free {}",
+							ZEN_WARN("{}garbage collect for '{}' FAILED, required disk space {}, free {}",
+									 LogPrefix,
 									 m_BlocksBasePath,
 									 m_MaxBlockSize,
 									 NiceBytes(Space.Free + ReclaimedSpace));
@@ -1254,7 +1276,8 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 							return false;
 						}
 
-						ZEN_INFO("using gc reserve for '{}', reclaimed {}, disk free {}",
+						ZEN_INFO("{}using gc reserve for '{}', reclaimed {}, disk free {}",
+								 LogPrefix,
 								 m_BlocksBasePath,
 								 ReclaimedSpace,
 								 NiceBytes(Space.Free + ReclaimedSpace));
@@ -1267,9 +1290,17 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 				NewBlockFile->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
 				MovedChunks.push_back(
 					{ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
-				WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
+				uint64_t WriteEndOffset = WriteOffset + ChunkLocation.Size;
+				MovedFromBlock += (WriteEndOffset - WriteOffset);
+				WriteOffset = RoundUp(WriteEndOffset, PayloadAlignment);
 				AddedSize += Chunk.size();
 			}
+			ZEN_INFO("{}moved {} chunks ({}) from '{}' to new block, freeing {}",
+					 LogPrefix,
+					 KeepChunkIndexes.size(),
+					 NiceBytes(MovedFromBlock),
+					 GetBlockPath(m_BlocksBasePath, BlockIndex).filename(),
+					 NiceBytes(OldBlockSize - MovedFromBlock));
 		}
 
 		if (!ReportChanges())
@@ -1279,7 +1310,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 
 		{
 			RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
-			ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+			ZEN_DEBUG("{}marking cas block store file '{}' for delete ({})",
+					  LogPrefix,
+					  OldBlockFile->GetPath().filename(),
+					  NiceBytes(OldBlockSize));
 			OldBlockFile->MarkAsDeleteOnClose();
 			m_ChunkBlocks.erase(BlockIndex);
 			m_TotalSize.fetch_sub(OldBlockSize);
@@ -1292,8 +1326,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 	{
 		ZEN_ASSERT_SLOW(NewBlockFile->IsOpen());
 		NewBlockFile->Flush();
-		MovedSize += NewBlockFile->FileSize();
+		uint64_t NewBlockSize = NewBlockFile->FileSize();
+		MovedSize += NewBlockSize;
 		NewBlockFile = nullptr;
+		ZEN_INFO("{}wrote block {} ({})", LogPrefix, GetBlockPath(m_BlocksBasePath, NewBlockIndex).filename(), NiceBytes(NewBlockSize));
 	}
 
 	ReportChanges();
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 7274626ee..1ffc959e7 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -2868,7 +2868,8 @@ public:
 								}
 								return true;
 							},
-							ClaimDiskReserveCallback);
+							ClaimDiskReserveCallback,
+							fmt::format("GCV2: cachebucket [COMPACT] '{}': ", m_Bucket.m_BucketDir));
 					}
 					else
 					{
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index 32a2a2aa3..7ef290cf5 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -871,15 +871,6 @@ namespace testutils {
 		return {Key, Buffer};
 	}
 
-	struct FalseType
-	{
-		static const bool Enabled = false;
-	};
-	struct TrueType
-	{
-		static const bool Enabled = true;
-	};
-
 }  // namespace testutils
 
 TEST_CASE("z$.store")
@@ -1217,7 +1208,7 @@ TEST_CASE("z$.gc")
 	}
 }
 
-TEST_CASE_TEMPLATE("z$.threadedinsert", GCV2, testutils::FalseType, testutils::TrueType)  // * doctest::skip(true))
+TEST_CASE_TEMPLATE("z$.threadedinsert", GCV2, FalseType, TrueType)	// * doctest::skip(true))
 {
 	//	for (uint32_t i = 0; i < 100; ++i)
 	{
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 64c1dadf8..b308ddd4f 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -565,7 +565,7 @@ public:
 				return;
 			}
 			ZEN_INFO("GCV2: compactcas [COMPACT] '{}': RemovedDisk: {} in {}",
-					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+					 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 					 NiceBytes(Stats.RemovedDisk),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
@@ -629,7 +629,7 @@ public:
 						if (Ctx.Settings.Verbose)
 						{
 							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': compacting {} blocks",
-									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 									 BlocksToCompact.size());
 						}
 
@@ -669,14 +669,16 @@ public:
 								}
 								return true;
 							},
-							ClaimDiskReserveCallback);
+							ClaimDiskReserveCallback,
+							fmt::format("GCV2: compactcas [COMPACT] '{}': ",
+										m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName));
 					}
 					else
 					{
 						if (Ctx.Settings.Verbose)
 						{
 							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': skipped compacting of {} eligible blocks",
-									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 									 BlocksToCompact.size());
 						}
 					}
@@ -708,7 +710,7 @@ public:
 				return;
 			}
 			ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Checked: {}, Deleted: {}, FreedMemory: {} in {}",
-					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+					 m_CasContainerStrategy.m_RootDirectory.filename() / m_CasContainerStrategy.m_ContainerBaseName,
 					 Stats.CheckedCount,
 					 Stats.DeletedCount,
 					 NiceBytes(Stats.FreedMemory),
@@ -725,41 +727,43 @@ public:
 			return nullptr;
 		}
 
-		std::vector<CasDiskIndexEntry> ExpiredEntries;
-		ExpiredEntries.reserve(UnusedCids.size());
+		if (!Ctx.Settings.CollectSmallObjects)
+		{
+			return nullptr;
+		}
 
+		if (Ctx.Settings.IsDeleteMode)
 		{
-			RwLock::ExclusiveLockScope __(m_CasContainerStrategy.m_LocationMapLock);
-			if (Ctx.IsCancelledFlag.load())
-			{
-				return nullptr;
-			}
+			std::vector<CasDiskIndexEntry> ExpiredEntries;
+			ExpiredEntries.reserve(UnusedCids.size());
 
-			for (const IoHash& Cid : UnusedCids)
 			{
-				auto It = m_CasContainerStrategy.m_LocationMap.find(Cid);
-				if (It == m_CasContainerStrategy.m_LocationMap.end())
+				RwLock::ExclusiveLockScope __(m_CasContainerStrategy.m_LocationMapLock);
+				if (Ctx.IsCancelledFlag.load())
 				{
-					continue;
+					return nullptr;
 				}
-				if (Ctx.Settings.CollectSmallObjects)
+
+				for (const IoHash& Cid : UnusedCids)
 				{
-					CasDiskIndexEntry ExpiredEntry = {.Key		= Cid,
-													  .Location = m_CasContainerStrategy.m_Locations[It->second],
-													  .Flags	= CasDiskIndexEntry::kTombstone};
-					ExpiredEntries.push_back(ExpiredEntry);
+					if (auto It = m_CasContainerStrategy.m_LocationMap.find(Cid); It != m_CasContainerStrategy.m_LocationMap.end())
+					{
+						ExpiredEntries.push_back({.Key		= Cid,
+												  .Location = m_CasContainerStrategy.m_Locations[It->second],
+												  .Flags	= CasDiskIndexEntry::kTombstone});
+					}
 				}
-			}
 
-			if (Ctx.Settings.IsDeleteMode)
-			{
-				for (const CasDiskIndexEntry& Entry : ExpiredEntries)
+				if (!ExpiredEntries.empty())
 				{
-					m_CasContainerStrategy.m_LocationMap.erase(Entry.Key);
-					Stats.DeletedCount++;
+					for (const CasDiskIndexEntry& Entry : ExpiredEntries)
+					{
+						m_CasContainerStrategy.m_LocationMap.erase(Entry.Key);
+						Stats.DeletedCount++;
+					}
+					m_CasContainerStrategy.m_CasLog.Append(ExpiredEntries);
+					m_CasContainerStrategy.m_CasLog.Flush();
 				}
-				m_CasContainerStrategy.m_CasLog.Append(ExpiredEntries);
-				m_CasContainerStrategy.m_CasLog.Flush();
 			}
 		}
 
@@ -1695,9 +1699,9 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
 	CHECK(ChunkHashes[5] == IoHash::HashBuffer(RetainChunk));
 }
 
-TEST_CASE("compactcas.threadedinsert")
+TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
 {
-	//	for (uint32_t i = 0; i < 100; ++i)
+	// for (uint32_t i = 0; i < 100; ++i)
 	{
 		ScopedTemporaryDirectory TempDir;
 
@@ -1812,7 +1816,8 @@ TEST_CASE("compactcas.threadedinsert")
 
 			std::unordered_set<IoHash, IoHash::Hasher> ChunksToDelete;
 			std::vector<IoHash>						   KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end());
-			size_t									   C = 0;
+
+			size_t C = 0;
 			while (C < KeepHashes.size())
 			{
 				if (C % 155 == 0)
@@ -1833,17 +1838,66 @@ TEST_CASE("compactcas.threadedinsert")
 				C++;
 			}
 
+			auto DoGC = [](CasContainerStrategy&							 Cas,
+						   const std::unordered_set<IoHash, IoHash::Hasher>& ChunksToDelete,
+						   const std::vector<IoHash>&						 KeepHashes,
+						   std::unordered_set<IoHash, IoHash::Hasher>&		 GcChunkHashes) {
+				if (GCV2::Enabled)
+				{
+					std::atomic_bool	  IsCancelledFlag = false;
+					GcCtx				  Ctx			  = {.Settings		  = {.CacheExpireTime					= GcClock::Now() - std::chrono::hours(24),
+																				 .ProjectStoreExpireTime			= GcClock::Now() - std::chrono::hours(24),
+																				 .CollectSmallObjects				= true,
+																				 .IsDeleteMode						= true,
+																				 .CompactBlockUsageThresholdPercent = 100},
+															 .IsCancelledFlag = IsCancelledFlag};
+					GcReferenceStoreStats PrunerStats;
+					GcReferencePruner*	  Pruner = Cas.CreateReferencePruner(Ctx, PrunerStats);
+					CHECK(Pruner);
+					HashKeySet		  Deleted;
+					GcStats			  Stats;
+					GcStoreCompactor* Compactor =
+						Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::vector<IoHash> {
+							std::vector<IoHash> Unreferenced;
+							HashKeySet			Retain;
+							Retain.AddHashesToSet(KeepHashes);
+							for (const IoHash& ChunkHash : References)
+							{
+								if (!Retain.ContainsHash(ChunkHash))
+								{
+									Unreferenced.push_back(ChunkHash);
+								}
+							}
+							Deleted.AddHashesToSet(Unreferenced);
+							return Unreferenced;
+						});
+					if (Compactor)
+					{
+						Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
+							CHECK(ChunksToDelete.contains(ChunkHash));
+							GcChunkHashes.erase(ChunkHash);
+						});
+						GcCompactStoreStats CompactStats;
+						Compactor->CompactStore(Ctx, CompactStats, []() { return 0; });
+					}
+				}
+				else
+				{
+					GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
+					GcCtx.CollectSmallObjects(true);
+					GcCtx.AddRetainedCids(KeepHashes);
+					Cas.CollectGarbage(GcCtx);
+					const HashKeySet& Deleted = GcCtx.DeletedCids();
+					Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
+						CHECK(ChunksToDelete.contains(ChunkHash));
+						GcChunkHashes.erase(ChunkHash);
+					});
+				}
+			};
+
 			while (AddedChunkCount.load() < NewChunks.size())
 			{
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-				GcCtx.CollectSmallObjects(true);
-				GcCtx.AddRetainedCids(KeepHashes);
-				Cas.CollectGarbage(GcCtx);
-				const HashKeySet& Deleted = GcCtx.DeletedCids();
-				Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
-					CHECK(ChunksToDelete.contains(ChunkHash));
-					GcChunkHashes.erase(ChunkHash);
-				});
+				DoGC(Cas, ChunksToDelete, KeepHashes, GcChunkHashes);
 			}
 
 			while (WorkCompleted < NewChunks.size() + Chunks.size())
@@ -1851,15 +1905,7 @@ TEST_CASE("compactcas.threadedinsert")
 				Sleep(1);
 			}
 
-			GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-			GcCtx.CollectSmallObjects(true);
-			GcCtx.AddRetainedCids(KeepHashes);
-			Cas.CollectGarbage(GcCtx);
-			const HashKeySet& Deleted = GcCtx.DeletedCids();
-			Deleted.IterateHashes([&GcChunkHashes, &ChunksToDelete](const IoHash& ChunkHash) {
-				CHECK(ChunksToDelete.contains(ChunkHash));
-				GcChunkHashes.erase(ChunkHash);
-			});
+			DoGC(Cas, ChunksToDelete, KeepHashes, GcChunkHashes);
 		}
 		{
 			WorkCompleted = 0;
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 7839c7132..7aa180b68 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1436,7 +1436,10 @@ public:
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		ZEN_INFO("GCV2: filecas [COMPACT] '{}': Removing {} files", m_FileCasStrategy.m_RootDirectory, m_ReferencesToClean.size());
+		ZEN_INFO("GCV2: filecas [COMPACT] '{}': {} {} files",
+				 m_FileCasStrategy.m_RootDirectory,
+				 Ctx.Settings.IsDeleteMode ? "Removing" : "Checking",
+				 m_ReferencesToClean.size());
 
 		size_t Skipped = 0;
 		for (const IoHash& ChunkHash : m_ReferencesToClean)
@@ -1456,9 +1459,12 @@ public:
 
 				if (Ctx.Settings.IsDeleteMode)
 				{
-					ZEN_DEBUG("GCV2: filecas [COMPACT] '{}': Deleting CAS payload file '{}'",
-							  m_FileCasStrategy.m_RootDirectory,
-							  Name.ShardedPath.ToUtf8());
+					if (Ctx.Settings.Verbose)
+					{
+						ZEN_INFO("GCV2: filecas [COMPACT] '{}': Deleting CAS payload file '{}'",
+								 m_FileCasStrategy.m_RootDirectory,
+								 Name.ShardedPath.ToUtf8());
+					}
 					std::error_code Ec;
 					uint64_t		SizeOnDisk = std::filesystem::file_size(Name.ShardedPath.c_str(), Ec);
 					if (Ec)
@@ -1486,10 +1492,13 @@ public:
 					bool			Existed = std::filesystem::is_regular_file(Name.ShardedPath.c_str(), Ec);
 					if (Ec)
 					{
-						ZEN_WARN("GCV2: filecas [COMPACT] '{}': Failed checking CAS payload file '{}'. Reason '{}'",
-								 m_FileCasStrategy.m_RootDirectory,
-								 Name.ShardedPath.ToUtf8(),
-								 Ec.message());
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: filecas [COMPACT] '{}': Failed checking CAS payload file '{}'. Reason '{}'",
+									 m_FileCasStrategy.m_RootDirectory,
+									 Name.ShardedPath.ToUtf8(),
+									 Ec.message());
+						}
 						continue;
 					}
 					if (!Existed)
@@ -1503,7 +1512,10 @@ public:
 
 		if (Skipped > 0)
 		{
-			ZEN_DEBUG("GCV2: filecas [COMPACT] '{}': Skipped deleting of {} eligible files", m_FileCasStrategy.m_RootDirectory, Skipped);
+			if (Ctx.Settings.Verbose)
+			{
+				ZEN_INFO("GCV2: filecas [COMPACT] '{}': Skipped deleting of {} eligible files", m_FileCasStrategy.m_RootDirectory, Skipped);
+			}
 		}
 
 		Reset(m_ReferencesToClean);
@@ -1539,46 +1551,51 @@ public:
 
 		std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
 		Stats.CheckedCount			   = m_Cids.size();
+		Stats.FoundCount			   = UnusedCids.size();
 		if (UnusedCids.empty())
 		{
 			// Nothing to collect
 			return nullptr;
 		}
 
-		Stats.FoundCount += UnusedCids.size();
-
-		if (!Ctx.Settings.IsDeleteMode)
-		{
-			return nullptr;
-		}
-
-		std::vector<IoHash> PrunedReferences;
-		PrunedReferences.reserve(UnusedCids.size());
+		if (Ctx.Settings.IsDeleteMode)
 		{
-			RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock);
-			for (const IoHash& ChunkHash : UnusedCids)
+			std::vector<FileCasStrategy::FileCasIndexEntry> ExpiredEntries;
+			ExpiredEntries.reserve(UnusedCids.size());
 			{
-				auto It = m_FileCasStrategy.m_Index.find(ChunkHash);
-				if (It == m_FileCasStrategy.m_Index.end())
+				RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock);
+				if (Ctx.IsCancelledFlag.load())
 				{
-					continue;
+					return nullptr;
 				}
-				uint64_t FileSize = It->second.Size;
-				m_FileCasStrategy.m_Index.erase(It);
-				m_FileCasStrategy.m_CasLog.Append(
-					{.Key = ChunkHash, .Flags = FileCasStrategy::FileCasIndexEntry::kTombStone, .Size = FileSize});
-				m_FileCasStrategy.m_TotalSize.fetch_sub(It->second.Size, std::memory_order_relaxed);
-				PrunedReferences.push_back(ChunkHash);
-				Stats.DeletedCount++;
-			}
-		}
 
-		if (PrunedReferences.empty())
-		{
-			return nullptr;
+				uint64_t RemovedSize = 0;
+				for (const IoHash& ChunkHash : UnusedCids)
+				{
+					if (auto It = m_FileCasStrategy.m_Index.find(ChunkHash); It != m_FileCasStrategy.m_Index.end())
+					{
+						uint64_t FileSize = It->second.Size;
+						ExpiredEntries.push_back(
+							{.Key = ChunkHash, .Flags = FileCasStrategy::FileCasIndexEntry::kTombStone, .Size = FileSize});
+						RemovedSize += FileSize;
+					}
+				}
+				m_FileCasStrategy.m_TotalSize.fetch_sub(RemovedSize, std::memory_order_relaxed);
+
+				if (!ExpiredEntries.empty())
+				{
+					for (const FileCasStrategy::FileCasIndexEntry& Entry : ExpiredEntries)
+					{
+						m_FileCasStrategy.m_Index.erase(Entry.Key);
+						Stats.DeletedCount++;
+					}
+					m_FileCasStrategy.m_CasLog.Append(ExpiredEntries);
+					m_FileCasStrategy.m_CasLog.Flush();
+				}
+			}
 		}
 
-		return new FileCasStoreCompactor(m_FileCasStrategy, std::move(PrunedReferences));
+		return new FileCasStoreCompactor(m_FileCasStrategy, std::move(UnusedCids));
 	}
 
 private:
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index bb36cb3cd..656040936 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -182,7 +182,8 @@ public:
 		const BlockStoreCompactState&	CompactState,
 		uint32_t						PayloadAlignment,
 		const CompactCallback&			ChangeCallback		= [](const MovedChunksArray&, uint64_t) { return true; },
-		const ClaimDiskReserveCallback& DiskReserveCallback = []() { return 0; });
+		const ClaimDiskReserveCallback& DiskReserveCallback = []() { return 0; },
+		std::string_view				LogPrefix			= {});
 
 	static const char*			 GetBlockFileExtension();
 	static std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex);