improved block store logging and more gcv2 tests (#659)

* improved gc/blockstore logging
* more gcv2 tests

1 files changed, 53 insertions, 36 deletions

diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 7839c7132..7aa180b68 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1436,7 +1436,10 @@ public:
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		ZEN_INFO("GCV2: filecas [COMPACT] '{}': Removing {} files", m_FileCasStrategy.m_RootDirectory, m_ReferencesToClean.size());
+		ZEN_INFO("GCV2: filecas [COMPACT] '{}': {} {} files",
+				 m_FileCasStrategy.m_RootDirectory,
+				 Ctx.Settings.IsDeleteMode ? "Removing" : "Checking",
+				 m_ReferencesToClean.size());
 
 		size_t Skipped = 0;
 		for (const IoHash& ChunkHash : m_ReferencesToClean)
@@ -1456,9 +1459,12 @@ public:
 
 				if (Ctx.Settings.IsDeleteMode)
 				{
-					ZEN_DEBUG("GCV2: filecas [COMPACT] '{}': Deleting CAS payload file '{}'",
-							  m_FileCasStrategy.m_RootDirectory,
-							  Name.ShardedPath.ToUtf8());
+					if (Ctx.Settings.Verbose)
+					{
+						ZEN_INFO("GCV2: filecas [COMPACT] '{}': Deleting CAS payload file '{}'",
+								 m_FileCasStrategy.m_RootDirectory,
+								 Name.ShardedPath.ToUtf8());
+					}
 					std::error_code Ec;
 					uint64_t		SizeOnDisk = std::filesystem::file_size(Name.ShardedPath.c_str(), Ec);
 					if (Ec)
@@ -1486,10 +1492,13 @@ public:
 					bool			Existed = std::filesystem::is_regular_file(Name.ShardedPath.c_str(), Ec);
 					if (Ec)
 					{
-						ZEN_WARN("GCV2: filecas [COMPACT] '{}': Failed checking CAS payload file '{}'. Reason '{}'",
-								 m_FileCasStrategy.m_RootDirectory,
-								 Name.ShardedPath.ToUtf8(),
-								 Ec.message());
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: filecas [COMPACT] '{}': Failed checking CAS payload file '{}'. Reason '{}'",
+									 m_FileCasStrategy.m_RootDirectory,
+									 Name.ShardedPath.ToUtf8(),
+									 Ec.message());
+						}
 						continue;
 					}
 					if (!Existed)
@@ -1503,7 +1512,10 @@ public:
 
 		if (Skipped > 0)
 		{
-			ZEN_DEBUG("GCV2: filecas [COMPACT] '{}': Skipped deleting of {} eligible files", m_FileCasStrategy.m_RootDirectory, Skipped);
+			if (Ctx.Settings.Verbose)
+			{
+				ZEN_INFO("GCV2: filecas [COMPACT] '{}': Skipped deleting of {} eligible files", m_FileCasStrategy.m_RootDirectory, Skipped);
+			}
 		}
 
 		Reset(m_ReferencesToClean);
@@ -1539,46 +1551,51 @@ public:
 
 		std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
 		Stats.CheckedCount			   = m_Cids.size();
+		Stats.FoundCount			   = UnusedCids.size();
 		if (UnusedCids.empty())
 		{
 			// Nothing to collect
 			return nullptr;
 		}
 
-		Stats.FoundCount += UnusedCids.size();
-
-		if (!Ctx.Settings.IsDeleteMode)
-		{
-			return nullptr;
-		}
-
-		std::vector<IoHash> PrunedReferences;
-		PrunedReferences.reserve(UnusedCids.size());
+		if (Ctx.Settings.IsDeleteMode)
 		{
-			RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock);
-			for (const IoHash& ChunkHash : UnusedCids)
+			std::vector<FileCasStrategy::FileCasIndexEntry> ExpiredEntries;
+			ExpiredEntries.reserve(UnusedCids.size());
 			{
-				auto It = m_FileCasStrategy.m_Index.find(ChunkHash);
-				if (It == m_FileCasStrategy.m_Index.end())
+				RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock);
+				if (Ctx.IsCancelledFlag.load())
 				{
-					continue;
+					return nullptr;
 				}
-				uint64_t FileSize = It->second.Size;
-				m_FileCasStrategy.m_Index.erase(It);
-				m_FileCasStrategy.m_CasLog.Append(
-					{.Key = ChunkHash, .Flags = FileCasStrategy::FileCasIndexEntry::kTombStone, .Size = FileSize});
-				m_FileCasStrategy.m_TotalSize.fetch_sub(It->second.Size, std::memory_order_relaxed);
-				PrunedReferences.push_back(ChunkHash);
-				Stats.DeletedCount++;
-			}
-		}
 
-		if (PrunedReferences.empty())
-		{
-			return nullptr;
+				uint64_t RemovedSize = 0;
+				for (const IoHash& ChunkHash : UnusedCids)
+				{
+					if (auto It = m_FileCasStrategy.m_Index.find(ChunkHash); It != m_FileCasStrategy.m_Index.end())
+					{
+						uint64_t FileSize = It->second.Size;
+						ExpiredEntries.push_back(
+							{.Key = ChunkHash, .Flags = FileCasStrategy::FileCasIndexEntry::kTombStone, .Size = FileSize});
+						RemovedSize += FileSize;
+					}
+				}
+				m_FileCasStrategy.m_TotalSize.fetch_sub(RemovedSize, std::memory_order_relaxed);
+
+				if (!ExpiredEntries.empty())
+				{
+					for (const FileCasStrategy::FileCasIndexEntry& Entry : ExpiredEntries)
+					{
+						m_FileCasStrategy.m_Index.erase(Entry.Key);
+						Stats.DeletedCount++;
+					}
+					m_FileCasStrategy.m_CasLog.Append(ExpiredEntries);
+					m_FileCasStrategy.m_CasLog.Flush();
+				}
+			}
 		}
 
-		return new FileCasStoreCompactor(m_FileCasStrategy, std::move(PrunedReferences));
+		return new FileCasStoreCompactor(m_FileCasStrategy, std::move(UnusedCids));
 	}
 
 private: