missing chunks bugfix (#424)

* make sure to close log file when resetting log
* drop entries that refers to missing blocks
* Don't scrub keys that has been rewritten
* currectly count added bytes / m_TotalSize
* fix negative sleep time in BlockStoreFile::Open()
* be defensive when fetching log position
* append to log files *after* we updated all state successfully
* explicitly close stuff in destructors with exception catching
* clean up empty size block store files

1 files changed, 70 insertions, 28 deletions

diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp
index afb7e4bee..c25f762f5 100644
--- a/src/zenstore/buildstore/buildstore.cpp
+++ b/src/zenstore/buildstore/buildstore.cpp
@@ -177,22 +177,60 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc)
 									m_Config.SmallBlobBlockStoreAlignement,
 									IsNew);
 		m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20);
+
+		BlockStore::BlockIndexSet KnownBlocks;
+		for (const BlobEntry& Blob : m_BlobEntries)
 		{
-			BlockStore::BlockIndexSet KnownBlocks;
-			for (const BlobEntry& Blob : m_BlobEntries)
+			if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex)
 			{
-				if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex)
-				{
-					const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex];
-					KnownBlocks.insert(Metadata.Location.BlockIndex);
-				}
+				const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex];
+				KnownBlocks.insert(Metadata.Location.BlockIndex);
 			}
-			m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
 		}
+		BlockStore::BlockIndexSet MissingBlocks = m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
 
 		m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite);
 		m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite);
 
+		if (!MissingBlocks.empty())
+		{
+			std::vector<MetadataDiskEntry> MissingMetadatas;
+			for (auto& It : m_BlobLookup)
+			{
+				const IoHash&	 BlobHash	   = It.first;
+				const BlobIndex	 ReadBlobIndex = It.second;
+				const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+				if (ReadBlobEntry.Metadata)
+				{
+					const MetadataEntry& MetaData = m_MetadataEntries[ReadBlobEntry.Metadata];
+					if (MissingBlocks.contains(MetaData.Location.BlockIndex))
+					{
+						MissingMetadatas.push_back(
+							MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = BlobHash});
+						MissingMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone;
+						m_MetadataEntries[ReadBlobEntry.Metadata] = {};
+						m_BlobEntries[ReadBlobIndex].Metadata	  = {};
+					}
+				}
+			}
+			ZEN_ASSERT(!MissingMetadatas.empty());
+
+			for (const MetadataDiskEntry& Entry : MissingMetadatas)
+			{
+				auto It = m_BlobLookup.find(Entry.BlobHash);
+				ZEN_ASSERT(It != m_BlobLookup.end());
+
+				const BlobIndex	 ReadBlobIndex = It->second;
+				const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+				if (!ReadBlobEntry.Payload)
+				{
+					m_BlobLookup.erase(It);
+				}
+			}
+			m_MetadatalogFile.Append(MissingMetadatas);
+			CompactState();
+		}
+
 		m_Gc.AddGcReferencer(*this);
 		m_Gc.AddGcReferenceLocker(*this);
 		m_Gc.AddGcStorage(this);
@@ -256,34 +294,36 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload)
 		ZEN_UNUSED(Result);
 		Entry = PayloadEntry(0, PayloadSize);
 	}
-	m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
 
-	RwLock::ExclusiveLockScope _(m_Lock);
-	if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
 	{
-		const BlobIndex ExistingBlobIndex = It->second;
-		BlobEntry&		Blob			  = m_BlobEntries[ExistingBlobIndex];
-		if (Blob.Payload)
+		RwLock::ExclusiveLockScope _(m_Lock);
+		if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
 		{
-			m_PayloadEntries[Blob.Payload] = Entry;
+			const BlobIndex ExistingBlobIndex = It->second;
+			BlobEntry&		Blob			  = m_BlobEntries[ExistingBlobIndex];
+			if (Blob.Payload)
+			{
+				m_PayloadEntries[Blob.Payload] = Entry;
+			}
+			else
+			{
+				Blob.Payload = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
+				m_PayloadEntries.push_back(Entry);
+			}
+			Blob.LastAccessTime = GcClock::TickCount();
 		}
 		else
 		{
-			Blob.Payload = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
+			PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
 			m_PayloadEntries.push_back(Entry);
-		}
-		Blob.LastAccessTime = GcClock::TickCount();
-	}
-	else
-	{
-		PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
-		m_PayloadEntries.push_back(Entry);
 
-		const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
-		// we only remove during GC and compact this then...
-		m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
-		m_BlobLookup.insert({BlobHash, NewBlobIndex});
+			const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
+			// we only remove during GC and compact this then...
+			m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
+			m_BlobLookup.insert({BlobHash, NewBlobIndex});
+		}
 	}
+	m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
 	m_LastAccessTimeUpdateCount++;
 }
 
@@ -370,7 +410,6 @@ BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoB
 			const BlockStoreLocation& Location = Locations[LocationIndex];
 
 			MetadataEntry Entry = {.Location = Location, .ContentType = Data.GetContentType(), .Flags = 0};
-			m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
 
 			if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
 			{
@@ -396,6 +435,9 @@ BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoB
 				m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
 				m_BlobLookup.insert({BlobHash, NewBlobIndex});
 			}
+
+			m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
+
 			m_LastAccessTimeUpdateCount++;
 			WriteBlobIndex++;
 			if (m_TrackedCacheKeys)