meta info store (#75)

- Feature: Added option `--gc-cache-attachment-store` which caches referenced attachments in cache records on disk for faster GC - default is `false`
- Feature: Added option `--gc-projectstore-attachment-store` which caches referenced attachments in project store oplogs on disk for faster GC - default is `false`

4 files changed, 284 insertions, 30 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 70ddcedaf..287a3f7fa 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -90,6 +90,7 @@ BlockStoreFile::Create(uint64_t InitialSize)
 		RetriesLeft--;
 		return true;
 	});
+	RemoveMeta();
 
 	void* FileHandle = m_File.Handle();
 
@@ -99,15 +100,26 @@ BlockStoreFile::Create(uint64_t InitialSize)
 }
 
 uint64_t
-BlockStoreFile::FileSize()
+BlockStoreFile::FileSize() const
 {
-	return m_CachedFileSize == 0 ? m_File.FileSize() : m_CachedFileSize;
+	if (m_CachedFileSize == 0)
+	{
+		std::error_code Ec;
+		uint64_t		Size = m_File.FileSize(Ec);
+		if (Ec)
+		{
+			return 0;
+		}
+		return Size;
+	}
+	return m_CachedFileSize;
 }
 
 void
 BlockStoreFile::MarkAsDeleteOnClose()
 {
 	m_IoBuffer.SetDeleteOnClose(true);
+	RemoveMeta();
 }
 
 IoBuffer
@@ -157,6 +169,90 @@ BlockStoreFile::IsOpen() const
 	return !!m_IoBuffer;
 }
 
+bool
+BlockStoreFile::SetMetaData(const IoBuffer& Payload)
+{
+	if (!Payload)
+	{
+		RemoveMeta();
+		return true;
+	}
+	const std::filesystem::path MetaPath = GetMetaPath();
+	std::error_code				Ec;
+	TemporaryFile::SafeWriteFile(MetaPath, Payload.GetView(), Ec);
+	if (Ec)
+	{
+		ZEN_WARN("Unable to set meta data for block '{}' at meta path: '{}'. Reason: '{}'", m_Path, MetaPath, Ec.message());
+		return false;
+	}
+	return true;
+}
+
+static bool
+IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::path& MetaPath)
+{
+	std::error_code					Ec;
+	std::filesystem::file_time_type MetaWriteTime = std::filesystem::last_write_time(MetaPath, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	std::filesystem::file_time_type BlockWriteTime = std::filesystem::last_write_time(BlockPath, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	if (MetaWriteTime < BlockWriteTime)
+	{
+		std::filesystem::remove(MetaPath, Ec);
+		return false;
+	}
+	return true;
+}
+
+IoBuffer
+BlockStoreFile::GetMetaData() const
+{
+	const std::filesystem::path MetaPath = GetMetaPath();
+	if (IsMetaDataValid(m_Path, MetaPath))
+	{
+		return IoBufferBuilder::MakeFromFile(MetaPath);
+	}
+	return {};
+}
+
+uint64_t
+BlockStoreFile::MetaSize() const
+{
+	const std::filesystem::path MetaPath = GetMetaPath();
+	if (IsMetaDataValid(m_Path, MetaPath))
+	{
+		std::error_code DummyEc;
+		if (uint64_t Size = std::filesystem::file_size(MetaPath, DummyEc); !DummyEc)
+		{
+			return Size;
+		}
+	}
+	return 0;
+}
+
+void
+BlockStoreFile::RemoveMeta()
+{
+	std::filesystem::path MetaPath = GetMetaPath();
+	std::error_code		  DummyEc;
+	std::filesystem::remove(MetaPath, DummyEc);
+}
+
+std::filesystem::path
+BlockStoreFile::GetMetaPath() const
+{
+	std::filesystem::path MetaPath(m_Path);
+	return MetaPath.replace_extension(".meta");
+}
+
+////////////////////////////////////////////////////////
+
 constexpr uint64_t DefaultIterateSmallChunkWindowSize = 2 * 1024 * 1024;
 constexpr uint64_t IterateSmallChunkMaxGapSize		  = 4 * 1024;
 
@@ -215,7 +311,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 					}
 					Ref<BlockStoreFile> BlockFile{new BlockStoreFile(Path)};
 					BlockFile->Open();
-					m_TotalSize.fetch_add(BlockFile->FileSize(), std::memory_order::relaxed);
+					m_TotalSize.fetch_add(BlockFile->TotalSize(), std::memory_order::relaxed);
 					m_ChunkBlocks[BlockIndex] = BlockFile;
 					FoundBlocks[BlockIndex]	  = BlockFile->FileSize();
 					if (BlockIndex >= NextBlockIndex)
@@ -283,7 +379,7 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownLocations)
 		std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex);
 		if (m_ChunkBlocks[BlockIndex])
 		{
-			m_TotalSize.fetch_sub(m_ChunkBlocks[BlockIndex]->FileSize(), std::memory_order::relaxed);
+			m_TotalSize.fetch_sub(m_ChunkBlocks[BlockIndex]->TotalSize(), std::memory_order::relaxed);
 			m_ChunkBlocks[BlockIndex]->MarkAsDeleteOnClose();
 		}
 		m_ChunkBlocks.erase(BlockIndex);
@@ -781,6 +877,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 			   {
 				   ZEN_DEBUG("dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
 				   m_TotalSize.fetch_sub(NewBlockFile->FileSize(), std::memory_order::relaxed);
+				   ZEN_ASSERT_SLOW(NewBlockFile->MetaSize() == 0);
 				   NewBlockFile->MarkAsDeleteOnClose();
 			   }
 		   });
@@ -833,7 +930,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 					ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
 					ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
 					m_ChunkBlocks.erase(BlockIndex);
-					m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
+					m_TotalSize.fetch_sub(OldBlockFile->TotalSize(), std::memory_order::relaxed);
 					OldBlockFile->MarkAsDeleteOnClose();
 				}
 				continue;
@@ -1003,7 +1100,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 				ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
 				ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
 				m_ChunkBlocks.erase(BlockIndex);
-				m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
+				m_TotalSize.fetch_sub(OldBlockFile->TotalSize(), std::memory_order::relaxed);
 				OldBlockFile->MarkAsDeleteOnClose();
 			}
 		}
@@ -1500,9 +1597,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 					  LogPrefix,
 					  OldBlockFile->GetPath().filename(),
 					  NiceBytes(OldBlockSize));
+			m_TotalSize.fetch_sub(OldBlockSize);
+			m_TotalSize.fetch_sub(OldBlockFile->MetaSize());
 			OldBlockFile->MarkAsDeleteOnClose();
 			m_ChunkBlocks.erase(BlockIndex);
-			m_TotalSize.fetch_sub(OldBlockSize);
 			RemovedSize += OldBlockSize;
 		}
 		return true;
@@ -1544,6 +1642,48 @@ BlockStore::GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint
 	return Path.ToPath();
 }
 
+bool
+BlockStore::IsWriting(uint32_t BlockIndex) const
+{
+	RwLock::SharedLockScope _(m_InsertLock);
+	if (std::find(m_ActiveWriteBlocks.begin(), m_ActiveWriteBlocks.end(), BlockIndex) != m_ActiveWriteBlocks.end())
+	{
+		return true;
+	}
+	if (BlockIndex == m_WriteBlockIndex.load() && m_WriteBlock)
+	{
+		return true;
+	}
+	return false;
+}
+
+void
+BlockStore::SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload)
+{
+	RwLock::ExclusiveLockScope _(m_InsertLock);
+	if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end() && It->second)
+	{
+		uint64_t OldMetaSize = It->second->MetaSize();
+		if (It->second->SetMetaData(Payload))
+		{
+			uint64_t NewMetaSize = It->second->MetaSize();
+			m_TotalSize += NewMetaSize;
+			m_TotalSize -= OldMetaSize;
+		}
+	}
+}
+
+IoBuffer
+BlockStore::GetMetaData(uint32_t BlockIndex) const
+{
+	RwLock::SharedLockScope _(m_InsertLock);
+	if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end() && It->second)
+	{
+		return It->second->GetMetaData();
+	}
+	return {};
+}
+
 #if ZEN_WITH_TESTS
 
 TEST_CASE("blockstore.blockstoredisklocation")
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index f85d05dce..940f78c30 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -14,6 +14,7 @@
 #include <zencore/trace.h>
 #include <zencore/workthreadpool.h>
 #include <zencore/xxhash.h>
+#include <zenutil/referencemetadata.h>
 #include <zenutil/workerpools.h>
 
 #include <future>
@@ -109,6 +110,8 @@ namespace {
 
 	static_assert(sizeof(BucketMetaHeader) == 32);
 
+	static constexpr uint32_t BlockMetaDataExpectedMagic = 0x61'74'6d'62;  // 'bmta';
+
 #pragma pack(pop)
 
 	//////////////////////////////////////////////////////////////////////////
@@ -3410,14 +3413,58 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 }
 
 bool
+ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t					 BlockIndex,
+															std::span<const IoHash>		 InlineKeys,
+															std::span<const std::size_t> ChunkIndexes,
+															std::vector<IoHash>&		 OutReferences) const
+{
+	ZEN_TRACE_CPU("Z$::Bucket::ReadAttachmentsFromMetaData");
+	IoBuffer MetaDataPayload = m_BlockStore.GetMetaData(BlockIndex);
+	if (MetaDataPayload)
+	{
+		std::unordered_set<IoHash, IoHash::Hasher> WantedKeys;
+		WantedKeys.reserve(ChunkIndexes.size());
+		for (const size_t ChunkIndex : ChunkIndexes)
+		{
+			WantedKeys.insert(InlineKeys[ChunkIndex]);
+		}
+		ZEN_TRACE_CPU("Z$::Bucket::GetAttachmentsFromMetaData");
+		return GetAttachmentsFromMetaData<IoHash, IoHash>(
+			MetaDataPayload,
+			BlockMetaDataExpectedMagic,
+			[&](std::span<const IoHash> Keys, std::span<const uint32_t> AttachmentCounts, std::span<const IoHash> Attachments) {
+				OutReferences.reserve(OutReferences.capacity() + Attachments.size());
+				auto AttachmentStart = Attachments.begin();
+				for (uint32_t Index = 0; Index < Keys.size(); Index++)
+				{
+					uint32_t AttachmentCount = AttachmentCounts[Index];
+					if (AttachmentCount > 0)
+					{
+						if (WantedKeys.contains(Keys[Index]))
+						{
+							OutReferences.insert(OutReferences.end(), AttachmentStart, AttachmentStart + AttachmentCount);
+						}
+						AttachmentStart += AttachmentCount;
+					}
+				}
+			});
+	}
+	return false;
+}
+
+bool
 ZenCacheDiskLayer::CacheBucket::GetReferencesLocked(GcCtx& Ctx, std::vector<IoHash>& OutReferences)
 {
-	auto GetAttachments = [&](MemoryView Data) {
+	ZEN_TRACE_CPU("Z$::Bucket::GetReferencesLocked");
+
+	auto GetAttachments = [&](MemoryView Data) -> bool {
 		if (ValidateCompactBinary(Data, CbValidateMode::Default) == CbValidateError::None)
 		{
 			CbObjectView Obj(Data.GetData());
 			Obj.IterateAttachments([&](CbFieldView Field) { OutReferences.emplace_back(Field.AsAttachment()); });
+			return true;
 		}
+		return false;
 	};
 
 	std::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys;
@@ -3471,26 +3518,77 @@ ZenCacheDiskLayer::CacheBucket::GetReferencesLocked(GcCtx& Ctx, std::vector<IoHa
 		{
 			ZEN_ASSERT(!ChunkIndexes.empty());
 
-			bool Continue = m_BlockStore.IterateBlock(
-				InlineLocations,
-				ChunkIndexes,
-				[&](size_t ChunkIndex, const void* Data, uint64_t Size) {
-					ZEN_UNUSED(ChunkIndex);
-					GetAttachments(MemoryView(Data, Size));
-					return !Ctx.IsCancelledFlag.load();
-				},
-				[&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
-					ZEN_UNUSED(ChunkIndex);
-					GetAttachments(File.GetChunk(Offset, Size).GetView());
-					return !Ctx.IsCancelledFlag.load();
-				});
+			uint32_t BlockIndex = InlineLocations[ChunkIndexes[0]].BlockIndex;
+			if (!m_Configuration.StoreAttachmentMetaData ||
+				!ReadAttachmentsFromMetaData(BlockIndex, InlineKeys, ChunkIndexes, OutReferences))
+			{
+				std::vector<IoHash>	  Keys;
+				std::vector<uint32_t> AttachmentCounts;
+				size_t				  PrecachedReferencesStart	   = OutReferences.size();
+				size_t				  NextPrecachedReferencesStart = PrecachedReferencesStart;
 
-			if (!Continue && Ctx.IsCancelledFlag.load())
+				bool WriteMetaData = m_Configuration.StoreAttachmentMetaData && !m_BlockStore.IsWriting(BlockIndex);
+				if (WriteMetaData)
+				{
+					Keys.reserve(InlineLocations.size());
+				}
+
+				auto CaptureAttachments = [&](size_t ChunkIndex, MemoryView Data) {
+					if (GetAttachments(Data))
+					{
+						size_t AttachmentCount = OutReferences.size() - NextPrecachedReferencesStart;
+						if (WriteMetaData && AttachmentCount > 0)
+						{
+							Keys.push_back(InlineKeys[ChunkIndex]);
+							AttachmentCounts.push_back(gsl::narrow<uint32_t>(AttachmentCount));
+							NextPrecachedReferencesStart += AttachmentCount;
+						}
+					}
+				};
+
+				bool Continue = m_BlockStore.IterateBlock(
+					InlineLocations,
+					ChunkIndexes,
+					[&](size_t ChunkIndex, const void* Data, uint64_t Size) {
+						ZEN_UNUSED(ChunkIndex);
+						CaptureAttachments(ChunkIndex, MemoryView(Data, Size));
+						return !Ctx.IsCancelledFlag.load();
+					},
+					[&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+						ZEN_UNUSED(ChunkIndex);
+						CaptureAttachments(ChunkIndex, File.GetChunk(Offset, Size).GetView());
+						return !Ctx.IsCancelledFlag.load();
+					});
+
+				if (Continue)
+				{
+					if (WriteMetaData)
+					{
+						ZEN_ASSERT(Keys.size() == AttachmentCounts.size());
+						IoBuffer MetaDataPayload =
+							BuildReferenceMetaData<IoHash>(
+								BlockMetaDataExpectedMagic,
+								Keys,
+								AttachmentCounts,
+								std::span<const IoHash>(OutReferences)
+									.subspan(PrecachedReferencesStart, OutReferences.size() - PrecachedReferencesStart))
+								.Flatten()
+								.AsIoBuffer();
+						m_BlockStore.SetMetaData(BlockIndex, MetaDataPayload);
+					}
+				}
+				else
+				{
+					return false;
+				}
+			}
+			if (Ctx.IsCancelledFlag.load())
 			{
 				return false;
 			}
 		}
 	}
+
 	for (const auto& It : StandaloneKeys)
 	{
 		if (Ctx.IsCancelledFlag.load())
@@ -3551,7 +3649,6 @@ public:
 		RwLock::SharedLockScope IndexLock(m_CacheBucket.m_IndexLock);
 		bool					Continue = m_CacheBucket.GetReferencesLocked(Ctx, m_References);
 		IndexLock.ReleaseNow();
-
 		if (!Continue)
 		{
 			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index ba8259c82..8f67c1c0f 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -94,16 +94,22 @@ struct BlockStoreFile : public RefCounted
 	void						 Open();
 	void						 Create(uint64_t InitialSize);
 	void						 MarkAsDeleteOnClose();
-	uint64_t					 FileSize();
+	uint64_t					 FileSize() const;
+	uint64_t					 MetaSize() const;
+	uint64_t					 TotalSize() const { return FileSize() + MetaSize(); }
 	IoBuffer					 GetChunk(uint64_t Offset, uint64_t Size);
 	void						 Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Flush();
 	BasicFile&					 GetBasicFile();
-	void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
-	bool IsOpen() const;
+	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+	bool	 IsOpen() const;
+	bool	 SetMetaData(const IoBuffer& Payload);
+	IoBuffer GetMetaData() const;
 
 private:
+	std::filesystem::path		GetMetaPath() const;
+	void						RemoveMeta();
 	const std::filesystem::path m_Path;
 	IoBuffer					m_IoBuffer;
 	BasicFile					m_File;
@@ -196,6 +202,11 @@ public:
 
 	inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); }
 
+	bool IsWriting(uint32_t BlockIndex) const;
+
+	void	 SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload);
+	IoBuffer GetMetaData(uint32_t BlockIndex) const;
+
 private:
 	static const char*			 GetBlockFileExtension();
 	static std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex);
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 537f4396a..e7c995081 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -105,10 +105,11 @@ class ZenCacheDiskLayer
 public:
 	struct BucketConfiguration
 	{
-		uint64_t MaxBlockSize		   = 1ull << 30;
-		uint32_t PayloadAlignment	   = 1u << 4;
-		uint64_t MemCacheSizeThreshold = 1 * 1024;
-		uint64_t LargeObjectThreshold  = 128 * 1024;
+		uint64_t MaxBlockSize			 = 1ull << 30;
+		uint32_t PayloadAlignment		 = 1u << 4;
+		uint64_t MemCacheSizeThreshold	 = 1 * 1024;
+		uint64_t LargeObjectThreshold	 = 128 * 1024;
+		bool	 StoreAttachmentMetaData = false;
 	};
 
 	struct Configuration
@@ -236,6 +237,11 @@ public:
 		RwLock::SharedLockScope GetGcReferencerLock();
 		bool					GetReferencesLocked(GcCtx& Ctx, std::vector<IoHash>& OutReferences);
 
+		bool ReadAttachmentsFromMetaData(uint32_t					  BlockIndex,
+										 std::span<const IoHash>	  InlineKeys,
+										 std::span<const std::size_t> ChunkIndexes,
+										 std::vector<IoHash>&		  OutReferences) const;
+
 		inline GcStorageSize StorageSize() const
 		{
 			return {.DiskSize	= m_StandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(),