meta info store (#75)

- Feature: Added option `--gc-cache-attachment-store` which caches referenced attachments in cache records on disk for faster GC - default is `false`
- Feature: Added option `--gc-projectstore-attachment-store` which caches referenced attachments in project store oplogs on disk for faster GC - default is `false`

2 files changed, 24 insertions, 7 deletions

diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index ba8259c82..8f67c1c0f 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -94,16 +94,22 @@ struct BlockStoreFile : public RefCounted
 	void						 Open();
 	void						 Create(uint64_t InitialSize);
 	void						 MarkAsDeleteOnClose();
-	uint64_t					 FileSize();
+	uint64_t					 FileSize() const;
+	uint64_t					 MetaSize() const;
+	uint64_t					 TotalSize() const { return FileSize() + MetaSize(); }
 	IoBuffer					 GetChunk(uint64_t Offset, uint64_t Size);
 	void						 Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Flush();
 	BasicFile&					 GetBasicFile();
-	void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
-	bool IsOpen() const;
+	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+	bool	 IsOpen() const;
+	bool	 SetMetaData(const IoBuffer& Payload);
+	IoBuffer GetMetaData() const;
 
 private:
+	std::filesystem::path		GetMetaPath() const;
+	void						RemoveMeta();
 	const std::filesystem::path m_Path;
 	IoBuffer					m_IoBuffer;
 	BasicFile					m_File;
@@ -196,6 +202,11 @@ public:
 
 	inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); }
 
+	bool IsWriting(uint32_t BlockIndex) const;
+
+	void	 SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload);
+	IoBuffer GetMetaData(uint32_t BlockIndex) const;
+
 private:
 	static const char*			 GetBlockFileExtension();
 	static std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex);
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 537f4396a..e7c995081 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -105,10 +105,11 @@ class ZenCacheDiskLayer
 public:
 	struct BucketConfiguration
 	{
-		uint64_t MaxBlockSize		   = 1ull << 30;
-		uint32_t PayloadAlignment	   = 1u << 4;
-		uint64_t MemCacheSizeThreshold = 1 * 1024;
-		uint64_t LargeObjectThreshold  = 128 * 1024;
+		uint64_t MaxBlockSize			 = 1ull << 30;
+		uint32_t PayloadAlignment		 = 1u << 4;
+		uint64_t MemCacheSizeThreshold	 = 1 * 1024;
+		uint64_t LargeObjectThreshold	 = 128 * 1024;
+		bool	 StoreAttachmentMetaData = false;
 	};
 
 	struct Configuration
@@ -236,6 +237,11 @@ public:
 		RwLock::SharedLockScope GetGcReferencerLock();
 		bool					GetReferencesLocked(GcCtx& Ctx, std::vector<IoHash>& OutReferences);
 
+		bool ReadAttachmentsFromMetaData(uint32_t					  BlockIndex,
+										 std::span<const IoHash>	  InlineKeys,
+										 std::span<const std::size_t> ChunkIndexes,
+										 std::vector<IoHash>&		  OutReferences) const;
+
 		inline GcStorageSize StorageSize() const
 		{
 			return {.DiskSize	= m_StandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(),