meta info store (#75)

- Feature: Added option `--gc-cache-attachment-store` which caches referenced attachments in cache records on disk for faster GC - default is `false`
- Feature: Added option `--gc-projectstore-attachment-store` which caches referenced attachments in project store oplogs on disk for faster GC - default is `false`

13 files changed, 618 insertions, 63 deletions

diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index b430db629..6c2bf40d8 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -518,6 +518,12 @@ ParseConfigFile(const std::filesystem::path& Path,
 						 "gc-compactblock-threshold"sv);
 	LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv);
 	LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv);
+	LuaOptions.AddOption("gc.cache.attachment.store"sv,
+						 ServerOptions.StructuredCacheConfig.StoreAttachmentMetaData,
+						 "gc-cache-attachment-store");
+	LuaOptions.AddOption("gc.projectstore.attachment.store"sv,
+						 ServerOptions.ProjectStoreConfig.StoreAttachmentMetaData,
+						 "gc-projectstore-attachment-store");
 
 	////// gc
 	LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv);
@@ -898,6 +904,20 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 	options.add_option("gc",
 					   "",
+					   "gc-cache-attachment-store",
+					   "Enable storing attachments referenced by a cache record in block store meta data.",
+					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.StoreAttachmentMetaData)->default_value("false"),
+					   "");
+
+	options.add_option("gc",
+					   "",
+					   "gc-projectstore-attachment-store",
+					   "Enable storing attachments referenced by project oplogs in meta data.",
+					   cxxopts::value<bool>(ServerOptions.ProjectStoreConfig.StoreAttachmentMetaData)->default_value("false"),
+					   "");
+
+	options.add_option("gc",
+					   "",
 					   "gc-enabled",
 					   "Whether garbage collection is enabled or not.",
 					   cxxopts::value<bool>(ServerOptions.GcConfig.Enabled)->default_value("true"),
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 4c9b9361c..6e45e7230 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -117,6 +117,12 @@ struct ZenStructuredCacheConfig
 	uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024;
 	uint64_t MemTrimIntervalSeconds	 = 60;
 	uint64_t MemMaxAgeSeconds		 = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count());
+	bool	 StoreAttachmentMetaData = false;
+};
+
+struct ZenProjectStoreConfig
+{
+	bool StoreAttachmentMetaData = false;
 };
 
 struct ZenWorkspacesConfig
@@ -132,6 +138,7 @@ struct ZenServerOptions
 	ZenObjectStoreConfig	 ObjectStoreConfig;
 	zen::HttpServerConfig	 HttpServerConfig;
 	ZenStructuredCacheConfig StructuredCacheConfig;
+	ZenProjectStoreConfig	 ProjectStoreConfig;
 	ZenStatsConfig			 StatsConfig;
 	ZenWorkspacesConfig		 WorksSpacesConfig;
 	std::filesystem::path	 SystemRootDir;				  // System root directory (used for machine level config)
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index bc1020b58..378423100 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -20,6 +20,7 @@
 #include <zenstore/scrubcontext.h>
 #include <zenutil/cache/rpcrecording.h>
 #include <zenutil/packageformat.h>
+#include <zenutil/referencemetadata.h>
 #include <zenutil/workerpools.h>
 
 #include "fileremoteprojectstore.h"
@@ -111,6 +112,22 @@ namespace {
 		return false;
 	}
 
+	bool IsFileOlderThan(const std::filesystem::path& CheckPath, const std::filesystem::path& ReferencePath)
+	{
+		std::error_code					Ec;
+		std::filesystem::file_time_type CheckWriteTime = std::filesystem::last_write_time(CheckPath, Ec);
+		if (Ec)
+		{
+			return true;
+		}
+		std::filesystem::file_time_type ReferenceWriteTime = std::filesystem::last_write_time(ReferencePath, Ec);
+		if (Ec)
+		{
+			return true;
+		}
+		return CheckWriteTime < ReferenceWriteTime;
+	}
+
 	struct CreateRemoteStoreResult
 	{
 		std::shared_ptr<RemoteProjectStore> Store;
@@ -308,6 +325,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 	{
 		if (Exists(BasePath))
 		{
+			std::error_code DummyEc;
 			return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath));
 		}
 		return 0;
@@ -845,6 +863,7 @@ ProjectStore::Oplog::Oplog(std::string_view				Id,
 , m_CidStore(Store)
 , m_BasePath(BasePath)
 , m_MarkerPath(MarkerPath)
+, m_MetaValid(false)
 , m_OplogId(Id)
 {
 	using namespace std::literals;
@@ -852,8 +871,9 @@ ProjectStore::Oplog::Oplog(std::string_view				Id,
 	m_Storage			   = new OplogStorage(this, m_BasePath);
 	const bool StoreExists = m_Storage->Exists();
 	m_Storage->Open(/* IsCreate */ !StoreExists);
-
-	m_TempPath = m_BasePath / "temp"sv;
+	m_TempPath	= m_BasePath / "temp"sv;
+	m_MetaPath	= m_BasePath / "ops.meta"sv;
+	m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath());
 
 	CleanDirectory(m_TempPath);
 }
@@ -871,6 +891,11 @@ ProjectStore::Oplog::Flush()
 {
 	ZEN_ASSERT(m_Storage);
 	m_Storage->Flush();
+	if (!m_MetaValid)
+	{
+		std::error_code DummyEc;
+		std::filesystem::remove(m_MetaPath, DummyEc);
+	}
 }
 
 void
@@ -923,12 +948,15 @@ ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
 
 			// Actually perform some clean-up
 			RwLock::ExclusiveLockScope _(m_OplogLock);
-
 			for (const auto& Key : BadEntryKeys)
 			{
 				m_LatestOpMap.erase(Key);
 				m_Storage->AppendTombstone(Key);
 			}
+			if (!BadEntryKeys.empty())
+			{
+				m_MetaValid = false;
+			}
 		}
 		else
 		{
@@ -970,6 +998,11 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath)
 	{
 		Size += std::filesystem::file_size(StateFilePath);
 	}
+	std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv;
+	if (std::filesystem::exists(MetaFilePath))
+	{
+		Size += std::filesystem::file_size(MetaFilePath);
+	}
 
 	return Size;
 }
@@ -1097,6 +1130,7 @@ ProjectStore::Oplog::Reset()
 			m_Storage			   = new OplogStorage(this, m_BasePath);
 			const bool StoreExists = m_Storage->Exists();
 			m_Storage->Open(/* IsCreate */ !StoreExists);
+			m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath());
 			return false;
 		}
 		m_ChunkMap.clear();
@@ -1106,6 +1140,7 @@ ProjectStore::Oplog::Reset()
 		m_LatestOpMap.clear();
 		m_Storage = new OplogStorage(this, m_BasePath);
 		m_Storage->Open(true);
+		m_MetaValid = false;
 		CleanDirectory(m_TempPath);
 		Write();
 	}
@@ -1491,6 +1526,80 @@ ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Hand
 	m_Storage->ReplayLogEntries(Entries, [&](CbObjectView Op) { Handler(Op); });
 }
 
+static constexpr uint32_t OplogMetaDataExpectedMagic = 0x6f'74'6d'62;  // 'omta';
+
+void
+ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, bool StoreMetaDataOnDisk)
+{
+	if (!m_Storage)
+	{
+		return;
+	}
+
+	if (StoreMetaDataOnDisk && m_MetaValid)
+	{
+		IoBuffer MetadataPayload = IoBufferBuilder::MakeFromFile(m_MetaPath);
+		if (MetadataPayload)
+		{
+			if (GetAttachmentsFromMetaData<Oid, IoHash>(
+					MetadataPayload,
+					OplogMetaDataExpectedMagic,
+					[&](std::span<const Oid> Keys, std::span<const uint32_t> AttachmentCounts, std::span<const IoHash> Attachments) {
+						ZEN_UNUSED(Keys);
+						ZEN_UNUSED(AttachmentCounts);
+						OutAttachments.insert(OutAttachments.end(), Attachments.begin(), Attachments.end());
+					}))
+			{
+				return;
+			}
+		}
+	}
+
+	std::vector<Oid>	  Keys;
+	std::vector<uint32_t> AttachmentCounts;
+	size_t				  AttachmentOffset = OutAttachments.size();
+
+	IterateOplogLocked([&](CbObjectView Op) {
+		using namespace std::literals;
+		size_t CurrentAttachmentCount = OutAttachments.size();
+		Op.IterateAttachments([&](CbFieldView Visitor) { OutAttachments.emplace_back(Visitor.AsAttachment()); });
+		if (StoreMetaDataOnDisk)
+		{
+			XXH3_128Stream KeyHasher;
+			Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
+			XXH3_128 KeyHash128 = KeyHasher.GetHash();
+			Oid		 KeyHash;
+			memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+			Keys.push_back(KeyHash);
+			AttachmentCounts.push_back(gsl::narrow<uint32_t>(OutAttachments.size() - CurrentAttachmentCount));
+		}
+	});
+	if (StoreMetaDataOnDisk)
+	{
+		const IoHash* FirstAttachment = OutAttachments.data() + AttachmentOffset;
+		size_t		  AttachmentCount = OutAttachments.size() - AttachmentOffset;
+		IoBuffer	  MetaPayload	  = BuildReferenceMetaData<Oid>(OplogMetaDataExpectedMagic,
+															Keys,
+															AttachmentCounts,
+															std::span<const IoHash>(FirstAttachment, AttachmentCount))
+								   .Flatten()
+								   .AsIoBuffer();
+
+		const std::filesystem::path MetaPath = m_MetaPath;
+		std::error_code				Ec;
+		TemporaryFile::SafeWriteFile(MetaPath, MetaPayload.GetView(), Ec);
+		if (Ec)
+		{
+			m_MetaValid = false;
+			ZEN_WARN("Unable to set meta data for oplog '{}' at meta path: '{}'. Reason: '{}'", OplogId(), MetaPath, Ec.message());
+		}
+		else
+		{
+			m_MetaValid = true;
+		}
+	}
+}
+
 size_t
 ProjectStore::Oplog::GetOplogEntryCount() const
 {
@@ -1954,7 +2063,7 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObjectView Core)
 	using namespace std::literals;
 
 	RefPtr<OplogStorage> Storage = GetStorage();
-	if (!m_Storage)
+	if (!Storage)
 	{
 		return 0xffffffffu;
 	}
@@ -1962,11 +2071,12 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObjectView Core)
 	OplogEntryMapping		   Mapping = GetMapping(Core);
 	OplogStorage::AppendOpData OpData  = OplogStorage::GetAppendOpData(Core);
 
-	const OplogEntry OpEntry = m_Storage->AppendOp(OpData);
+	const OplogEntry OpEntry = Storage->AppendOp(OpData);
 
 	RwLock::ExclusiveLockScope OplogLock(m_OplogLock);
 	const uint32_t			   EntryId = RegisterOplogEntry(OplogLock, Mapping, OpEntry);
 	CaptureUpdatedLSNs(std::array<uint32_t, 1u>({EntryId}));
+	m_MetaValid = false;
 
 	return EntryId;
 }
@@ -1979,7 +2089,7 @@ ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores)
 	using namespace std::literals;
 
 	RefPtr<OplogStorage> Storage = GetStorage();
-	if (!m_Storage)
+	if (!Storage)
 	{
 		return std::vector<uint32_t>(Cores.size(), 0xffffffffu);
 	}
@@ -2008,6 +2118,7 @@ ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores)
 			EntryIds[OpIndex] = RegisterOplogEntry(OplogLock, Mappings[OpIndex], OpEntries[OpIndex]);
 		}
 		CaptureUpdatedLSNs(EntryIds);
+		m_MetaValid = false;
 	}
 	return EntryIds;
 }
@@ -2664,12 +2775,13 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const
 
 //////////////////////////////////////////////////////////////////////////
 
-ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue)
+ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, const Configuration& Config)
 : m_Log(logging::Get("project"))
 , m_Gc(Gc)
 , m_CidStore(Store)
 , m_JobQueue(JobQueue)
 , m_ProjectBasePath(BasePath)
+, m_Config(Config)
 , m_DiskWriteBlocker(Gc.GetDiskWriteBlocker())
 {
 	ZEN_INFO("initializing project store at '{}'", m_ProjectBasePath);
@@ -4675,10 +4787,7 @@ public:
 						 Oplog->OplogId());
 			});
 
-			Oplog->IterateOplogLocked([&](const CbObjectView& UpdateOp) -> bool {
-				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
-				return true;
-			});
+			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
 	}
 
@@ -4779,9 +4888,8 @@ public:
 			{
 				return;
 			}
-			m_Oplog->IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
-			});
+
+			m_Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
 	}
 
@@ -5021,7 +5129,7 @@ TEST_CASE("project.store.create")
 
 	std::string_view	  ProjectName("proj1"sv);
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -5050,7 +5158,7 @@ TEST_CASE("project.store.lifetimes")
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -5112,7 +5220,7 @@ TEST_CASE_TEMPLATE("project.store.export",
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -5193,7 +5301,7 @@ TEST_CASE("project.store.gc")
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		= TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
 
@@ -5477,7 +5585,7 @@ TEST_CASE("project.store.partial.read")
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		= TempDir.Path() / "root"sv;
 	std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
 
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index bb042e331..bed3c83b7 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -63,7 +63,12 @@ class ProjectStore : public RefCounted, public GcStorage, public GcContributor,
 	struct OplogStorage;
 
 public:
-	ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue);
+	struct Configuration
+	{
+		bool StoreAttachmentMetaData = false;
+	};
+
+	ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, const Configuration& Config);
 	~ProjectStore();
 
 	struct Project;
@@ -162,6 +167,8 @@ public:
 		uint32_t GetUnusedSpacePercent() const;
 		void	 Compact(bool DryRun, bool RetainLSNs, std::string_view LogPrefix);
 
+		void GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, bool StoreMetaDataOnDisk);
+
 	private:
 		struct FileMapEntry
 		{
@@ -177,6 +184,7 @@ public:
 		std::filesystem::path m_BasePath;
 		std::filesystem::path m_MarkerPath;
 		std::filesystem::path m_TempPath;
+		std::filesystem::path m_MetaPath;
 
 		mutable RwLock								m_OplogLock;
 		OidMap<IoHash>								m_ChunkMap;			// output data chunk id -> CAS address
@@ -185,6 +193,7 @@ public:
 		int32_t										m_ManifestVersion;	// File system manifest version
 		tsl::robin_map<uint32_t, OplogEntryAddress> m_OpAddressMap;		// Index LSN -> op data in ops blob file
 		OidMap<uint32_t>							m_LatestOpMap;		// op key -> latest op LSN for key
+		std::atomic<bool>							m_MetaValid = false;
 
 		mutable RwLock						   m_UpdateCaptureLock;
 		uint32_t							   m_UpdateCaptureRefCounter = 0;
@@ -429,6 +438,7 @@ private:
 	CidStore&								  m_CidStore;
 	JobQueue&								  m_JobQueue;
 	std::filesystem::path					  m_ProjectBasePath;
+	const Configuration						  m_Config;
 	mutable RwLock							  m_ProjectsLock;
 	std::map<std::string, Ref<Project>>		  m_Projects;
 	const DiskWriteBlocker*					  m_DiskWriteBlocker = nullptr;
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index dcfef7530..ebc56d7d9 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -232,7 +232,12 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 
 	ZEN_INFO("instantiating project service");
 
-	m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, *m_JobQueue);
+	m_ProjectStore =
+		new ProjectStore(*m_CidStore,
+						 m_DataRoot / "projects",
+						 m_GcManager,
+						 *m_JobQueue,
+						 ProjectStore::Configuration{.StoreAttachmentMetaData = ServerOptions.ProjectStoreConfig.StoreAttachmentMetaData});
 	m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr});
 
 	if (ServerOptions.WorksSpacesConfig.Enabled)
@@ -513,6 +518,8 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 	Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes		  = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes;
 	Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds		  = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds;
 	Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds			  = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds;
+	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.StoreAttachmentMetaData =
+		ServerOptions.StructuredCacheConfig.StoreAttachmentMetaData;
 
 	if (ServerOptions.IsDedicated)
 	{
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 70ddcedaf..287a3f7fa 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -90,6 +90,7 @@ BlockStoreFile::Create(uint64_t InitialSize)
 		RetriesLeft--;
 		return true;
 	});
+	RemoveMeta();
 
 	void* FileHandle = m_File.Handle();
 
@@ -99,15 +100,26 @@ BlockStoreFile::Create(uint64_t InitialSize)
 }
 
 uint64_t
-BlockStoreFile::FileSize()
+BlockStoreFile::FileSize() const
 {
-	return m_CachedFileSize == 0 ? m_File.FileSize() : m_CachedFileSize;
+	if (m_CachedFileSize == 0)
+	{
+		std::error_code Ec;
+		uint64_t		Size = m_File.FileSize(Ec);
+		if (Ec)
+		{
+			return 0;
+		}
+		return Size;
+	}
+	return m_CachedFileSize;
 }
 
 void
 BlockStoreFile::MarkAsDeleteOnClose()
 {
 	m_IoBuffer.SetDeleteOnClose(true);
+	RemoveMeta();
 }
 
 IoBuffer
@@ -157,6 +169,90 @@ BlockStoreFile::IsOpen() const
 	return !!m_IoBuffer;
 }
 
+bool
+BlockStoreFile::SetMetaData(const IoBuffer& Payload)
+{
+	if (!Payload)
+	{
+		RemoveMeta();
+		return true;
+	}
+	const std::filesystem::path MetaPath = GetMetaPath();
+	std::error_code				Ec;
+	TemporaryFile::SafeWriteFile(MetaPath, Payload.GetView(), Ec);
+	if (Ec)
+	{
+		ZEN_WARN("Unable to set meta data for block '{}' at meta path: '{}'. Reason: '{}'", m_Path, MetaPath, Ec.message());
+		return false;
+	}
+	return true;
+}
+
+static bool
+IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::path& MetaPath)
+{
+	std::error_code					Ec;
+	std::filesystem::file_time_type MetaWriteTime = std::filesystem::last_write_time(MetaPath, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	std::filesystem::file_time_type BlockWriteTime = std::filesystem::last_write_time(BlockPath, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	if (MetaWriteTime < BlockWriteTime)
+	{
+		std::filesystem::remove(MetaPath, Ec);
+		return false;
+	}
+	return true;
+}
+
+IoBuffer
+BlockStoreFile::GetMetaData() const
+{
+	const std::filesystem::path MetaPath = GetMetaPath();
+	if (IsMetaDataValid(m_Path, MetaPath))
+	{
+		return IoBufferBuilder::MakeFromFile(MetaPath);
+	}
+	return {};
+}
+
+uint64_t
+BlockStoreFile::MetaSize() const
+{
+	const std::filesystem::path MetaPath = GetMetaPath();
+	if (IsMetaDataValid(m_Path, MetaPath))
+	{
+		std::error_code DummyEc;
+		if (uint64_t Size = std::filesystem::file_size(MetaPath, DummyEc); !DummyEc)
+		{
+			return Size;
+		}
+	}
+	return 0;
+}
+
+void
+BlockStoreFile::RemoveMeta()
+{
+	std::filesystem::path MetaPath = GetMetaPath();
+	std::error_code		  DummyEc;
+	std::filesystem::remove(MetaPath, DummyEc);
+}
+
+std::filesystem::path
+BlockStoreFile::GetMetaPath() const
+{
+	std::filesystem::path MetaPath(m_Path);
+	return MetaPath.replace_extension(".meta");
+}
+
+////////////////////////////////////////////////////////
+
 constexpr uint64_t DefaultIterateSmallChunkWindowSize = 2 * 1024 * 1024;
 constexpr uint64_t IterateSmallChunkMaxGapSize		  = 4 * 1024;
 
@@ -215,7 +311,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 					}
 					Ref<BlockStoreFile> BlockFile{new BlockStoreFile(Path)};
 					BlockFile->Open();
-					m_TotalSize.fetch_add(BlockFile->FileSize(), std::memory_order::relaxed);
+					m_TotalSize.fetch_add(BlockFile->TotalSize(), std::memory_order::relaxed);
 					m_ChunkBlocks[BlockIndex] = BlockFile;
 					FoundBlocks[BlockIndex]	  = BlockFile->FileSize();
 					if (BlockIndex >= NextBlockIndex)
@@ -283,7 +379,7 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownLocations)
 		std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex);
 		if (m_ChunkBlocks[BlockIndex])
 		{
-			m_TotalSize.fetch_sub(m_ChunkBlocks[BlockIndex]->FileSize(), std::memory_order::relaxed);
+			m_TotalSize.fetch_sub(m_ChunkBlocks[BlockIndex]->TotalSize(), std::memory_order::relaxed);
 			m_ChunkBlocks[BlockIndex]->MarkAsDeleteOnClose();
 		}
 		m_ChunkBlocks.erase(BlockIndex);
@@ -781,6 +877,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 			   {
 				   ZEN_DEBUG("dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
 				   m_TotalSize.fetch_sub(NewBlockFile->FileSize(), std::memory_order::relaxed);
+				   ZEN_ASSERT_SLOW(NewBlockFile->MetaSize() == 0);
 				   NewBlockFile->MarkAsDeleteOnClose();
 			   }
 		   });
@@ -833,7 +930,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 					ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
 					ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
 					m_ChunkBlocks.erase(BlockIndex);
-					m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
+					m_TotalSize.fetch_sub(OldBlockFile->TotalSize(), std::memory_order::relaxed);
 					OldBlockFile->MarkAsDeleteOnClose();
 				}
 				continue;
@@ -1003,7 +1100,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 				ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
 				ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
 				m_ChunkBlocks.erase(BlockIndex);
-				m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
+				m_TotalSize.fetch_sub(OldBlockFile->TotalSize(), std::memory_order::relaxed);
 				OldBlockFile->MarkAsDeleteOnClose();
 			}
 		}
@@ -1500,9 +1597,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 					  LogPrefix,
 					  OldBlockFile->GetPath().filename(),
 					  NiceBytes(OldBlockSize));
+			m_TotalSize.fetch_sub(OldBlockSize);
+			m_TotalSize.fetch_sub(OldBlockFile->MetaSize());
 			OldBlockFile->MarkAsDeleteOnClose();
 			m_ChunkBlocks.erase(BlockIndex);
-			m_TotalSize.fetch_sub(OldBlockSize);
 			RemovedSize += OldBlockSize;
 		}
 		return true;
@@ -1544,6 +1642,48 @@ BlockStore::GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint
 	return Path.ToPath();
 }
 
+bool
+BlockStore::IsWriting(uint32_t BlockIndex) const
+{
+	RwLock::SharedLockScope _(m_InsertLock);
+	if (std::find(m_ActiveWriteBlocks.begin(), m_ActiveWriteBlocks.end(), BlockIndex) != m_ActiveWriteBlocks.end())
+	{
+		return true;
+	}
+	if (BlockIndex == m_WriteBlockIndex.load() && m_WriteBlock)
+	{
+		return true;
+	}
+	return false;
+}
+
+void
+BlockStore::SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload)
+{
+	RwLock::ExclusiveLockScope _(m_InsertLock);
+	if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end() && It->second)
+	{
+		uint64_t OldMetaSize = It->second->MetaSize();
+		if (It->second->SetMetaData(Payload))
+		{
+			uint64_t NewMetaSize = It->second->MetaSize();
+			m_TotalSize += NewMetaSize;
+			m_TotalSize -= OldMetaSize;
+		}
+	}
+}
+
+IoBuffer
+BlockStore::GetMetaData(uint32_t BlockIndex) const
+{
+	RwLock::SharedLockScope _(m_InsertLock);
+	if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end() && It->second)
+	{
+		return It->second->GetMetaData();
+	}
+	return {};
+}
+
 #if ZEN_WITH_TESTS
 
 TEST_CASE("blockstore.blockstoredisklocation")
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index f85d05dce..940f78c30 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -14,6 +14,7 @@
 #include <zencore/trace.h>
 #include <zencore/workthreadpool.h>
 #include <zencore/xxhash.h>
+#include <zenutil/referencemetadata.h>
 #include <zenutil/workerpools.h>
 
 #include <future>
@@ -109,6 +110,8 @@ namespace {
 
 	static_assert(sizeof(BucketMetaHeader) == 32);
 
+	static constexpr uint32_t BlockMetaDataExpectedMagic = 0x61'74'6d'62;  // 'bmta';
+
 #pragma pack(pop)
 
 	//////////////////////////////////////////////////////////////////////////
@@ -3410,14 +3413,58 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 }
 
 bool
+ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t					 BlockIndex,
+															std::span<const IoHash>		 InlineKeys,
+															std::span<const std::size_t> ChunkIndexes,
+															std::vector<IoHash>&		 OutReferences) const
+{
+	ZEN_TRACE_CPU("Z$::Bucket::ReadAttachmentsFromMetaData");
+	IoBuffer MetaDataPayload = m_BlockStore.GetMetaData(BlockIndex);
+	if (MetaDataPayload)
+	{
+		std::unordered_set<IoHash, IoHash::Hasher> WantedKeys;
+		WantedKeys.reserve(ChunkIndexes.size());
+		for (const size_t ChunkIndex : ChunkIndexes)
+		{
+			WantedKeys.insert(InlineKeys[ChunkIndex]);
+		}
+		ZEN_TRACE_CPU("Z$::Bucket::GetAttachmentsFromMetaData");
+		return GetAttachmentsFromMetaData<IoHash, IoHash>(
+			MetaDataPayload,
+			BlockMetaDataExpectedMagic,
+			[&](std::span<const IoHash> Keys, std::span<const uint32_t> AttachmentCounts, std::span<const IoHash> Attachments) {
+				OutReferences.reserve(OutReferences.capacity() + Attachments.size());
+				auto AttachmentStart = Attachments.begin();
+				for (uint32_t Index = 0; Index < Keys.size(); Index++)
+				{
+					uint32_t AttachmentCount = AttachmentCounts[Index];
+					if (AttachmentCount > 0)
+					{
+						if (WantedKeys.contains(Keys[Index]))
+						{
+							OutReferences.insert(OutReferences.end(), AttachmentStart, AttachmentStart + AttachmentCount);
+						}
+						AttachmentStart += AttachmentCount;
+					}
+				}
+			});
+	}
+	return false;
+}
+
+bool
 ZenCacheDiskLayer::CacheBucket::GetReferencesLocked(GcCtx& Ctx, std::vector<IoHash>& OutReferences)
 {
-	auto GetAttachments = [&](MemoryView Data) {
+	ZEN_TRACE_CPU("Z$::Bucket::GetReferencesLocked");
+
+	auto GetAttachments = [&](MemoryView Data) -> bool {
 		if (ValidateCompactBinary(Data, CbValidateMode::Default) == CbValidateError::None)
 		{
 			CbObjectView Obj(Data.GetData());
 			Obj.IterateAttachments([&](CbFieldView Field) { OutReferences.emplace_back(Field.AsAttachment()); });
+			return true;
 		}
+		return false;
 	};
 
 	std::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys;
@@ -3471,26 +3518,77 @@ ZenCacheDiskLayer::CacheBucket::GetReferencesLocked(GcCtx& Ctx, std::vector<IoHa
 		{
 			ZEN_ASSERT(!ChunkIndexes.empty());
 
-			bool Continue = m_BlockStore.IterateBlock(
-				InlineLocations,
-				ChunkIndexes,
-				[&](size_t ChunkIndex, const void* Data, uint64_t Size) {
-					ZEN_UNUSED(ChunkIndex);
-					GetAttachments(MemoryView(Data, Size));
-					return !Ctx.IsCancelledFlag.load();
-				},
-				[&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
-					ZEN_UNUSED(ChunkIndex);
-					GetAttachments(File.GetChunk(Offset, Size).GetView());
-					return !Ctx.IsCancelledFlag.load();
-				});
+			uint32_t BlockIndex = InlineLocations[ChunkIndexes[0]].BlockIndex;
+			if (!m_Configuration.StoreAttachmentMetaData ||
+				!ReadAttachmentsFromMetaData(BlockIndex, InlineKeys, ChunkIndexes, OutReferences))
+			{
+				std::vector<IoHash>	  Keys;
+				std::vector<uint32_t> AttachmentCounts;
+				size_t				  PrecachedReferencesStart	   = OutReferences.size();
+				size_t				  NextPrecachedReferencesStart = PrecachedReferencesStart;
 
-			if (!Continue && Ctx.IsCancelledFlag.load())
+				bool WriteMetaData = m_Configuration.StoreAttachmentMetaData && !m_BlockStore.IsWriting(BlockIndex);
+				if (WriteMetaData)
+				{
+					Keys.reserve(InlineLocations.size());
+				}
+
+				auto CaptureAttachments = [&](size_t ChunkIndex, MemoryView Data) {
+					if (GetAttachments(Data))
+					{
+						size_t AttachmentCount = OutReferences.size() - NextPrecachedReferencesStart;
+						if (WriteMetaData && AttachmentCount > 0)
+						{
+							Keys.push_back(InlineKeys[ChunkIndex]);
+							AttachmentCounts.push_back(gsl::narrow<uint32_t>(AttachmentCount));
+							NextPrecachedReferencesStart += AttachmentCount;
+						}
+					}
+				};
+
+				bool Continue = m_BlockStore.IterateBlock(
+					InlineLocations,
+					ChunkIndexes,
+					[&](size_t ChunkIndex, const void* Data, uint64_t Size) {
+						ZEN_UNUSED(ChunkIndex);
+						CaptureAttachments(ChunkIndex, MemoryView(Data, Size));
+						return !Ctx.IsCancelledFlag.load();
+					},
+					[&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+						ZEN_UNUSED(ChunkIndex);
+						CaptureAttachments(ChunkIndex, File.GetChunk(Offset, Size).GetView());
+						return !Ctx.IsCancelledFlag.load();
+					});
+
+				if (Continue)
+				{
+					if (WriteMetaData)
+					{
+						ZEN_ASSERT(Keys.size() == AttachmentCounts.size());
+						IoBuffer MetaDataPayload =
+							BuildReferenceMetaData<IoHash>(
+								BlockMetaDataExpectedMagic,
+								Keys,
+								AttachmentCounts,
+								std::span<const IoHash>(OutReferences)
+									.subspan(PrecachedReferencesStart, OutReferences.size() - PrecachedReferencesStart))
+								.Flatten()
+								.AsIoBuffer();
+						m_BlockStore.SetMetaData(BlockIndex, MetaDataPayload);
+					}
+				}
+				else
+				{
+					return false;
+				}
+			}
+			if (Ctx.IsCancelledFlag.load())
 			{
 				return false;
 			}
 		}
 	}
+
 	for (const auto& It : StandaloneKeys)
 	{
 		if (Ctx.IsCancelledFlag.load())
@@ -3551,7 +3649,6 @@ public:
 		RwLock::SharedLockScope IndexLock(m_CacheBucket.m_IndexLock);
 		bool					Continue = m_CacheBucket.GetReferencesLocked(Ctx, m_References);
 		IndexLock.ReleaseNow();
-
 		if (!Continue)
 		{
 			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index ba8259c82..8f67c1c0f 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -94,16 +94,22 @@ struct BlockStoreFile : public RefCounted
 	void						 Open();
 	void						 Create(uint64_t InitialSize);
 	void						 MarkAsDeleteOnClose();
-	uint64_t					 FileSize();
+	uint64_t					 FileSize() const;
+	uint64_t					 MetaSize() const;
+	uint64_t					 TotalSize() const { return FileSize() + MetaSize(); }
 	IoBuffer					 GetChunk(uint64_t Offset, uint64_t Size);
 	void						 Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Flush();
 	BasicFile&					 GetBasicFile();
-	void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
-	bool IsOpen() const;
+	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+	bool	 IsOpen() const;
+	bool	 SetMetaData(const IoBuffer& Payload);
+	IoBuffer GetMetaData() const;
 
 private:
+	std::filesystem::path		GetMetaPath() const;
+	void						RemoveMeta();
 	const std::filesystem::path m_Path;
 	IoBuffer					m_IoBuffer;
 	BasicFile					m_File;
@@ -196,6 +202,11 @@ public:
 
 	inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); }
 
+	bool IsWriting(uint32_t BlockIndex) const;
+
+	void	 SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload);
+	IoBuffer GetMetaData(uint32_t BlockIndex) const;
+
 private:
 	static const char*			 GetBlockFileExtension();
 	static std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex);
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 537f4396a..e7c995081 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -105,10 +105,11 @@ class ZenCacheDiskLayer
 public:
 	struct BucketConfiguration
 	{
-		uint64_t MaxBlockSize		   = 1ull << 30;
-		uint32_t PayloadAlignment	   = 1u << 4;
-		uint64_t MemCacheSizeThreshold = 1 * 1024;
-		uint64_t LargeObjectThreshold  = 128 * 1024;
+		uint64_t MaxBlockSize			 = 1ull << 30;
+		uint32_t PayloadAlignment		 = 1u << 4;
+		uint64_t MemCacheSizeThreshold	 = 1 * 1024;
+		uint64_t LargeObjectThreshold	 = 128 * 1024;
+		bool	 StoreAttachmentMetaData = false;
 	};
 
 	struct Configuration
@@ -236,6 +237,11 @@ public:
 		RwLock::SharedLockScope GetGcReferencerLock();
 		bool					GetReferencesLocked(GcCtx& Ctx, std::vector<IoHash>& OutReferences);
 
+		bool ReadAttachmentsFromMetaData(uint32_t					  BlockIndex,
+										 std::span<const IoHash>	  InlineKeys,
+										 std::span<const std::size_t> ChunkIndexes,
+										 std::vector<IoHash>&		  OutReferences) const;
+
 		inline GcStorageSize StorageSize() const
 		{
 			return {.DiskSize	= m_StandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(),
diff --git a/src/zenutil/basicfile.cpp b/src/zenutil/basicfile.cpp
index 266146ca1..174125069 100644
--- a/src/zenutil/basicfile.cpp
+++ b/src/zenutil/basicfile.cpp
@@ -372,7 +372,7 @@ BasicFile::Flush()
 }
 
 uint64_t
-BasicFile::FileSize()
+BasicFile::FileSize() const
 {
 #if ZEN_PLATFORM_WINDOWS
 	ULARGE_INTEGER liFileSize;
@@ -401,7 +401,7 @@ BasicFile::FileSize()
 }
 
 uint64_t
-BasicFile::FileSize(std::error_code& Ec)
+BasicFile::FileSize(std::error_code& Ec) const
 {
 #if ZEN_PLATFORM_WINDOWS
 	ULARGE_INTEGER liFileSize;
@@ -565,20 +565,27 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::
 //////////////////////////////////////////////////////////////////////////
 
 void
-TemporaryFile::SafeWriteFile(std::filesystem::path Path, MemoryView Data)
+TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data)
 {
 	TemporaryFile	TempFile;
 	std::error_code Ec;
-	TempFile.CreateTemporary(Path.parent_path(), Ec);
+	SafeWriteFile(Path, Data, Ec);
 	if (Ec)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to create temp file for file at '{}'", Path));
+		throw std::system_error(Ec, fmt::format("Failed to safely write file '{}'", Path));
 	}
-	TempFile.Write(Data, 0);
-	TempFile.MoveTemporaryIntoPlace(Path, Ec);
-	if (Ec)
+}
+
+void
+TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data, std::error_code& OutEc)
+{
+	TemporaryFile TempFile;
+	if (TempFile.CreateTemporary(Path.parent_path(), OutEc); !OutEc)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", TempFile.GetPath(), Path));
+		if (TempFile.Write(Data, 0, OutEc); !OutEc)
+		{
+			TempFile.MoveTemporaryIntoPlace(Path, OutEc);
+		}
 	}
 }
 
diff --git a/src/zenutil/include/zenutil/basicfile.h b/src/zenutil/include/zenutil/basicfile.h
index 375da20c3..674457196 100644
--- a/src/zenutil/include/zenutil/basicfile.h
+++ b/src/zenutil/include/zenutil/basicfile.h
@@ -60,8 +60,8 @@ public:
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
 	void	 Flush();
-	[[nodiscard]] uint64_t FileSize();
-	[[nodiscard]] uint64_t FileSize(std::error_code& Ec);
+	[[nodiscard]] uint64_t FileSize() const;
+	[[nodiscard]] uint64_t FileSize(std::error_code& Ec) const;
 	void				   SetFileSize(uint64_t FileSize);
 	IoBuffer			   ReadAll();
 	void				   WriteAll(IoBuffer Data, std::error_code& Ec);
@@ -101,7 +101,8 @@ public:
 	void						 MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec);
 	const std::filesystem::path& GetPath() const { return m_TempPath; }
 
-	static void SafeWriteFile(std::filesystem::path Path, MemoryView Data);
+	static void SafeWriteFile(const std::filesystem::path& Path, MemoryView Data);
+	static void SafeWriteFile(const std::filesystem::path& Path, MemoryView Data, std::error_code& OutEc);
 
 private:
 	std::filesystem::path m_TempPath;
diff --git a/src/zenutil/include/zenutil/referencemetadata.h b/src/zenutil/include/zenutil/referencemetadata.h
new file mode 100644
index 000000000..5160bfb8b
--- /dev/null
+++ b/src/zenutil/include/zenutil/referencemetadata.h
@@ -0,0 +1,109 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compositebuffer.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct ReferenceMetaDataHeader
+{
+	static constexpr uint32_t Version1		 = 1;
+	static constexpr uint32_t CurrentVersion = Version1;
+
+	ReferenceMetaDataHeader(uint32_t InMagic, uint32_t InEntryCount, uint64_t InAttachmentCount)
+	: Magic(InMagic)
+	, EntryCount(InEntryCount)
+	, AttachmentCount(InAttachmentCount)
+	, Checksum(ComputeChecksum())
+	{
+	}
+
+	uint32_t Magic	 = 0xffffffffu;
+	uint32_t Version = CurrentVersion;
+
+	uint32_t EntryCount		 = 0;
+	uint64_t AttachmentCount = 0;
+	uint64_t Padding		 = 0;
+
+	uint32_t Checksum = 0;
+
+	bool IsValid(uint32_t ExpectedMagic) const;
+
+	template<typename KeyType, typename AttachmentType>
+	static uint64_t ExpectedSize(uint32_t EntryCount, uint64_t AttachmentCount)
+	{
+		return sizeof(ReferenceMetaDataHeader) + sizeof(KeyType) * EntryCount + sizeof(uint32_t) * EntryCount +
+			   sizeof(AttachmentType) * AttachmentCount;
+	}
+
+	ReferenceMetaDataHeader() = delete;
+
+private:
+	uint32_t ComputeChecksum() const;
+};
+
+static_assert(sizeof(ReferenceMetaDataHeader) == 32);
+
+#pragma pack(pop)
+
+template<typename KeyType, typename AttachmentType>
+CompositeBuffer
+BuildReferenceMetaData(uint32_t						   HeaderMagic,
+					   std::span<const KeyType>		   Keys,
+					   std::span<const uint32_t>	   AttachmentCounts,
+					   std::span<const AttachmentType> Attachments)
+{
+	uint32_t KeyCount = gsl::narrow<uint32_t>(Keys.size());
+
+	ReferenceMetaDataHeader Header(HeaderMagic, KeyCount, Attachments.size());
+	return CompositeBuffer(std::vector<SharedBuffer>{
+		SharedBuffer(IoBuffer(IoBuffer::Clone, &Header, sizeof(ReferenceMetaDataHeader))),
+		SharedBuffer(IoBuffer(IoBuffer::Wrap, Keys.data(), sizeof(KeyType) * Keys.size())),
+		SharedBuffer(IoBuffer(IoBuffer::Wrap, AttachmentCounts.data(), sizeof(uint32_t) * AttachmentCounts.size())),
+		SharedBuffer(IoBuffer(IoBuffer::Wrap, Attachments.data(), sizeof(AttachmentType) * Attachments.size()))});
+}
+
+template<typename KeyType, typename AttachmentType>
+bool
+GetAttachmentsFromMetaData(const IoBuffer&													  MetaData,
+						   const uint32_t													  ExpectedHeaderMagic,
+						   std::function<void(std::span<const KeyType>		  Keys,
+											  std::span<const uint32_t>		  AttachmentCounts,
+											  std::span<const AttachmentType> Attachments)>&& Parser)
+{
+	MemoryView PayloadView = MetaData.GetView();
+	if (PayloadView.GetSize() >= sizeof(ReferenceMetaDataHeader))
+	{
+		const ReferenceMetaDataHeader* Header = (const ReferenceMetaDataHeader*)PayloadView.GetData();
+		if (Header->IsValid(ExpectedHeaderMagic))
+		{
+			if (MetaData.GetSize() ==
+				ReferenceMetaDataHeader::ExpectedSize<KeyType, AttachmentType>(Header->EntryCount, Header->AttachmentCount))
+			{
+				PayloadView.MidInline(sizeof(ReferenceMetaDataHeader));
+				std::span<const KeyType> PayloadKeys = {(const KeyType*)PayloadView.GetData(), Header->EntryCount};
+				PayloadView.MidInline(sizeof(KeyType) * Header->EntryCount);
+				std::span<const uint32_t> PayloadAttachmentCounts = {(const uint32_t*)PayloadView.GetData(), Header->EntryCount};
+				PayloadView.MidInline(sizeof(uint32_t) * Header->EntryCount);
+				std::span<const AttachmentType> PayloadAttachments = {(const AttachmentType*)PayloadView.GetData(),
+																	  Header->AttachmentCount};
+				PayloadView.MidInline(sizeof(AttachmentType) * Header->AttachmentCount);
+				ZEN_ASSERT(PayloadView.GetSize() == 0);
+
+				Parser(PayloadKeys, PayloadAttachmentCounts, PayloadAttachments);
+
+				return true;
+			}
+		}
+	}
+	return false;
+}
+}  // namespace zen
diff --git a/src/zenutil/referencemetadata.cpp b/src/zenutil/referencemetadata.cpp
new file mode 100644
index 000000000..bbcafcfba
--- /dev/null
+++ b/src/zenutil/referencemetadata.cpp
@@ -0,0 +1,32 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/referencemetadata.h>
+//////////////////////////////////////////////////////////////////////////
+
+#include <zencore/xxhash.h>
+
+namespace zen {
+
+uint32_t
+ReferenceMetaDataHeader::ComputeChecksum() const
+{
+	return XXH32(&Magic, sizeof(ReferenceMetaDataHeader) - sizeof(uint32_t), 0xC0C0'BABA);
+}
+
+bool
+ReferenceMetaDataHeader::IsValid(uint32_t ExpectedMagic) const
+{
+	if (Magic != ExpectedMagic)
+	{
+		return false;
+	}
+
+	if (Checksum != ComputeChecksum())
+	{
+		return false;
+	}
+
+	return true;
+}
+
+}  // namespace zen