meta info store (#75)

- Feature: Added option `--gc-cache-attachment-store` which caches referenced attachments in cache records on disk for faster GC - default is `false`
- Feature: Added option `--gc-projectstore-attachment-store` which caches referenced attachments in project store oplogs on disk for faster GC - default is `false`

1 files changed, 127 insertions, 19 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index bc1020b58..378423100 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -20,6 +20,7 @@
 #include <zenstore/scrubcontext.h>
 #include <zenutil/cache/rpcrecording.h>
 #include <zenutil/packageformat.h>
+#include <zenutil/referencemetadata.h>
 #include <zenutil/workerpools.h>
 
 #include "fileremoteprojectstore.h"
@@ -111,6 +112,22 @@ namespace {
 		return false;
 	}
 
+	bool IsFileOlderThan(const std::filesystem::path& CheckPath, const std::filesystem::path& ReferencePath)
+	{
+		std::error_code					Ec;
+		std::filesystem::file_time_type CheckWriteTime = std::filesystem::last_write_time(CheckPath, Ec);
+		if (Ec)
+		{
+			return true;
+		}
+		std::filesystem::file_time_type ReferenceWriteTime = std::filesystem::last_write_time(ReferencePath, Ec);
+		if (Ec)
+		{
+			return true;
+		}
+		return CheckWriteTime < ReferenceWriteTime;
+	}
+
 	struct CreateRemoteStoreResult
 	{
 		std::shared_ptr<RemoteProjectStore> Store;
@@ -308,6 +325,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 	{
 		if (Exists(BasePath))
 		{
+			std::error_code DummyEc;
 			return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath));
 		}
 		return 0;
@@ -845,6 +863,7 @@ ProjectStore::Oplog::Oplog(std::string_view				Id,
 , m_CidStore(Store)
 , m_BasePath(BasePath)
 , m_MarkerPath(MarkerPath)
+, m_MetaValid(false)
 , m_OplogId(Id)
 {
 	using namespace std::literals;
@@ -852,8 +871,9 @@ ProjectStore::Oplog::Oplog(std::string_view				Id,
 	m_Storage			   = new OplogStorage(this, m_BasePath);
 	const bool StoreExists = m_Storage->Exists();
 	m_Storage->Open(/* IsCreate */ !StoreExists);
-
-	m_TempPath = m_BasePath / "temp"sv;
+	m_TempPath	= m_BasePath / "temp"sv;
+	m_MetaPath	= m_BasePath / "ops.meta"sv;
+	m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath());
 
 	CleanDirectory(m_TempPath);
 }
@@ -871,6 +891,11 @@ ProjectStore::Oplog::Flush()
 {
 	ZEN_ASSERT(m_Storage);
 	m_Storage->Flush();
+	if (!m_MetaValid)
+	{
+		std::error_code DummyEc;
+		std::filesystem::remove(m_MetaPath, DummyEc);
+	}
 }
 
 void
@@ -923,12 +948,15 @@ ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
 
 			// Actually perform some clean-up
 			RwLock::ExclusiveLockScope _(m_OplogLock);
-
 			for (const auto& Key : BadEntryKeys)
 			{
 				m_LatestOpMap.erase(Key);
 				m_Storage->AppendTombstone(Key);
 			}
+			if (!BadEntryKeys.empty())
+			{
+				m_MetaValid = false;
+			}
 		}
 		else
 		{
@@ -970,6 +998,11 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath)
 	{
 		Size += std::filesystem::file_size(StateFilePath);
 	}
+	std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv;
+	if (std::filesystem::exists(MetaFilePath))
+	{
+		Size += std::filesystem::file_size(MetaFilePath);
+	}
 
 	return Size;
 }
@@ -1097,6 +1130,7 @@ ProjectStore::Oplog::Reset()
 			m_Storage			   = new OplogStorage(this, m_BasePath);
 			const bool StoreExists = m_Storage->Exists();
 			m_Storage->Open(/* IsCreate */ !StoreExists);
+			m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath());
 			return false;
 		}
 		m_ChunkMap.clear();
@@ -1106,6 +1140,7 @@ ProjectStore::Oplog::Reset()
 		m_LatestOpMap.clear();
 		m_Storage = new OplogStorage(this, m_BasePath);
 		m_Storage->Open(true);
+		m_MetaValid = false;
 		CleanDirectory(m_TempPath);
 		Write();
 	}
@@ -1491,6 +1526,80 @@ ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Hand
 	m_Storage->ReplayLogEntries(Entries, [&](CbObjectView Op) { Handler(Op); });
 }
 
+static constexpr uint32_t OplogMetaDataExpectedMagic = 0x6f'74'6d'62;  // 'omta';
+
+void
+ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, bool StoreMetaDataOnDisk)
+{
+	if (!m_Storage)
+	{
+		return;
+	}
+
+	if (StoreMetaDataOnDisk && m_MetaValid)
+	{
+		IoBuffer MetadataPayload = IoBufferBuilder::MakeFromFile(m_MetaPath);
+		if (MetadataPayload)
+		{
+			if (GetAttachmentsFromMetaData<Oid, IoHash>(
+					MetadataPayload,
+					OplogMetaDataExpectedMagic,
+					[&](std::span<const Oid> Keys, std::span<const uint32_t> AttachmentCounts, std::span<const IoHash> Attachments) {
+						ZEN_UNUSED(Keys);
+						ZEN_UNUSED(AttachmentCounts);
+						OutAttachments.insert(OutAttachments.end(), Attachments.begin(), Attachments.end());
+					}))
+			{
+				return;
+			}
+		}
+	}
+
+	std::vector<Oid>	  Keys;
+	std::vector<uint32_t> AttachmentCounts;
+	size_t				  AttachmentOffset = OutAttachments.size();
+
+	IterateOplogLocked([&](CbObjectView Op) {
+		using namespace std::literals;
+		size_t CurrentAttachmentCount = OutAttachments.size();
+		Op.IterateAttachments([&](CbFieldView Visitor) { OutAttachments.emplace_back(Visitor.AsAttachment()); });
+		if (StoreMetaDataOnDisk)
+		{
+			XXH3_128Stream KeyHasher;
+			Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
+			XXH3_128 KeyHash128 = KeyHasher.GetHash();
+			Oid		 KeyHash;
+			memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+			Keys.push_back(KeyHash);
+			AttachmentCounts.push_back(gsl::narrow<uint32_t>(OutAttachments.size() - CurrentAttachmentCount));
+		}
+	});
+	if (StoreMetaDataOnDisk)
+	{
+		const IoHash* FirstAttachment = OutAttachments.data() + AttachmentOffset;
+		size_t		  AttachmentCount = OutAttachments.size() - AttachmentOffset;
+		IoBuffer	  MetaPayload	  = BuildReferenceMetaData<Oid>(OplogMetaDataExpectedMagic,
+															Keys,
+															AttachmentCounts,
+															std::span<const IoHash>(FirstAttachment, AttachmentCount))
+								   .Flatten()
+								   .AsIoBuffer();
+
+		const std::filesystem::path MetaPath = m_MetaPath;
+		std::error_code				Ec;
+		TemporaryFile::SafeWriteFile(MetaPath, MetaPayload.GetView(), Ec);
+		if (Ec)
+		{
+			m_MetaValid = false;
+			ZEN_WARN("Unable to set meta data for oplog '{}' at meta path: '{}'. Reason: '{}'", OplogId(), MetaPath, Ec.message());
+		}
+		else
+		{
+			m_MetaValid = true;
+		}
+	}
+}
+
 size_t
 ProjectStore::Oplog::GetOplogEntryCount() const
 {
@@ -1954,7 +2063,7 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObjectView Core)
 	using namespace std::literals;
 
 	RefPtr<OplogStorage> Storage = GetStorage();
-	if (!m_Storage)
+	if (!Storage)
 	{
 		return 0xffffffffu;
 	}
@@ -1962,11 +2071,12 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObjectView Core)
 	OplogEntryMapping		   Mapping = GetMapping(Core);
 	OplogStorage::AppendOpData OpData  = OplogStorage::GetAppendOpData(Core);
 
-	const OplogEntry OpEntry = m_Storage->AppendOp(OpData);
+	const OplogEntry OpEntry = Storage->AppendOp(OpData);
 
 	RwLock::ExclusiveLockScope OplogLock(m_OplogLock);
 	const uint32_t			   EntryId = RegisterOplogEntry(OplogLock, Mapping, OpEntry);
 	CaptureUpdatedLSNs(std::array<uint32_t, 1u>({EntryId}));
+	m_MetaValid = false;
 
 	return EntryId;
 }
@@ -1979,7 +2089,7 @@ ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores)
 	using namespace std::literals;
 
 	RefPtr<OplogStorage> Storage = GetStorage();
-	if (!m_Storage)
+	if (!Storage)
 	{
 		return std::vector<uint32_t>(Cores.size(), 0xffffffffu);
 	}
@@ -2008,6 +2118,7 @@ ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores)
 			EntryIds[OpIndex] = RegisterOplogEntry(OplogLock, Mappings[OpIndex], OpEntries[OpIndex]);
 		}
 		CaptureUpdatedLSNs(EntryIds);
+		m_MetaValid = false;
 	}
 	return EntryIds;
 }
@@ -2664,12 +2775,13 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const
 
 //////////////////////////////////////////////////////////////////////////
 
-ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue)
+ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, const Configuration& Config)
 : m_Log(logging::Get("project"))
 , m_Gc(Gc)
 , m_CidStore(Store)
 , m_JobQueue(JobQueue)
 , m_ProjectBasePath(BasePath)
+, m_Config(Config)
 , m_DiskWriteBlocker(Gc.GetDiskWriteBlocker())
 {
 	ZEN_INFO("initializing project store at '{}'", m_ProjectBasePath);
@@ -4675,10 +4787,7 @@ public:
 						 Oplog->OplogId());
 			});
 
-			Oplog->IterateOplogLocked([&](const CbObjectView& UpdateOp) -> bool {
-				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
-				return true;
-			});
+			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
 	}
 
@@ -4779,9 +4888,8 @@ public:
 			{
 				return;
 			}
-			m_Oplog->IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
-			});
+
+			m_Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
 	}
 
@@ -5021,7 +5129,7 @@ TEST_CASE("project.store.create")
 
 	std::string_view	  ProjectName("proj1"sv);
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -5050,7 +5158,7 @@ TEST_CASE("project.store.lifetimes")
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -5112,7 +5220,7 @@ TEST_CASE_TEMPLATE("project.store.export",
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -5193,7 +5301,7 @@ TEST_CASE("project.store.gc")
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		= TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
 
@@ -5477,7 +5585,7 @@ TEST_CASE("project.store.partial.read")
 	CidStore.Initialize(CidConfig);
 
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue);
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{.StoreAttachmentMetaData = true});
 	std::filesystem::path RootDir		= TempDir.Path() / "root"sv;
 	std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;