Map DDC cache key to content ID.

1 files changed, 142 insertions, 26 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 2268b5caf..9874b88ce 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -2,6 +2,7 @@
 
 #include "projectstore.h"
 
+#include <cache/structuredcachestore.h>
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryvalidation.h>
@@ -266,9 +267,14 @@ private:
 
 //////////////////////////////////////////////////////////////////////////
 
-ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath)
+ProjectStore::Oplog::Oplog(std::string_view		 Id,
+						   Project*				 Project,
+						   CidStore&			 Store,
+						   ZenCacheStore&		 CacheStore,
+						   std::filesystem::path BasePath)
 : m_OuterProject(Project)
 , m_CidStore(Store)
+, m_CacheStore(CacheStore)
 , m_BasePath(BasePath)
 , m_OplogId(Id)
 {
@@ -379,6 +385,22 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
 		return Chunk;
 	}
 
+	if (auto DerivedIt = m_DerivedDataMap.find(ChunkId); DerivedIt != m_DerivedDataMap.end())
+	{
+		const DerivedDataEntry& Entry = DerivedIt->second;
+		if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Entry.Cid))
+		{
+			Chunk.SetContentType(ZenContentType::kCompressedBinary);
+			return Chunk;
+		}
+
+		ZenCacheValue CacheValue;
+		if (m_CacheStore.Get(Entry.Namespace, Entry.Bucket, Entry.Hash, CacheValue))
+		{
+			return CacheValue.Value;
+		}
+	}
+
 	if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end())
 	{
 		std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath;
@@ -492,11 +514,6 @@ ProjectStore::Oplog::AddFileMapping(const RwLock::ExclusiveLockScope&,
 
 	m_FileMap[FileId] = std::move(Entry);
 
-	if (Hash != IoHash::Zero)
-	{
-		m_ChunkMap.insert_or_assign(FileId, Hash);
-	}
-
 	return true;
 }
 
@@ -512,6 +529,57 @@ ProjectStore::Oplog::AddMetaMapping(const RwLock::ExclusiveLockScope&, Oid Chunk
 	m_MetaMap.insert_or_assign(ChunkId, Hash);
 }
 
+bool
+ProjectStore::Oplog::AddDerivedDataMapping(const RwLock::ExclusiveLockScope&,
+										   std::string_view Namespace,
+										   std::string_view Bucket,
+										   const IoHash&	Hash,
+										   const Oid&		ValueId,
+										   const Oid&		ChunkId)
+{
+	using namespace std::literals;
+
+	if (Namespace.empty() || Bucket.empty() || Hash == IoHash::Zero)
+	{
+		return false;
+	}
+
+	if (ChunkId == Oid::Zero)
+	{
+		return false;
+	}
+
+	IoHash Cid = IoHash::Zero;
+
+	ZenCacheValue CacheValue;
+	if (m_CacheStore.Get(Namespace, Bucket, Hash, CacheValue))
+	{
+		const bool IsStructured = CacheValue.Value.GetContentType() == ZenContentType::kCbObject;
+		if (IsStructured)
+		{
+			CbObjectView RecordObj = CbObjectView(CacheValue.Value.GetData());
+			CbArrayView	 Values	   = RecordObj["Values"sv].AsArrayView();
+
+			for (CbFieldView Value : Values)
+			{
+				CbObjectView ValueObj = Value.AsObjectView();
+				const Oid	 Id		  = ValueObj["Id"sv].AsObjectId();
+
+				if (Id == ValueId)
+				{
+					Cid = ValueObj["RawHash"sv].AsHash();
+				}
+			}
+		}
+	}
+
+	m_DerivedDataMap.insert_or_assign(
+		ChunkId,
+		DerivedDataEntry{.Namespace = std::string(Namespace), .Bucket = std::string(Bucket), .Hash = Hash, .Cid = Cid, .ValueId = ValueId});
+
+	return true;
+}
+
 uint32_t
 ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate)
 {
@@ -537,6 +605,31 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 		AddChunkMapping(OplogLock, PackageId, PackageHash);
 
 		ZEN_DEBUG("package data {} -> {}", PackageId, PackageHash);
+
+		if (PkgObj["meta"])
+		{
+			CbObjectView MetaObj = PkgObj["meta"sv].AsObjectView();
+
+			for (CbFieldView& Entry : MetaObj["deriveddata"sv])
+			{
+				CbObjectView	 DerivedDataObj = Entry.AsObjectView();
+				std::string_view Namespace		= DerivedDataObj["namespace"sv].AsString(std::string_view("ue.ddc"));
+				CbObjectView	 CacheKey		= DerivedDataObj["cachekey"sv].AsObjectView();
+				std::string_view Bucket			= CacheKey["Bucket"sv].AsString();
+				const IoHash	 Hash			= CacheKey["Hash"sv].AsHash();
+				const Oid		 ValueId		= DerivedDataObj["valueid"sv].AsObjectId();
+				const Oid		 ChunkId		= DerivedDataObj["chunkid"sv].AsObjectId();
+
+				if (AddDerivedDataMapping(OplogLock, Namespace, Bucket, Hash, ValueId, ChunkId))
+				{
+					ZEN_DEBUG("derived data {} -> '{}/{}/{}'", ChunkId, Namespace, Bucket, Hash);
+				}
+				else
+				{
+					ZEN_WARN("invalid derived data reference");
+				}
+			}
+		}
 	}
 
 	for (CbFieldView& Entry : Core["bulkdata"sv])
@@ -548,7 +641,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 
 		AddChunkMapping(OplogLock, BulkDataId, BulkDataHash);
 
-		ZEN_DEBUG("bulkdata {} -> {}", BulkDataId, BulkDataHash);
+		ZEN_DEBUG("bulk data {} -> {}", BulkDataId, BulkDataHash);
 	}
 
 	if (Core["files"sv])
@@ -567,7 +660,16 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 
 			if (AddFileMapping(OplogLock, FileId, FileDataHash, ServerPath, ClientPath))
 			{
-				++FileCount;
+				if (FileDataHash != IoHash::Zero)
+				{
+					ZEN_DEBUG("file data {} -> '{}'", FileId, ClientPath);
+					++ChunkCount;
+				}
+				else
+				{
+					ZEN_DEBUG("file mapping '{}' -> '{}'", ServerPath, ClientPath);
+					++FileCount;
+				}
 			}
 			else
 			{
@@ -656,9 +758,10 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObject Core)
 
 //////////////////////////////////////////////////////////////////////////
 
-ProjectStore::Project::Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath)
+ProjectStore::Project::Project(ProjectStore* PrjStore, CidStore& CidStore, ZenCacheStore& CacheStore, std::filesystem::path BasePath)
 : m_ProjectStore(PrjStore)
-, m_CidStore(Store)
+, m_CidStore(CidStore)
+, m_CacheStore(CacheStore)
 , m_OplogStoragePath(BasePath)
 {
 }
@@ -753,9 +856,10 @@ ProjectStore::Project::NewOplog(std::string_view OplogId)
 
 	try
 	{
-		Oplog* Log =
-			m_Oplogs.try_emplace(std::string{OplogId}, std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, OplogBasePath))
-				.first->second.get();
+		Oplog* Log = m_Oplogs
+						 .try_emplace(std::string{OplogId},
+									  std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, m_CacheStore, OplogBasePath))
+						 .first->second.get();
 
 		return Log;
 	}
@@ -795,9 +899,10 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
 
 		try
 		{
-			Oplog* Log =
-				m_Oplogs.try_emplace(std::string{OplogId}, std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, OplogBasePath))
-					.first->second.get();
+			Oplog* Log = m_Oplogs
+							 .try_emplace(std::string{OplogId},
+										  std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, m_CacheStore, OplogBasePath))
+							 .first->second.get();
 
 			Log->ReplayLog();
 
@@ -947,11 +1052,12 @@ ProjectStore::Project::IsExpired() const
 
 //////////////////////////////////////////////////////////////////////////
 
-ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc)
+ProjectStore::ProjectStore(CidStore& CidStore, ZenCacheStore& CacheStore, std::filesystem::path BasePath, GcManager& Gc)
 : GcStorage(Gc)
 , GcContributor(Gc)
 , m_Log(logging::Get("project"))
-, m_CidStore(Store)
+, m_CidStore(CidStore)
+, m_CacheStore(CacheStore)
 , m_ProjectBasePath(BasePath)
 {
 	ZEN_INFO("initializing project store at '{}'", BasePath);
@@ -1189,7 +1295,8 @@ ProjectStore::OpenProject(std::string_view ProjectId)
 
 			Ref<Project>& Prj =
 				m_Projects
-					.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+					.try_emplace(std::string{ProjectId},
+								 Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, m_CacheStore, BasePath)))
 					.first->second;
 			Prj->Identifier = ProjectId;
 			Prj->Read();
@@ -1215,9 +1322,10 @@ ProjectStore::NewProject(std::filesystem::path BasePath,
 {
 	RwLock::ExclusiveLockScope _(m_ProjectsLock);
 
-	Ref<Project>& Prj =
-		m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
-			.first->second;
+	Ref<Project>& Prj = m_Projects
+							.try_emplace(std::string{ProjectId},
+										 Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, m_CacheStore, BasePath)))
+							.first->second;
 	Prj->Identifier		 = ProjectId;
 	Prj->RootDir		 = RootDir;
 	Prj->EngineRootDir	 = EngineRootDir;
@@ -2581,9 +2689,11 @@ TEST_CASE("project.store.create")
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
+	ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"});
+
 	std::string_view	  ProjectName("proj1"sv);
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc);
+	ProjectStore		  ProjectStore(CidStore, CacheStore, BasePath, Gc);
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -2610,8 +2720,10 @@ TEST_CASE("project.store.lifetimes")
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
+	ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"});
+
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc);
+	ProjectStore		  ProjectStore(CidStore, CacheStore, BasePath, Gc);
 	std::filesystem::path RootDir		  = TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
 	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
@@ -2648,8 +2760,10 @@ TEST_CASE("project.store.gc")
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
+	ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"});
+
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc);
+	ProjectStore		  ProjectStore(CidStore, CacheStore, BasePath, Gc);
 	std::filesystem::path RootDir		= TempDir.Path() / "root";
 	std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
 
@@ -2738,8 +2852,10 @@ TEST_CASE("project.store.partial.read")
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
+	ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"});
+
 	std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
-	ProjectStore		  ProjectStore(CidStore, BasePath, Gc);
+	ProjectStore		  ProjectStore(CidStore, CacheStore, BasePath, Gc);
 	std::filesystem::path RootDir		= TempDir.Path() / "root"sv;
 	std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;