From ef9856d2d9c7ed73fa0440b8267c1767abc5585d Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 23 Nov 2022 14:09:30 +0100 Subject: Map DDC cache key to content ID. --- zenserver/projectstore.cpp | 168 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 142 insertions(+), 26 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 2268b5caf..9874b88ce 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -2,6 +2,7 @@ #include "projectstore.h" +#include #include #include #include @@ -266,9 +267,14 @@ private: ////////////////////////////////////////////////////////////////////////// -ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath) +ProjectStore::Oplog::Oplog(std::string_view Id, + Project* Project, + CidStore& Store, + ZenCacheStore& CacheStore, + std::filesystem::path BasePath) : m_OuterProject(Project) , m_CidStore(Store) +, m_CacheStore(CacheStore) , m_BasePath(BasePath) , m_OplogId(Id) { @@ -379,6 +385,22 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) return Chunk; } + if (auto DerivedIt = m_DerivedDataMap.find(ChunkId); DerivedIt != m_DerivedDataMap.end()) + { + const DerivedDataEntry& Entry = DerivedIt->second; + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Entry.Cid)) + { + Chunk.SetContentType(ZenContentType::kCompressedBinary); + return Chunk; + } + + ZenCacheValue CacheValue; + if (m_CacheStore.Get(Entry.Namespace, Entry.Bucket, Entry.Hash, CacheValue)) + { + return CacheValue.Value; + } + } + if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end()) { std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; @@ -492,11 +514,6 @@ ProjectStore::Oplog::AddFileMapping(const RwLock::ExclusiveLockScope&, m_FileMap[FileId] = std::move(Entry); - if (Hash != IoHash::Zero) - { - m_ChunkMap.insert_or_assign(FileId, Hash); - } - return true; } @@ -512,6 +529,57 @@ ProjectStore::Oplog::AddMetaMapping(const RwLock::ExclusiveLockScope&, Oid Chunk m_MetaMap.insert_or_assign(ChunkId, Hash); } +bool +ProjectStore::Oplog::AddDerivedDataMapping(const RwLock::ExclusiveLockScope&, + std::string_view Namespace, + std::string_view Bucket, + const IoHash& Hash, + const Oid& ValueId, + const Oid& ChunkId) +{ + using namespace std::literals; + + if (Namespace.empty() || Bucket.empty() || Hash == IoHash::Zero) + { + return false; + } + + if (ChunkId == Oid::Zero) + { + return false; + } + + IoHash Cid = IoHash::Zero; + + ZenCacheValue CacheValue; + if (m_CacheStore.Get(Namespace, Bucket, Hash, CacheValue)) + { + const bool IsStructured = CacheValue.Value.GetContentType() == ZenContentType::kCbObject; + if (IsStructured) + { + CbObjectView RecordObj = CbObjectView(CacheValue.Value.GetData()); + CbArrayView Values = RecordObj["Values"sv].AsArrayView(); + + for (CbFieldView Value : Values) + { + CbObjectView ValueObj = Value.AsObjectView(); + const Oid Id = ValueObj["Id"sv].AsObjectId(); + + if (Id == ValueId) + { + Cid = ValueObj["RawHash"sv].AsHash(); + } + } + } + } + + m_DerivedDataMap.insert_or_assign( + ChunkId, + DerivedDataEntry{.Namespace = std::string(Namespace), .Bucket = std::string(Bucket), .Hash = Hash, .Cid = Cid, .ValueId = ValueId}); + + return true; +} + uint32_t ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate) { @@ -537,6 +605,31 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry AddChunkMapping(OplogLock, PackageId, PackageHash); ZEN_DEBUG("package data {} -> {}", PackageId, PackageHash); + + if (PkgObj["meta"]) + { + CbObjectView MetaObj = PkgObj["meta"sv].AsObjectView(); + + for (CbFieldView& Entry : MetaObj["deriveddata"sv]) + { + CbObjectView DerivedDataObj = Entry.AsObjectView(); + std::string_view Namespace = DerivedDataObj["namespace"sv].AsString(std::string_view("ue.ddc")); + CbObjectView CacheKey = DerivedDataObj["cachekey"sv].AsObjectView(); + std::string_view Bucket = CacheKey["Bucket"sv].AsString(); + const IoHash Hash = CacheKey["Hash"sv].AsHash(); + const Oid ValueId = DerivedDataObj["valueid"sv].AsObjectId(); + const Oid ChunkId = DerivedDataObj["chunkid"sv].AsObjectId(); + + if (AddDerivedDataMapping(OplogLock, Namespace, Bucket, Hash, ValueId, ChunkId)) + { + ZEN_DEBUG("derived data {} -> '{}/{}/{}'", ChunkId, Namespace, Bucket, Hash); + } + else + { + ZEN_WARN("invalid derived data reference"); + } + } + } } for (CbFieldView& Entry : Core["bulkdata"sv]) @@ -548,7 +641,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry AddChunkMapping(OplogLock, BulkDataId, BulkDataHash); - ZEN_DEBUG("bulkdata {} -> {}", BulkDataId, BulkDataHash); + ZEN_DEBUG("bulk data {} -> {}", BulkDataId, BulkDataHash); } if (Core["files"sv]) @@ -567,7 +660,16 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry if (AddFileMapping(OplogLock, FileId, FileDataHash, ServerPath, ClientPath)) { - ++FileCount; + if (FileDataHash != IoHash::Zero) + { + ZEN_DEBUG("file data {} -> '{}'", FileId, ClientPath); + ++ChunkCount; + } + else + { + ZEN_DEBUG("file mapping '{}' -> '{}'", ServerPath, ClientPath); + ++FileCount; + } } else { @@ -656,9 +758,10 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObject Core) ////////////////////////////////////////////////////////////////////////// -ProjectStore::Project::Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath) +ProjectStore::Project::Project(ProjectStore* PrjStore, CidStore& CidStore, ZenCacheStore& CacheStore, std::filesystem::path BasePath) : m_ProjectStore(PrjStore) -, m_CidStore(Store) +, m_CidStore(CidStore) +, m_CacheStore(CacheStore) , m_OplogStoragePath(BasePath) { } @@ -753,9 +856,10 @@ ProjectStore::Project::NewOplog(std::string_view OplogId) try { - Oplog* Log = - m_Oplogs.try_emplace(std::string{OplogId}, std::make_unique(OplogId, this, m_CidStore, OplogBasePath)) - .first->second.get(); + Oplog* Log = m_Oplogs + .try_emplace(std::string{OplogId}, + std::make_unique(OplogId, this, m_CidStore, m_CacheStore, OplogBasePath)) + .first->second.get(); return Log; } @@ -795,9 +899,10 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId) try { - Oplog* Log = - m_Oplogs.try_emplace(std::string{OplogId}, std::make_unique(OplogId, this, m_CidStore, OplogBasePath)) - .first->second.get(); + Oplog* Log = m_Oplogs + .try_emplace(std::string{OplogId}, + std::make_unique(OplogId, this, m_CidStore, m_CacheStore, OplogBasePath)) + .first->second.get(); Log->ReplayLog(); @@ -947,11 +1052,12 @@ ProjectStore::Project::IsExpired() const ////////////////////////////////////////////////////////////////////////// -ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc) +ProjectStore::ProjectStore(CidStore& CidStore, ZenCacheStore& CacheStore, std::filesystem::path BasePath, GcManager& Gc) : GcStorage(Gc) , GcContributor(Gc) , m_Log(logging::Get("project")) -, m_CidStore(Store) +, m_CidStore(CidStore) +, m_CacheStore(CacheStore) , m_ProjectBasePath(BasePath) { ZEN_INFO("initializing project store at '{}'", BasePath); @@ -1189,7 +1295,8 @@ ProjectStore::OpenProject(std::string_view ProjectId) Ref& Prj = m_Projects - .try_emplace(std::string{ProjectId}, Ref(new ProjectStore::Project(this, m_CidStore, BasePath))) + .try_emplace(std::string{ProjectId}, + Ref(new ProjectStore::Project(this, m_CidStore, m_CacheStore, BasePath))) .first->second; Prj->Identifier = ProjectId; Prj->Read(); @@ -1215,9 +1322,10 @@ ProjectStore::NewProject(std::filesystem::path BasePath, { RwLock::ExclusiveLockScope _(m_ProjectsLock); - Ref& Prj = - m_Projects.try_emplace(std::string{ProjectId}, Ref(new ProjectStore::Project(this, m_CidStore, BasePath))) - .first->second; + Ref& Prj = m_Projects + .try_emplace(std::string{ProjectId}, + Ref(new ProjectStore::Project(this, m_CidStore, m_CacheStore, BasePath))) + .first->second; Prj->Identifier = ProjectId; Prj->RootDir = RootDir; Prj->EngineRootDir = EngineRootDir; @@ -2581,9 +2689,11 @@ TEST_CASE("project.store.create") CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); + ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"}); + std::string_view ProjectName("proj1"sv); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc); + ProjectStore ProjectStore(CidStore, CacheStore, BasePath, Gc); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -2610,8 +2720,10 @@ TEST_CASE("project.store.lifetimes") CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); + ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"}); + std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc); + ProjectStore ProjectStore(CidStore, CacheStore, BasePath, Gc); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -2648,8 +2760,10 @@ TEST_CASE("project.store.gc") CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); + ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"}); + std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc); + ProjectStore ProjectStore(CidStore, CacheStore, BasePath, Gc); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -2738,8 +2852,10 @@ TEST_CASE("project.store.partial.read") CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); + ZenCacheStore CacheStore(Gc, {.BasePath = TempDir.Path() / "cache"}); + std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc); + ProjectStore ProjectStore(CidStore, CacheStore, BasePath, Gc); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; -- cgit v1.2.3