aboutsummaryrefslogtreecommitdiff
path: root/zenserver
diff options
context:
space:
mode:
Diffstat (limited to 'zenserver')
-rw-r--r--zenserver/projectstore.cpp121
-rw-r--r--zenserver/projectstore.h28
-rw-r--r--zenserver/zenserver.cpp11
3 files changed, 137 insertions, 23 deletions
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index ec2dcb503..124d10b65 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -19,6 +19,8 @@
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
+#include "cache/structuredcachestore.h"
+
#include "config.h"
#if ZEN_PLATFORM_WINDOWS
@@ -306,9 +308,14 @@ private:
//////////////////////////////////////////////////////////////////////////
-ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath)
+ProjectStore::Oplog::Oplog(std::string_view Id,
+ Project* Project,
+ CidStore& Store,
+ ZenCacheStore& CacheStore,
+ std::filesystem::path BasePath)
: m_OuterProject(Project)
, m_CidStore(Store)
+, m_CacheStore(CacheStore)
, m_BasePath(BasePath)
, m_OplogId(Id)
{
@@ -389,6 +396,30 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
return Chunk;
}
+ if (auto CacheIt = m_CacheMap.find(ChunkId); CacheIt != m_CacheMap.end())
+ {
+ CacheMapEntry Entry = CacheIt->second;
+
+ _.ReleaseNow();
+
+ if (Entry.Cid != IoHash::Zero)
+ {
+ IoBuffer Chunk = m_CidStore.FindChunkByCid(Entry.Cid);
+ Chunk.SetContentType(ZenContentType::kCompressedBinary);
+
+ return Chunk;
+ }
+
+ if (IoHash Cid = ResolveCacheMapping(Entry.Key, Entry.ValueId); Cid != IoHash::Zero)
+ {
+ CacheMapEntry ResolvedEntry = Entry;
+ ResolvedEntry.Cid = Cid;
+
+ RwLock::ExclusiveLockScope __(m_OplogLock);
+ m_CacheMap.insert_or_assign(ChunkId, ResolvedEntry);
+ }
+ }
+
if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end())
{
_.ReleaseNow();
@@ -505,6 +536,51 @@ ProjectStore::Oplog::AddMetaMapping(Oid ChunkId, IoHash Hash)
m_MetaMap.insert_or_assign(ChunkId, Hash);
}
+IoHash
+ProjectStore::Oplog::AddCacheMapping(const Oid& ChunkId, const CacheKey& Key, const Oid& ValueId)
+{
+ // NOTE: Caller must hold an exclusive lock on m_OplogLock
+
+ const IoHash Cid = ResolveCacheMapping(Key, ValueId);
+ auto Entry = CacheMapEntry{.Key = Key, .ValueId = ValueId, .Cid = Cid};
+
+ m_CacheMap.insert_or_assign(ChunkId, Entry);
+
+ return Cid;
+}
+
+IoHash
+ProjectStore::Oplog::ResolveCacheMapping(const CacheKey& Key, const Oid& ValueId)
+{
+ using namespace std::literals;
+
+ auto GetCidFromValueId = [](const Oid& ValueId, CbObjectView Record) -> IoHash {
+ CbArrayView Values = Record["Values"sv].AsArrayView();
+
+ for (CbFieldView Value : Values)
+ {
+ CbObjectView ValueObject = Value.AsObjectView();
+ if (ValueObject["Id"sv].AsObjectId() == ValueId)
+ {
+ return ValueObject["RawHash"sv].AsHash();
+ }
+ }
+
+ return IoHash::Zero;
+ };
+
+ if (ValueId)
+ {
+ ZenCacheValue CacheValue;
+ if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue))
+ {
+ return GetCidFromValueId(ValueId, CbObjectView{CacheValue.Value.GetData()});
+ }
+ }
+
+ return IoHash::Zero;
+}
+
uint32_t
ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate)
{
@@ -586,12 +662,30 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
{
CbObjectView DerivedObj = Entry.AsObjectView();
- Oid DerivedDataId = DerivedObj["id"sv].AsObjectId();
- IoHash DerivedDataHash = DerivedObj["data"sv].AsBinaryAttachment();
+ Oid DerivedDataId = DerivedObj["id"sv].AsObjectId();
+ CbObjectView CacheKeyObj = DerivedObj["cachekey"sv].AsObjectView();
+ IoHash DerivedDataHash = DerivedObj["data"sv].AsBinaryAttachment();
- AddChunkMapping(DerivedDataId, DerivedDataHash);
+ if (DerivedDataHash != IoHash::Zero)
+ {
+ AddChunkMapping(DerivedDataId, DerivedDataHash);
+ ZEN_DEBUG("derived data {} -> {}", DerivedDataId, DerivedDataHash);
+ }
+ else
+ {
+ CbObjectView KeyObj = CacheKeyObj["CacheKey"sv].AsObjectView();
+ CacheKey Key = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"sv].AsHash());
+ Oid ValueId = CacheKeyObj["ValueId"sv].AsObjectId();
- ZEN_DEBUG("derived data {} -> {}", DerivedDataId, DerivedDataHash);
+ if (IoHash Cid = AddCacheMapping(DerivedDataId, Key, ValueId); Cid != IoHash::Zero)
+ {
+ ZEN_DEBUG("derived data {} -> {}", DerivedDataId, Cid);
+ }
+ else
+ {
+ ZEN_DEBUG("derived data {} -> {}/{}/{} (UNRESOLVED)", DerivedDataId, Key.Bucket, Key.Hash, ValueId);
+ }
+ }
}
m_OpAddressMap.emplace(OpEntry.OpLsn, OplogEntryAddress{.Offset = OpEntry.OpCoreOffset, .Size = OpEntry.OpCoreSize});
@@ -644,9 +738,10 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage)
//////////////////////////////////////////////////////////////////////////
-ProjectStore::Project::Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath)
+ProjectStore::Project::Project(ProjectStore* PrjStore, CidStore& Store, ZenCacheStore& CacheStore, std::filesystem::path BasePath)
: m_ProjectStore(PrjStore)
, m_CidStore(Store)
+, m_CacheStore(CacheStore)
, m_OplogStoragePath(BasePath)
{
}
@@ -735,7 +830,7 @@ ProjectStore::Project::NewOplog(std::string_view OplogId)
try
{
- Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CidStore, OplogBasePath).first->second;
+ Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CidStore, m_CacheStore, OplogBasePath).first->second;
return &Log;
}
@@ -775,7 +870,7 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
try
{
- Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CidStore, OplogBasePath).first->second;
+ Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CidStore, m_CacheStore, OplogBasePath).first->second;
Log.ReplayLog();
@@ -888,10 +983,11 @@ ProjectStore::Project::GatherReferences(GcContext& GcCtx)
//////////////////////////////////////////////////////////////////////////
-ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, CasGc& Gc)
+ProjectStore::ProjectStore(CidStore& Store, ZenCacheStore& CacheStore, std::filesystem::path BasePath, CasGc& Gc)
: GcContributor(Gc)
, m_Log(zen::logging::Get("project"))
, m_CidStore(Store)
+, m_CacheStore(CacheStore)
, m_ProjectBasePath(BasePath)
{
ZEN_INFO("initializing project store at '{}'", BasePath);
@@ -1024,8 +1120,9 @@ ProjectStore::OpenProject(std::string_view ProjectId)
{
ZEN_INFO("opening project {} @ {}", ProjectId, ProjectBasePath);
- ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CidStore, ProjectBasePath).first->second;
- Prj.Identifier = ProjectId;
+ ProjectStore::Project& Prj =
+ m_Projects.try_emplace(std::string{ProjectId}, this, m_CidStore, m_CacheStore, ProjectBasePath).first->second;
+ Prj.Identifier = ProjectId;
Prj.Read();
return &Prj;
}
@@ -1048,7 +1145,7 @@ ProjectStore::NewProject(std::filesystem::path BasePath,
{
RwLock::ExclusiveLockScope _(m_ProjectsLock);
- ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CidStore, BasePath).first->second;
+ ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CidStore, m_CacheStore, BasePath).first->second;
Prj.Identifier = ProjectId;
Prj.RootDir = RootDir;
Prj.EngineRootDir = EngineRootDir;
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index f71434783..e7985eaa7 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -10,6 +10,7 @@
#include <zenstore/caslog.h>
#include <zenstore/cidstore.h>
#include <zenstore/gc.h>
+#include <zenutil/cache/cache.h>
#include <filesystem>
#include <map>
@@ -23,6 +24,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
class CbPackage;
+class ZenCacheStore;
struct OplogEntry
{
@@ -55,14 +57,14 @@ class ProjectStore : public RefCounted, public GcContributor
struct OplogStorage;
public:
- ProjectStore(CidStore& Store, std::filesystem::path BasePath, CasGc& Gc);
+ ProjectStore(CidStore& Store, ZenCacheStore& CacheStore, std::filesystem::path BasePath, CasGc& Gc);
~ProjectStore();
struct Project;
struct Oplog
{
- Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath);
+ Oplog(std::string_view Id, Project* Project, CidStore& Store, ZenCacheStore& CacheStore, std::filesystem::path BasePath);
~Oplog();
[[nodiscard]] static bool ExistsAt(std::filesystem::path BasePath);
@@ -117,11 +119,19 @@ public:
std::string ClientPath;
};
+ struct CacheMapEntry
+ {
+ CacheKey Key;
+ Oid ValueId;
+ IoHash Cid;
+ };
+
template<class V>
using OidMap = tsl::robin_map<Oid, V, Oid::Hasher>;
Project* m_OuterProject = nullptr;
CidStore& m_CidStore;
+ ZenCacheStore& m_CacheStore;
std::filesystem::path m_BasePath;
std::filesystem::path m_TempPath;
@@ -129,6 +139,7 @@ public:
OidMap<IoHash> m_ChunkMap; // output data chunk id -> CAS address
OidMap<IoHash> m_MetaMap; // meta chunk id -> CAS address
OidMap<FileMapEntry> m_FileMap; // file id -> file map entry
+ OidMap<CacheMapEntry> m_CacheMap; // chunk id -> cache map entry
int32_t m_ManifestVersion; // File system manifest version
std::map<int, OplogEntryAddress> m_OpAddressMap; // Index LSN -> op data in ops blob file
OidMap<int> m_LatestOpMap; // op key -> latest op LSN for key
@@ -136,9 +147,12 @@ public:
RefPtr<OplogStorage> m_Storage;
std::string m_OplogId;
- bool AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath);
- void AddChunkMapping(Oid ChunkId, IoHash Hash);
- void AddMetaMapping(Oid ChunkId, IoHash Hash);
+ bool AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath);
+ void AddChunkMapping(Oid ChunkId, IoHash Hash);
+ void AddMetaMapping(Oid ChunkId, IoHash Hash);
+ IoHash AddCacheMapping(const Oid& ChunkId, const CacheKey& Key, const Oid& ValueId);
+
+ IoHash ResolveCacheMapping(const CacheKey& Key, const Oid& ValueId);
};
struct Project
@@ -155,7 +169,7 @@ public:
void IterateOplogs(std::function<void(Oplog&)>&& Fn);
void DiscoverOplogs();
- Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath);
+ Project(ProjectStore* PrjStore, CidStore& Store, ZenCacheStore& CacheStore, std::filesystem::path BasePath);
~Project();
void Read();
@@ -169,6 +183,7 @@ public:
private:
ProjectStore* m_ProjectStore;
CidStore& m_CidStore;
+ ZenCacheStore& m_CacheStore;
mutable RwLock m_ProjectLock;
std::map<std::string, Oplog> m_Oplogs;
std::filesystem::path m_OplogStoragePath;
@@ -199,6 +214,7 @@ public:
private:
spdlog::logger& m_Log;
CidStore& m_CidStore;
+ ZenCacheStore& m_CacheStore;
std::filesystem::path m_ProjectBasePath;
RwLock m_ProjectsLock;
std::map<std::string, Project> m_Projects;
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index abaec888a..e54dc15f2 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -263,11 +263,6 @@ public:
m_CidStore = std::make_unique<zen::CidStore>(*m_CasStore, m_DataRoot / "cid");
m_CasGc.SetCidStore(m_CidStore.get());
- ZEN_INFO("instantiating project service");
-
- m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects", m_CasGc);
- m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore});
-
#if ZEN_USE_NAMED_PIPES
m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore);
#endif
@@ -308,6 +303,12 @@ public:
{
ZEN_INFO("NOT instantiating structured cache service");
}
+
+ ZEN_INFO("instantiating project service");
+
+ m_ProjectStore = new zen::ProjectStore(*m_CidStore, *m_CacheStore, m_DataRoot / "projects", m_CasGc);
+ m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore});
+
#if ZEN_ENABLE_MESH
if (ServerOptions.MeshEnabled)
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-25 07:48:26 +0000