diff options
| author | Dan Engelbrecht <[email protected]> | 2025-08-20 12:33:03 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-08-20 12:33:03 +0200 |
| commit | 4c05d1041461b630cd5770dae5e8d03147d5674b (patch) | |
| tree | 3f5d6b1b4b2b3f167f94e98f902a5f60c2e3d753 /src/zenserver/projectstore | |
| parent | zen print fixes/improvements (#469) (diff) | |
| download | zen-4c05d1041461b630cd5770dae5e8d03147d5674b.tar.xz zen-4c05d1041461b630cd5770dae5e8d03147d5674b.zip | |
per namespace/project cas prep refactor (#470)
- Refactor so we can have more than one cas store for project store and cache.
- Refactor `UpstreamCacheClient` so it is not tied to a specific CidStore
- Refactor scrub to keep the GC interface ScrubStorage function separate from scrub accessor functions (renamed to Scrub).
- Refactor storage size to keep GC interface StorageSize function separate from size accessor functions (renamed to TotalSize)
- Refactor cache storage so `ZenCacheDiskLayer::CacheBucket` implements GcStorage interface rather than `ZenCacheNamespace`
Diffstat (limited to 'src/zenserver/projectstore')
| -rw-r--r-- | src/zenserver/projectstore/httpprojectstore.cpp | 167 | ||||
| -rw-r--r-- | src/zenserver/projectstore/httpprojectstore.h | 7 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 184 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.h | 13 |
4 files changed, 265 insertions, 106 deletions
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 317a419eb..9600133f3 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -235,13 +235,11 @@ namespace { ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(CidStore& Store, - ProjectStore* Projects, +HttpProjectService::HttpProjectService(ProjectStore* Projects, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr) : m_Log(logging::Get("project")) -, m_CidStore(Store) , m_ProjectStore(Projects) , m_StatusService(StatusService) , m_StatsService(StatsService) @@ -407,8 +405,45 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { ZEN_TRACE_CPU("ProjectService::Stats"); - const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); - const CidStoreSize CidSize = m_CidStore.TotalSize(); + bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true"; + + const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); + uint64_t TotalChunkHitCount = 0; + uint64_t TotalChunkMissCount = 0; + uint64_t TotalChunkWriteCount = 0; + CidStoreSize TotalCidSize; + + tsl::robin_map<CidStore*, std::string> UniqueStores; + { + m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) { + CidStore* Store = &Project.GetCidStore(); + if (auto It = UniqueStores.find(Store); It == UniqueStores.end()) + { + UniqueStores.insert_or_assign(Store, Project.Identifier); + } + else + { + UniqueStores.insert_or_assign(Store, std::string{}); + } + }); + + for (auto It : UniqueStores) + { + CidStore* ChunkStore = It.first; + + CidStoreStats ChunkStoreStats = ChunkStore->Stats(); + CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); + + TotalChunkHitCount += ChunkStoreStats.HitCount; + TotalChunkMissCount += ChunkStoreStats.MissCount; + TotalChunkWriteCount += ChunkStoreStats.WriteCount; + + TotalCidSize.TinySize += ChunkStoreSize.TinySize; + TotalCidSize.SmallSize += ChunkStoreSize.SmallSize; + TotalCidSize.LargeSize += ChunkStoreSize.LargeSize; + TotalCidSize.TotalSize += ChunkStoreSize.TotalSize; + } + } CbObjectWriter Cbo; @@ -460,12 +495,66 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { Cbo.BeginObject("size"); { - Cbo << "tiny" << CidSize.TinySize; - Cbo << "small" << CidSize.SmallSize; - Cbo << "large" << CidSize.LargeSize; - Cbo << "total" << CidSize.TotalSize; + Cbo << "tiny" << TotalCidSize.TinySize; + Cbo << "small" << TotalCidSize.SmallSize; + Cbo << "large" << TotalCidSize.LargeSize; + Cbo << "total" << TotalCidSize.TotalSize; } Cbo.EndObject(); + + if (ShowCidStoreStats) + { + Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount; + const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount; + Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0); + + Cbo.BeginObject("store"); + + auto OutputStats = [&](CidStore& ChunkStore) { + CidStoreStats StoreStats = ChunkStore.Stats(); + Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount; + const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount; + Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0); + EmitSnapshot("read", StoreStats.FindChunkOps, Cbo); + EmitSnapshot("write", StoreStats.AddChunkOps, Cbo); + }; + + if (UniqueStores.size() > 1) + { + Cbo.BeginArray("projects"); + for (auto It : UniqueStores) + { + CidStore* ChunkStore = It.first; + const std::string& ProjectId = It.second; + CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); + + Cbo.BeginObject(); + { + Cbo << "project" << ProjectId; + Cbo.BeginObject("stats"); + OutputStats(*ChunkStore); + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + Cbo << "tiny" << ChunkStoreSize.TinySize; + Cbo << "small" << ChunkStoreSize.SmallSize; + Cbo << "large" << ChunkStoreSize.LargeSize; + Cbo << "total" << ChunkStoreSize.TotalSize; + } + Cbo.EndObject(); + } + Cbo.EndObject(); + } + Cbo.EndArray(); // projects + } + else if (UniqueStores.size() != 0) + { + CidStore& ChunkStore = *UniqueStores.begin()->first; + OutputStats(ChunkStore); + } + Cbo.EndObject(); + } } Cbo.EndObject(); @@ -1125,6 +1214,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; IoBuffer Payload = HttpReq.ReadPayload(); @@ -1137,7 +1228,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) std::vector<IoHash> MissingChunks; CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { - if (m_CidStore.ContainsChunk(Hash)) + if (ChunkStore.ContainsChunk(Hash)) { // Return null attachment as we already have it, no point in reading it and storing it again return {}; @@ -1393,6 +1484,8 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString)) @@ -1407,7 +1500,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) Op.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsAttachment(); - IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash); + IoBuffer Payload = ChunkStore.FindChunkByCid(AttachmentHash); if (Payload) { switch (Payload.GetContentType()) @@ -2036,11 +2129,14 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) CSVHeader(Details, AttachmentDetails, CSVWriter); m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { + CidStore& ChunkStore = Project.GetCidStore(); + Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); }); }); @@ -2054,8 +2150,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) m_ProjectStore->DiscoverProjects(); m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { - std::vector<std::string> OpLogs = Project.ScanForOplogs(); - CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + CidStore& ChunkStore = Project.GetCidStore(); + std::vector<std::string> OpLogs = Project.ScanForOplogs(); + CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); }); } Cbo.EndArray(); @@ -2084,7 +2181,8 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Project& Project = *FoundProject.Get(); + CidStore& ChunkStore = Project.GetCidStore(); if (CSV) { @@ -2092,10 +2190,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) CSVHeader(Details, AttachmentDetails, CSVWriter); FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } @@ -2105,7 +2204,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) std::vector<std::string> OpLogs = FoundProject->ScanForOplogs(); Cbo.BeginArray("projects"); { - CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); @@ -2141,16 +2240,17 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Project& Project = *FoundProject.Get(); - ProjectStore::Oplog& Oplog = *FoundLog; + ProjectStore::Project& Project = *FoundProject.Get(); + CidStore& ChunkStore = Project.GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; if (CSV) { ExtendableStringBuilder<4096> CSVWriter; CSVHeader(Details, AttachmentDetails, CSVWriter); Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + [this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } @@ -2159,7 +2259,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) CbObjectWriter Cbo; Cbo.BeginArray("oplogs"); { - CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); + CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); @@ -2204,9 +2304,10 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId)); } - const Oid ObjId = Oid::FromHexString(OpId); - ProjectStore::Project& Project = *FoundProject.Get(); - ProjectStore::Oplog& Oplog = *FoundLog; + const Oid ObjId = Oid::FromHexString(OpId); + ProjectStore::Project& Project = *FoundProject.Get(); + CidStore& ChunkStore = Project.GetCidStore(); + ProjectStore::Oplog& Oplog = *FoundLog; std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId); if (!Op.has_value()) @@ -2224,7 +2325,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) ExtendableStringBuilder<4096> CSVWriter; CSVHeader(Details, AttachmentDetails, CSVWriter); - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter); + CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } else @@ -2232,7 +2333,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) CbObjectWriter Cbo; Cbo.BeginArray("ops"); { - CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo); + CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 295defa5c..5782188e6 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -35,11 +35,7 @@ class ProjectStore; class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(CidStore& Store, - ProjectStore* InProjectStore, - HttpStatusService& StatusService, - HttpStatsService& StatsService, - AuthMgr& AuthMgr); + HttpProjectService(ProjectStore* InProjectStore, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr); ~HttpProjectService(); virtual const char* BaseUri() const override; @@ -92,7 +88,6 @@ private: inline LoggerRef Log() { return m_Log; } LoggerRef m_Log; - CidStore& m_CidStore; HttpRequestRouter m_Router; Ref<ProjectStore> m_ProjectStore; HttpStatusService& m_StatusService; diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 61c279afa..d6dd6ef9b 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1176,7 +1176,7 @@ ProjectStore::Oplog::Flush() } void -ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx) +ProjectStore::Oplog::Scrub(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetProjectstoreTag()); @@ -3575,7 +3575,7 @@ ProjectStore::Project::Flush() } void -ProjectStore::Project::ScrubStorage(ScrubContext& Ctx) +ProjectStore::Project::Scrub(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetProjectstoreTag()); // Scrubbing needs to check all existing oplogs @@ -3587,7 +3587,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx) IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) { if (!IsExpired(GcClock::TimePoint::min(), Ops)) { - Ops.ScrubStorage(Ctx); + Ops.Scrub(Ctx); } }); } @@ -3832,7 +3832,7 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const ////////////////////////////////////////////////////////////////////////// -ProjectStore::ProjectStore(CidStore& Store, +ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, @@ -3840,7 +3840,7 @@ ProjectStore::ProjectStore(CidStore& Store, const Configuration& Config) : m_Log(logging::Get("project")) , m_Gc(Gc) -, m_CidStore(Store) +, m_GetCidStore(std::move(GetCidStore)) , m_JobQueue(JobQueue) , m_OpenProcessCache(InOpenProcessCache) , m_ProjectBasePath(BasePath) @@ -3973,7 +3973,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx) } for (const Ref<Project>& Project : Projects) { - Project->ScrubStorage(Ctx); + Project->Scrub(Ctx); } } @@ -4025,6 +4025,8 @@ ProjectStore::OpenProject(std::string_view ProjectId) } } + CidStore& ChunkStore = m_GetCidStore(ProjectId); + RwLock::ExclusiveLockScope _(m_ProjectsLock); if (auto ProjIt = m_Projects.find(std::string{ProjectId}); ProjIt != m_Projects.end()) { @@ -4041,7 +4043,7 @@ ProjectStore::OpenProject(std::string_view ProjectId) Ref<Project>& Prj = m_Projects - .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) + .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath))) .first->second; Prj->Identifier = ProjectId; Prj->Read(); @@ -4068,12 +4070,14 @@ ProjectStore::NewProject(const std::filesystem::path& BasePath, ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Store::NewProject"); + CidStore& ChunkStore = m_GetCidStore(ProjectId); + RwLock::ExclusiveLockScope _(m_ProjectsLock); ZEN_INFO("project '{}': creating project at '{}'", ProjectId, BasePath); Ref<Project>& Prj = - m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) + m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath))) .first->second; Prj->Identifier = ProjectId; Prj->RootDir = RootDir; @@ -4802,7 +4806,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } const IoHash Hash = IoHash::FromHexString(Cid); - OutChunk = m_CidStore.FindChunkByCid(Hash); + OutChunk = Project->GetCidStore().FindChunkByCid(Hash); if (!OutChunk) { @@ -4865,7 +4869,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId, } FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash}); - CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash); + CidStore::InsertResult Result = Project->GetCidStore().AddChunk(Chunk, Hash); return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}}; } @@ -4894,18 +4898,19 @@ ProjectStore::GetChunks(const std::string_view ProjectId, } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + if (RequestObject["chunks"sv].IsArray()) { // Legacy full chunks only by rawhash - CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); - + CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); CbObjectWriter ResponseWriter; ResponseWriter.BeginArray("chunks"sv); for (CbFieldView FieldView : ChunksArray) { IoHash RawHash = FieldView.AsHash(); - IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); + IoBuffer ChunkBuffer = ChunkStore.FindChunkByCid(RawHash); if (ChunkBuffer) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)); @@ -5057,7 +5062,7 @@ ProjectStore::GetChunks(const std::string_view ProjectId, if (ChunkRequest.Input.Id.index() == 0) { const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id); - IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash); + IoBuffer Payload = ChunkStore.FindChunkByCid(ChunkHash); if (Payload) { ChunkRequest.Output.Exists = true; @@ -5244,7 +5249,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie return {HttpResponseCode::BadRequest, "Invalid payload format"}; } - CidStore& ChunkStore = m_CidStore; + CidStore& ChunkStore = Project->GetCidStore(); RwLock AttachmentsLock; tsl::robin_set<IoHash, IoHash::Hasher> Attachments; @@ -5350,7 +5355,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId, } } - CidStore& ChunkStore = m_CidStore; + CidStore& ChunkStore = Project->GetCidStore(); RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( ChunkStore, @@ -5462,6 +5467,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, } Project->TouchOplog(OplogId); + CidStore& ChunkStore = Project->GetCidStore(); + if (Method == "import"sv) { if (!AreDiskWritesAllowed()) @@ -5543,7 +5550,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, } Oplog->CaptureAddedAttachments(WriteRawHashes); - m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); + ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); } HttpReq.WriteResponse(HttpResponseCode::OK); return true; @@ -5716,14 +5723,14 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, ResponseObj.EndArray(); } - // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new - // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have + // Ops that have moved chunks to a compressed buffer for storage in ChunkStore have been rewritten with references to the new + // chunk(s). Make sure we add the chunks to ChunkStore, and do it after we update the oplog so GC doesn't think we have // unreferenced chunks. for (auto It : AddedChunks) { const IoHash& RawHash = It.first; AddedChunk& Chunk = It.second; - CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash); + CidStore::InsertResult Result = ChunkStore.AddChunk(Chunk.Buffer, RawHash); if (Result.New) { InlinedBytes += Chunk.RawSize; @@ -5786,7 +5793,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op EmbedLooseFile, Force, IgnoreMissingAttachments](JobContext& Context) { - RemoteProjectStore::Result Result = SaveOplog(m_CidStore, + RemoteProjectStore::Result Result = SaveOplog(Project->GetCidStore(), *ActualRemoteStore, *Project.Get(), *OplogPtr, @@ -5831,14 +5838,20 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog, } std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + CidStore& ChunkStore = Project.GetCidStore(); ZEN_INFO("Loading oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description); JobId JobId = m_JobQueue.QueueJob( fmt::format("Import oplog '{}/{}'", Project.Identifier, Oplog.OplogId()), - [this, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, Force, IgnoreMissingAttachments, CleanOplog]( - JobContext& Context) { + [this, + ChunkStore = &ChunkStore, + ActualRemoteStore = std::move(RemoteStore), + OplogPtr = &Oplog, + Force, + IgnoreMissingAttachments, + CleanOplog](JobContext& Context) { RemoteProjectStore::Result Result = - LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context); + LoadOplog(*ChunkStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context); auto Response = ConvertResult(Result); ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); if (!IsHttpSuccessCode(Response.first)) @@ -6877,6 +6890,11 @@ namespace testutils { return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags); } + ProjectStore::GetCidStoreFunc SingleChunkStore(CidStore& ChunkStore) + { + return [ChunkStore = &ChunkStore](std::string_view) -> CidStore& { return *ChunkStore; }; + } + } // namespace testutils TEST_CASE("project.opkeys") @@ -6939,13 +6957,18 @@ TEST_CASE("project.store.create") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::string_view ProjectName("proj1"sv); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -6970,12 +6993,17 @@ TEST_CASE("project.store.lifetimes") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -7033,12 +7061,17 @@ TEST_CASE_TEMPLATE("project.store.export", auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -7072,7 +7105,7 @@ TEST_CASE_TEMPLATE("project.store.export", std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options); RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - RemoteProjectStore::Result ExportResult = SaveOplog(CidStore, + RemoteProjectStore::Result ExportResult = SaveOplog(ChunkStore, *RemoteStore, *Project.Get(), *Oplog, @@ -7089,7 +7122,7 @@ TEST_CASE_TEMPLATE("project.store.export", ProjectStore::Oplog* OplogImport = Project->NewOplog("oplog2", {}); CHECK(OplogImport != nullptr); - RemoteProjectStore::Result ImportResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ false, @@ -7098,7 +7131,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportResult.ErrorCode == 0); - RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportForceResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ true, @@ -7107,7 +7140,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportForceResult.ErrorCode == 0); - RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportCleanResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ false, @@ -7116,7 +7149,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportCleanResult.ErrorCode == 0); - RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore, + RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(ChunkStore, *RemoteStore, *OplogImport, /*Force*/ true, @@ -7136,12 +7169,17 @@ TEST_CASE("project.store.gc") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -7337,12 +7375,17 @@ TEST_CASE("project.store.gc.prep") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -7385,7 +7428,7 @@ TEST_CASE("project.store.gc.prep") // Equivalent of a `prep` existance check call for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7399,7 +7442,7 @@ TEST_CASE("project.store.gc.prep") // If a gc comes in between our prep and op write the chunks will be removed for (auto Attachment : OpAttachments) { - CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7419,7 +7462,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7433,7 +7476,7 @@ TEST_CASE("project.store.gc.prep") // Attachments should now be retained for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7447,7 +7490,7 @@ TEST_CASE("project.store.gc.prep") // Attachments should now be retained across multiple GCs if retain time is still valud for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7459,7 +7502,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); @@ -7476,7 +7519,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7509,7 +7552,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } Sleep(200); @@ -7524,7 +7567,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } // This pass the retention time has expired and the last GC pass cleared the entries @@ -7538,7 +7581,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); } } @@ -7552,12 +7595,17 @@ TEST_CASE("project.store.rpc.getchunks") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; @@ -8474,12 +8522,17 @@ TEST_CASE("project.store.partial.read") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; @@ -8652,12 +8705,17 @@ TEST_CASE("project.store.iterateoplog") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore CidStore(Gc); + CidStore ChunkStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); + ChunkStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), + BasePath, + Gc, + *JobQueue, + ProcessCache, + ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv"; diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 368da5ea4..eb27665f9 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -68,7 +68,9 @@ public: { }; - ProjectStore(CidStore& Store, + typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc; + + ProjectStore(GetCidStoreFunc&& GetCidStore, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, @@ -156,7 +158,7 @@ public: LoggerRef Log() { return m_OuterProject->Log(); } void Flush(); - void ScrubStorage(ScrubContext& Ctx); + void Scrub(ScrubContext& Ctx); static uint64_t TotalSize(const std::filesystem::path& BasePath); uint64_t TotalSize() const; @@ -326,11 +328,13 @@ public: Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath); virtual ~Project(); + CidStore& GetCidStore() { return m_CidStore; }; + void Read(); void Write(); [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath); void Flush(); - void ScrubStorage(ScrubContext& Ctx); + void Scrub(ScrubContext& Ctx); LoggerRef Log() const; static uint64_t TotalSize(const std::filesystem::path& BasePath); uint64_t TotalSize() const; @@ -405,6 +409,7 @@ public: LoggerRef Log() { return m_Log; } const std::filesystem::path& BasePath() const { return m_ProjectBasePath; } + // GcStorage virtual void ScrubStorage(ScrubContext& Ctx) override; virtual GcStorageSize StorageSize() const override; @@ -498,7 +503,7 @@ public: private: LoggerRef m_Log; GcManager& m_Gc; - CidStore& m_CidStore; + GetCidStoreFunc m_GetCidStore; JobQueue& m_JobQueue; OpenProcessCache& m_OpenProcessCache; std::filesystem::path m_ProjectBasePath; |