aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/projectstore
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-08-20 12:33:03 +0200
committerGitHub Enterprise <[email protected]>2025-08-20 12:33:03 +0200
commit4c05d1041461b630cd5770dae5e8d03147d5674b (patch)
tree3f5d6b1b4b2b3f167f94e98f902a5f60c2e3d753 /src/zenserver/projectstore
parentzen print fixes/improvements (#469) (diff)
downloadzen-4c05d1041461b630cd5770dae5e8d03147d5674b.tar.xz
zen-4c05d1041461b630cd5770dae5e8d03147d5674b.zip
per namespace/project cas prep refactor (#470)
- Refactor so we can have more than one cas store for project store and cache. - Refactor `UpstreamCacheClient` so it is not tied to a specific CidStore - Refactor scrub to keep the GC interface ScrubStorage function separate from scrub accessor functions (renamed to Scrub). - Refactor storage size to keep GC interface StorageSize function separate from size accessor functions (renamed to TotalSize) - Refactor cache storage so `ZenCacheDiskLayer::CacheBucket` implements GcStorage interface rather than `ZenCacheNamespace`
Diffstat (limited to 'src/zenserver/projectstore')
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp167
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h7
-rw-r--r--src/zenserver/projectstore/projectstore.cpp184
-rw-r--r--src/zenserver/projectstore/projectstore.h13
4 files changed, 265 insertions, 106 deletions
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 317a419eb..9600133f3 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -235,13 +235,11 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store,
- ProjectStore* Projects,
+HttpProjectService::HttpProjectService(ProjectStore* Projects,
HttpStatusService& StatusService,
HttpStatsService& StatsService,
AuthMgr& AuthMgr)
: m_Log(logging::Get("project"))
-, m_CidStore(Store)
, m_ProjectStore(Projects)
, m_StatusService(StatusService)
, m_StatsService(StatsService)
@@ -407,8 +405,45 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
ZEN_TRACE_CPU("ProjectService::Stats");
- const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
- const CidStoreSize CidSize = m_CidStore.TotalSize();
+ bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true";
+
+ const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+ uint64_t TotalChunkHitCount = 0;
+ uint64_t TotalChunkMissCount = 0;
+ uint64_t TotalChunkWriteCount = 0;
+ CidStoreSize TotalCidSize;
+
+ tsl::robin_map<CidStore*, std::string> UniqueStores;
+ {
+ m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) {
+ CidStore* Store = &Project.GetCidStore();
+ if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
+ {
+ UniqueStores.insert_or_assign(Store, Project.Identifier);
+ }
+ else
+ {
+ UniqueStores.insert_or_assign(Store, std::string{});
+ }
+ });
+
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+
+ CidStoreStats ChunkStoreStats = ChunkStore->Stats();
+ CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
+
+ TotalChunkHitCount += ChunkStoreStats.HitCount;
+ TotalChunkMissCount += ChunkStoreStats.MissCount;
+ TotalChunkWriteCount += ChunkStoreStats.WriteCount;
+
+ TotalCidSize.TinySize += ChunkStoreSize.TinySize;
+ TotalCidSize.SmallSize += ChunkStoreSize.SmallSize;
+ TotalCidSize.LargeSize += ChunkStoreSize.LargeSize;
+ TotalCidSize.TotalSize += ChunkStoreSize.TotalSize;
+ }
+ }
CbObjectWriter Cbo;
@@ -460,12 +495,66 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
Cbo.BeginObject("size");
{
- Cbo << "tiny" << CidSize.TinySize;
- Cbo << "small" << CidSize.SmallSize;
- Cbo << "large" << CidSize.LargeSize;
- Cbo << "total" << CidSize.TotalSize;
+ Cbo << "tiny" << TotalCidSize.TinySize;
+ Cbo << "small" << TotalCidSize.SmallSize;
+ Cbo << "large" << TotalCidSize.LargeSize;
+ Cbo << "total" << TotalCidSize.TotalSize;
}
Cbo.EndObject();
+
+ if (ShowCidStoreStats)
+ {
+ Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
+ const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
+ Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
+
+ Cbo.BeginObject("store");
+
+ auto OutputStats = [&](CidStore& ChunkStore) {
+ CidStoreStats StoreStats = ChunkStore.Stats();
+ Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
+ const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
+ Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
+ EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
+ EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
+ };
+
+ if (UniqueStores.size() > 1)
+ {
+ Cbo.BeginArray("projects");
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+ const std::string& ProjectId = It.second;
+ CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
+
+ Cbo.BeginObject();
+ {
+ Cbo << "project" << ProjectId;
+ Cbo.BeginObject("stats");
+ OutputStats(*ChunkStore);
+ Cbo.EndObject();
+
+ Cbo.BeginObject("size");
+ {
+ Cbo << "tiny" << ChunkStoreSize.TinySize;
+ Cbo << "small" << ChunkStoreSize.SmallSize;
+ Cbo << "large" << ChunkStoreSize.LargeSize;
+ Cbo << "total" << ChunkStoreSize.TotalSize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndArray(); // projects
+ }
+ else if (UniqueStores.size() != 0)
+ {
+ CidStore& ChunkStore = *UniqueStores.begin()->first;
+ OutputStats(ChunkStore);
+ }
+ Cbo.EndObject();
+ }
}
Cbo.EndObject();
@@ -1125,6 +1214,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
ProjectStore::Oplog& Oplog = *FoundLog;
IoBuffer Payload = HttpReq.ReadPayload();
@@ -1137,7 +1228,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
std::vector<IoHash> MissingChunks;
CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
- if (m_CidStore.ContainsChunk(Hash))
+ if (ChunkStore.ContainsChunk(Hash))
{
// Return null attachment as we already have it, no point in reading it and storing it again
return {};
@@ -1393,6 +1484,8 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
ProjectStore::Oplog& Oplog = *FoundLog;
if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
@@ -1407,7 +1500,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
Op.IterateAttachments([&](CbFieldView FieldView) {
const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(AttachmentHash);
if (Payload)
{
switch (Payload.GetContentType())
@@ -2036,11 +2129,14 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+ CidStore& ChunkStore = Project.GetCidStore();
+
Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
+ const Oid& Key,
+ CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
});
@@ -2054,8 +2150,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
m_ProjectStore->DiscoverProjects();
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- std::vector<std::string> OpLogs = Project.ScanForOplogs();
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ CidStore& ChunkStore = Project.GetCidStore();
+ std::vector<std::string> OpLogs = Project.ScanForOplogs();
+ CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
});
}
Cbo.EndArray();
@@ -2084,7 +2181,8 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
{
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
if (CSV)
{
@@ -2092,10 +2190,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
+ const Oid& Key,
+ CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2105,7 +2204,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
Cbo.BeginArray("projects");
{
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2141,16 +2240,17 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
+ ProjectStore::Oplog& Oplog = *FoundLog;
if (CSV)
{
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ [this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2159,7 +2259,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("oplogs");
{
- CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2204,9 +2304,10 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId));
}
- const Oid ObjId = Oid::FromHexString(OpId);
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ const Oid ObjId = Oid::FromHexString(OpId);
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
+ ProjectStore::Oplog& Oplog = *FoundLog;
std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId);
if (!Op.has_value())
@@ -2224,7 +2325,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
else
@@ -2232,7 +2333,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("ops");
{
- CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
+ CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 295defa5c..5782188e6 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -35,11 +35,7 @@ class ProjectStore;
class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(CidStore& Store,
- ProjectStore* InProjectStore,
- HttpStatusService& StatusService,
- HttpStatsService& StatsService,
- AuthMgr& AuthMgr);
+ HttpProjectService(ProjectStore* InProjectStore, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr);
~HttpProjectService();
virtual const char* BaseUri() const override;
@@ -92,7 +88,6 @@ private:
inline LoggerRef Log() { return m_Log; }
LoggerRef m_Log;
- CidStore& m_CidStore;
HttpRequestRouter m_Router;
Ref<ProjectStore> m_ProjectStore;
HttpStatusService& m_StatusService;
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 61c279afa..d6dd6ef9b 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1176,7 +1176,7 @@ ProjectStore::Oplog::Flush()
}
void
-ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
+ProjectStore::Oplog::Scrub(ScrubContext& Ctx)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
@@ -3575,7 +3575,7 @@ ProjectStore::Project::Flush()
}
void
-ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
+ProjectStore::Project::Scrub(ScrubContext& Ctx)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
// Scrubbing needs to check all existing oplogs
@@ -3587,7 +3587,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) {
if (!IsExpired(GcClock::TimePoint::min(), Ops))
{
- Ops.ScrubStorage(Ctx);
+ Ops.Scrub(Ctx);
}
});
}
@@ -3832,7 +3832,7 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const
//////////////////////////////////////////////////////////////////////////
-ProjectStore::ProjectStore(CidStore& Store,
+ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -3840,7 +3840,7 @@ ProjectStore::ProjectStore(CidStore& Store,
const Configuration& Config)
: m_Log(logging::Get("project"))
, m_Gc(Gc)
-, m_CidStore(Store)
+, m_GetCidStore(std::move(GetCidStore))
, m_JobQueue(JobQueue)
, m_OpenProcessCache(InOpenProcessCache)
, m_ProjectBasePath(BasePath)
@@ -3973,7 +3973,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx)
}
for (const Ref<Project>& Project : Projects)
{
- Project->ScrubStorage(Ctx);
+ Project->Scrub(Ctx);
}
}
@@ -4025,6 +4025,8 @@ ProjectStore::OpenProject(std::string_view ProjectId)
}
}
+ CidStore& ChunkStore = m_GetCidStore(ProjectId);
+
RwLock::ExclusiveLockScope _(m_ProjectsLock);
if (auto ProjIt = m_Projects.find(std::string{ProjectId}); ProjIt != m_Projects.end())
{
@@ -4041,7 +4043,7 @@ ProjectStore::OpenProject(std::string_view ProjectId)
Ref<Project>& Prj =
m_Projects
- .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+ .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->Read();
@@ -4068,12 +4070,14 @@ ProjectStore::NewProject(const std::filesystem::path& BasePath,
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::NewProject");
+ CidStore& ChunkStore = m_GetCidStore(ProjectId);
+
RwLock::ExclusiveLockScope _(m_ProjectsLock);
ZEN_INFO("project '{}': creating project at '{}'", ProjectId, BasePath);
Ref<Project>& Prj =
- m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+ m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->RootDir = RootDir;
@@ -4802,7 +4806,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
}
const IoHash Hash = IoHash::FromHexString(Cid);
- OutChunk = m_CidStore.FindChunkByCid(Hash);
+ OutChunk = Project->GetCidStore().FindChunkByCid(Hash);
if (!OutChunk)
{
@@ -4865,7 +4869,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
}
FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash});
- CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash);
+ CidStore::InsertResult Result = Project->GetCidStore().AddChunk(Chunk, Hash);
return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}};
}
@@ -4894,18 +4898,19 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
if (RequestObject["chunks"sv].IsArray())
{
// Legacy full chunks only by rawhash
- CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
-
+ CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
CbObjectWriter ResponseWriter;
ResponseWriter.BeginArray("chunks"sv);
for (CbFieldView FieldView : ChunksArray)
{
IoHash RawHash = FieldView.AsHash();
- IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash);
+ IoBuffer ChunkBuffer = ChunkStore.FindChunkByCid(RawHash);
if (ChunkBuffer)
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer));
@@ -5057,7 +5062,7 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
if (ChunkRequest.Input.Id.index() == 0)
{
const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id);
- IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(ChunkHash);
if (Payload)
{
ChunkRequest.Output.Exists = true;
@@ -5244,7 +5249,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
return {HttpResponseCode::BadRequest, "Invalid payload format"};
}
- CidStore& ChunkStore = m_CidStore;
+ CidStore& ChunkStore = Project->GetCidStore();
RwLock AttachmentsLock;
tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
@@ -5350,7 +5355,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId,
}
}
- CidStore& ChunkStore = m_CidStore;
+ CidStore& ChunkStore = Project->GetCidStore();
RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer(
ChunkStore,
@@ -5462,6 +5467,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
if (Method == "import"sv)
{
if (!AreDiskWritesAllowed())
@@ -5543,7 +5550,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Oplog->CaptureAddedAttachments(WriteRawHashes);
- m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
+ ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
}
HttpReq.WriteResponse(HttpResponseCode::OK);
return true;
@@ -5716,14 +5723,14 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
ResponseObj.EndArray();
}
- // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new
- // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have
+ // Ops that have moved chunks to a compressed buffer for storage in ChunkStore have been rewritten with references to the new
+ // chunk(s). Make sure we add the chunks to ChunkStore, and do it after we update the oplog so GC doesn't think we have
// unreferenced chunks.
for (auto It : AddedChunks)
{
const IoHash& RawHash = It.first;
AddedChunk& Chunk = It.second;
- CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash);
+ CidStore::InsertResult Result = ChunkStore.AddChunk(Chunk.Buffer, RawHash);
if (Result.New)
{
InlinedBytes += Chunk.RawSize;
@@ -5786,7 +5793,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
EmbedLooseFile,
Force,
IgnoreMissingAttachments](JobContext& Context) {
- RemoteProjectStore::Result Result = SaveOplog(m_CidStore,
+ RemoteProjectStore::Result Result = SaveOplog(Project->GetCidStore(),
*ActualRemoteStore,
*Project.Get(),
*OplogPtr,
@@ -5831,14 +5838,20 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
}
std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
+ CidStore& ChunkStore = Project.GetCidStore();
ZEN_INFO("Loading oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description);
JobId JobId = m_JobQueue.QueueJob(
fmt::format("Import oplog '{}/{}'", Project.Identifier, Oplog.OplogId()),
- [this, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, Force, IgnoreMissingAttachments, CleanOplog](
- JobContext& Context) {
+ [this,
+ ChunkStore = &ChunkStore,
+ ActualRemoteStore = std::move(RemoteStore),
+ OplogPtr = &Oplog,
+ Force,
+ IgnoreMissingAttachments,
+ CleanOplog](JobContext& Context) {
RemoteProjectStore::Result Result =
- LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
+ LoadOplog(*ChunkStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
auto Response = ConvertResult(Result);
ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
@@ -6877,6 +6890,11 @@ namespace testutils {
return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags);
}
+ ProjectStore::GetCidStoreFunc SingleChunkStore(CidStore& ChunkStore)
+ {
+ return [ChunkStore = &ChunkStore](std::string_view) -> CidStore& { return *ChunkStore; };
+ }
+
} // namespace testutils
TEST_CASE("project.opkeys")
@@ -6939,13 +6957,18 @@ TEST_CASE("project.store.create")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::string_view ProjectName("proj1"sv);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -6970,12 +6993,17 @@ TEST_CASE("project.store.lifetimes")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7033,12 +7061,17 @@ TEST_CASE_TEMPLATE("project.store.export",
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7072,7 +7105,7 @@ TEST_CASE_TEMPLATE("project.store.export",
std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
- RemoteProjectStore::Result ExportResult = SaveOplog(CidStore,
+ RemoteProjectStore::Result ExportResult = SaveOplog(ChunkStore,
*RemoteStore,
*Project.Get(),
*Oplog,
@@ -7089,7 +7122,7 @@ TEST_CASE_TEMPLATE("project.store.export",
ProjectStore::Oplog* OplogImport = Project->NewOplog("oplog2", {});
CHECK(OplogImport != nullptr);
- RemoteProjectStore::Result ImportResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7098,7 +7131,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportForceResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7107,7 +7140,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportForceResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportCleanResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7116,7 +7149,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportCleanResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7136,12 +7169,17 @@ TEST_CASE("project.store.gc")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7337,12 +7375,17 @@ TEST_CASE("project.store.gc.prep")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7385,7 +7428,7 @@ TEST_CASE("project.store.gc.prep")
// Equivalent of a `prep` existance check call
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7399,7 +7442,7 @@ TEST_CASE("project.store.gc.prep")
// If a gc comes in between our prep and op write the chunks will be removed
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7419,7 +7462,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7433,7 +7476,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7447,7 +7490,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained across multiple GCs if retain time is still valud
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7459,7 +7502,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
@@ -7476,7 +7519,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7509,7 +7552,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
Sleep(200);
@@ -7524,7 +7567,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
// This pass the retention time has expired and the last GC pass cleared the entries
@@ -7538,7 +7581,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
}
@@ -7552,12 +7595,17 @@ TEST_CASE("project.store.rpc.getchunks")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8474,12 +8522,17 @@ TEST_CASE("project.store.partial.read")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8652,12 +8705,17 @@ TEST_CASE("project.store.iterateoplog")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv";
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 368da5ea4..eb27665f9 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -68,7 +68,9 @@ public:
{
};
- ProjectStore(CidStore& Store,
+ typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
+
+ ProjectStore(GetCidStoreFunc&& GetCidStore,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -156,7 +158,7 @@ public:
LoggerRef Log() { return m_OuterProject->Log(); }
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
+ void Scrub(ScrubContext& Ctx);
static uint64_t TotalSize(const std::filesystem::path& BasePath);
uint64_t TotalSize() const;
@@ -326,11 +328,13 @@ public:
Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath);
virtual ~Project();
+ CidStore& GetCidStore() { return m_CidStore; };
+
void Read();
void Write();
[[nodiscard]] static bool Exists(const std::filesystem::path& BasePath);
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
+ void Scrub(ScrubContext& Ctx);
LoggerRef Log() const;
static uint64_t TotalSize(const std::filesystem::path& BasePath);
uint64_t TotalSize() const;
@@ -405,6 +409,7 @@ public:
LoggerRef Log() { return m_Log; }
const std::filesystem::path& BasePath() const { return m_ProjectBasePath; }
+ // GcStorage
virtual void ScrubStorage(ScrubContext& Ctx) override;
virtual GcStorageSize StorageSize() const override;
@@ -498,7 +503,7 @@ public:
private:
LoggerRef m_Log;
GcManager& m_Gc;
- CidStore& m_CidStore;
+ GetCidStoreFunc m_GetCidStore;
JobQueue& m_JobQueue;
OpenProcessCache& m_OpenProcessCache;
std::filesystem::path m_ProjectBasePath;