aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/projectstore/projectstore.cpp
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-05-17 17:00:35 +0200
committerGitHub <[email protected]>2023-05-17 17:00:35 +0200
commitd7ce2295395e23ca6fa822fa6171d16226889566 (patch)
tree43ac05cdcd9fddfb402969a7aff237f5e286ebfa /src/zenserver/projectstore/projectstore.cpp
parentSb/oplog export fixes (#315) (diff)
downloadzen-d7ce2295395e23ca6fa822fa6171d16226889566.tar.xz
zen-d7ce2295395e23ca6fa822fa6171d16226889566.zip
project store refactor (#316)
moved HttpProjectService into own file to improve maintainability
Diffstat (limited to 'src/zenserver/projectstore/projectstore.cpp')
-rw-r--r--src/zenserver/projectstore/projectstore.cpp1476
1 files changed, 0 insertions, 1476 deletions
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index dd4c7597c..08b4a6db5 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -201,228 +201,10 @@ namespace {
: fmt::format("{}. Reason: '{}'", Result.Text, Result.Reason)};
}
- void CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter)
- {
- if (AttachmentDetails)
- {
- CSVWriter << "Project, Oplog, LSN, Key, Cid, Size";
- }
- else if (Details)
- {
- CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize";
- }
- else
- {
- CSVWriter << "Project, Oplog, Key";
- }
- }
-
- void CSVWriteOp(CidStore& CidStore,
- std::string_view ProjectId,
- std::string_view OplogId,
- bool Details,
- bool AttachmentDetails,
- int LSN,
- const Oid& Key,
- CbObject Op,
- StringBuilderBase& CSVWriter)
- {
- StringBuilder<32> KeyStringBuilder;
- Key.ToString(KeyStringBuilder);
- const std::string_view KeyString = KeyStringBuilder.ToView();
-
- SharedBuffer Buffer = Op.GetBuffer();
- if (AttachmentDetails)
- {
- Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- CSVWriter << "\r\n"
- << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << AttachmentHash.ToHexString()
- << ", " << gsl::narrow<uint64_t>(Attachment.GetSize());
- });
- }
- else if (Details)
- {
- uint64_t AttachmentCount = 0;
- size_t AttachmentsSize = 0;
- Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- AttachmentCount++;
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- AttachmentsSize += Attachment.GetSize();
- });
- CSVWriter << "\r\n"
- << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Buffer.GetSize())
- << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
- }
- else
- {
- CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString;
- }
- };
-
- void CbWriteOp(CidStore& CidStore,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- int LSN,
- const Oid& Key,
- CbObject Op,
- CbObjectWriter& CbWriter)
- {
- CbWriter.BeginObject();
- {
- SharedBuffer Buffer = Op.GetBuffer();
- CbWriter.AddObjectId("key", Key);
- if (Details)
- {
- CbWriter.AddInteger("lsn", LSN);
- CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Buffer.GetSize()));
- }
- if (AttachmentDetails)
- {
- CbWriter.BeginArray("attachments");
- Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- CbWriter.BeginObject();
- {
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- CbWriter.AddString("cid", AttachmentHash.ToHexString());
- CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Attachment.GetSize()));
- }
- CbWriter.EndObject();
- });
- CbWriter.EndArray();
- }
- else if (Details)
- {
- uint64_t AttachmentCount = 0;
- size_t AttachmentsSize = 0;
- Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- AttachmentCount++;
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- AttachmentsSize += Attachment.GetSize();
- });
- if (AttachmentCount > 0)
- {
- CbWriter.AddInteger("attachments", AttachmentCount);
- CbWriter.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
- }
- }
- if (OpDetails)
- {
- CbWriter.BeginObject("op");
- for (const CbFieldView& Field : Op)
- {
- if (!Field.HasName())
- {
- CbWriter.AddField(Field);
- continue;
- }
- std::string_view FieldName = Field.GetName();
- CbWriter.AddField(FieldName, Field);
- }
- CbWriter.EndObject();
- }
- }
- CbWriter.EndObject();
- };
-
- void CbWriteOplogOps(CidStore& CidStore,
- ProjectStore::Oplog& Oplog,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginArray("ops");
- {
- Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
- CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
- });
- }
- Cbo.EndArray();
- }
-
- void CbWriteOplog(CidStore& CidStore,
- ProjectStore::Oplog& Oplog,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginObject();
- {
- Cbo.AddString("name", Oplog.OplogId());
- CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndObject();
- }
-
- void CbWriteOplogs(CidStore& CidStore,
- ProjectStore::Project& Project,
- std::vector<std::string> OpLogs,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginArray("oplogs");
- {
- for (const std::string& OpLogId : OpLogs)
- {
- ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId);
- if (Oplog != nullptr)
- {
- CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
- }
- }
- }
- Cbo.EndArray();
- }
-
- void CbWriteProject(CidStore& CidStore,
- ProjectStore::Project& Project,
- std::vector<std::string> OpLogs,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginObject();
- {
- Cbo.AddString("name", Project.Identifier);
- CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndObject();
- }
-
} // namespace
//////////////////////////////////////////////////////////////////////////
-Oid
-OpKeyStringAsOId(std::string_view OpKey)
-{
- using namespace std::literals;
-
- CbObjectWriter Writer;
- Writer << "key"sv << OpKey;
-
- XXH3_128Stream KeyHasher;
- Writer.Save()["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash = KeyHasher.GetHash();
-
- Oid OpId;
- memcpy(OpId.OidBits, &KeyHash, sizeof(OpId.OidBits));
-
- return OpId;
-}
-
-//////////////////////////////////////////////////////////////////////////
-
struct ProjectStore::OplogStorage : public RefCounted
{
OplogStorage(ProjectStore::Oplog* OwnerOplog, std::filesystem::path BasePath) : m_OwnerOplog(OwnerOplog), m_OplogStoragePath(BasePath)
@@ -2728,1264 +2510,6 @@ ProjectStore::AreDiskWritesAllowed() const
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
-: m_Log(logging::Get("project"))
-, m_CidStore(Store)
-, m_ProjectStore(Projects)
-, m_StatsService(StatsService)
-, m_AuthMgr(AuthMgr)
-{
- using namespace std::literals;
-
- m_StatsService.RegisterHandler("prj", *this);
-
- m_Router.AddPattern("project", "([[:alnum:]_.]+)");
- m_Router.AddPattern("log", "([[:alnum:]_.]+)");
- m_Router.AddPattern("op", "([[:digit:]]+?)");
- m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
- m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
-
- m_Router.RegisterRoute(
- "",
- [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_ProjectStore->GetProjectsList()); },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "list",
- [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_ProjectStore->GetProjectsList()); },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/batch",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- // Parse Request
-
- IoBuffer Payload = HttpReq.ReadPayload();
- BinaryReader Reader(Payload);
-
- struct RequestHeader
- {
- enum
- {
- kMagic = 0xAAAA'77AC
- };
- uint32_t Magic;
- uint32_t ChunkCount;
- uint32_t Reserved1;
- uint32_t Reserved2;
- };
-
- struct RequestChunkEntry
- {
- Oid ChunkId;
- uint32_t CorrelationId;
- uint64_t Offset;
- uint64_t RequestBytes;
- };
-
- if (Payload.Size() <= sizeof(RequestHeader))
- {
- HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
-
- RequestHeader RequestHdr;
- Reader.Read(&RequestHdr, sizeof RequestHdr);
-
- if (RequestHdr.Magic != RequestHeader::kMagic)
- {
- HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
-
- std::vector<RequestChunkEntry> RequestedChunks;
- RequestedChunks.resize(RequestHdr.ChunkCount);
- Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount);
-
- // Make Response
-
- struct ResponseHeader
- {
- uint32_t Magic = 0xbada'b00f;
- uint32_t ChunkCount;
- uint32_t Reserved1 = 0;
- uint32_t Reserved2 = 0;
- };
-
- struct ResponseChunkEntry
- {
- uint32_t CorrelationId;
- uint32_t Flags = 0;
- uint64_t ChunkSize;
- };
-
- std::vector<IoBuffer> OutBlobs;
- OutBlobs.emplace_back(sizeof(ResponseHeader) + RequestHdr.ChunkCount * sizeof(ResponseChunkEntry));
- for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
- {
- const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
- IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId);
- if (FoundChunk)
- {
- if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1))
- {
- uint64_t Offset = RequestedChunk.Offset;
- if (Offset > FoundChunk.Size())
- {
- Offset = FoundChunk.Size();
- }
- uint64_t Size = RequestedChunk.RequestBytes;
- if ((Offset + Size) > FoundChunk.Size())
- {
- Size = FoundChunk.Size() - Offset;
- }
- FoundChunk = IoBuffer(FoundChunk, Offset, Size);
- }
- }
- OutBlobs.emplace_back(std::move(FoundChunk));
- }
- uint8_t* ResponsePtr = reinterpret_cast<uint8_t*>(OutBlobs[0].MutableData());
- ResponseHeader ResponseHdr;
- ResponseHdr.ChunkCount = RequestHdr.ChunkCount;
- memcpy(ResponsePtr, &ResponseHdr, sizeof(ResponseHdr));
- ResponsePtr += sizeof(ResponseHdr);
- for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
- {
- // const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
- const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]);
- ResponseChunkEntry ResponseChunk;
- ResponseChunk.CorrelationId = ChunkIndex;
- if (FoundChunk)
- {
- ResponseChunk.ChunkSize = FoundChunk.Size();
- }
- else
- {
- ResponseChunk.ChunkSize = uint64_t(-1);
- }
- memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk));
- ResponsePtr += sizeof(ResponseChunk);
- }
- return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/files",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- // File manifest fetch, returns the client file list
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
-
- const bool FilterClient = Params.GetValue("filter"sv) == "client"sv;
-
- CbObject ResponsePayload;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetProjectFiles(ProjectId, OplogId, FilterClient, ResponsePayload);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{chunk}/info",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& ChunkId = Req.GetCapture(3);
-
- CbObject ResponsePayload;
- std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunkInfo(ProjectId, OplogId, ChunkId, ResponsePayload);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
- }
- else if (Result.first == HttpResponseCode::NotFound)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{chunk}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& ChunkId = Req.GetCapture(3);
-
- uint64_t Offset = 0;
- uint64_t Size = ~(0ull);
-
- auto QueryParms = HttpReq.GetQueryParams();
-
- if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false)
- {
- if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm))
- {
- Offset = OffsetVal.value();
- }
- else
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
- }
-
- if (auto SizeParm = QueryParms.GetValue("size"); SizeParm.empty() == false)
- {
- if (auto SizeVal = ParseInt<uint64_t>(SizeParm))
- {
- Size = SizeVal.value();
- }
- else
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
- }
-
- HttpContentType AcceptType = HttpReq.AcceptContentType();
-
- IoBuffer Chunk;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk);
- if (Result.first == HttpResponseCode::OK)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Chunk.GetContentType()));
- return HttpReq.WriteResponse(HttpResponseCode::OK, Chunk.GetContentType(), Chunk);
- }
- else if (Result.first == HttpResponseCode::NotFound)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet | HttpVerb::kHead);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{hash}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& Cid = Req.GetCapture(3);
- HttpContentType AcceptType = HttpReq.AcceptContentType();
- HttpContentType RequestType = HttpReq.RequestContentType();
-
- switch (HttpReq.RequestVerb())
- {
- case HttpVerb::kGet:
- {
- IoBuffer Value;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value);
-
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
- }
- else if (Result.first == HttpResponseCode::NotFound)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- }
- case HttpVerb::kPost:
- {
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->PutChunk(ProjectId, OplogId, Cid, RequestType, HttpReq.ReadPayload());
- if (Result.first == HttpResponseCode::OK || Result.first == HttpResponseCode::Created)
- {
- return HttpReq.WriteResponse(Result.first);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- }
- break;
- }
- },
- HttpVerb::kGet | HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/prep",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogPrep");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- // This operation takes a list of referenced hashes and decides which
- // chunks are not present on this server. This list is then returned in
- // the "need" list in the response
-
- IoBuffer Payload = HttpReq.ReadPayload();
- CbObject RequestObject = LoadCompactBinaryObject(Payload);
-
- std::vector<IoHash> NeedList;
-
- for (auto Entry : RequestObject["have"sv])
- {
- const IoHash FileHash = Entry.AsHash();
-
- if (!m_CidStore.ContainsChunk(FileHash))
- {
- ZEN_DEBUG("prep - NEED: {}", FileHash);
-
- NeedList.push_back(FileHash);
- }
- }
-
- CbObjectWriter Cbo;
- Cbo.BeginArray("need");
-
- for (const IoHash& Hash : NeedList)
- {
- Cbo << Hash;
- }
-
- Cbo.EndArray();
- CbObject Response = Cbo.Save();
-
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/new",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogNew");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
-
- bool IsUsingSalt = false;
- IoHash SaltHash = IoHash::Zero;
-
- if (std::string_view SaltParam = Params.GetValue("salt"); SaltParam.empty() == false)
- {
- const uint32_t Salt = std::stoi(std::string(SaltParam));
- SaltHash = IoHash::HashBuffer(&Salt, sizeof Salt);
- IsUsingSalt = true;
- }
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- ProjectStore::Oplog& Oplog = *FoundLog;
-
- IoBuffer Payload = HttpReq.ReadPayload();
-
- // This will attempt to open files which may not exist for the case where
- // the prep step rejected the chunk. This should be fixed since there's
- // a performance cost associated with any file system activity
-
- bool IsValid = true;
- std::vector<IoHash> MissingChunks;
-
- CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
- if (m_CidStore.ContainsChunk(Hash))
- {
- // Return null attachment as we already have it, no point in reading it and storing it again
- return {};
- }
-
- IoHash AttachmentId;
- if (IsUsingSalt)
- {
- IoHash AttachmentSpec[]{SaltHash, Hash};
- AttachmentId = IoHash::HashBuffer(MakeMemoryView(AttachmentSpec));
- }
- else
- {
- AttachmentId = Hash;
- }
-
- std::filesystem::path AttachmentPath = Oplog.TempPath() / AttachmentId.ToHexString();
- if (IoBuffer Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath))
- {
- return SharedBuffer(std::move(Data));
- }
- else
- {
- IsValid = false;
- MissingChunks.push_back(Hash);
-
- return {};
- }
- };
-
- CbPackage Package;
-
- if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver))
- {
- std::filesystem::path BadPackagePath =
- Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId());
-
- ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath);
-
- WriteFile(BadPackagePath, Payload);
-
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package");
- }
-
- if (!IsValid)
- {
- // TODO: emit diagnostics identifying missing chunks
-
- return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing chunk reference");
- }
-
- CbObject Core = Package.GetObject();
-
- if (!Core["key"sv])
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
- }
-
- // Write core to oplog
-
- const uint32_t OpLsn = Oplog.AppendNewOplogEntry(Package);
-
- if (OpLsn == ProjectStore::Oplog::kInvalidOp)
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
-
- ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString());
-
- HttpReq.WriteResponse(HttpResponseCode::Created);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{op}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const std::string& ProjectId = Req.GetCapture(1);
- const std::string& OplogId = Req.GetCapture(2);
- const std::string& OpIdString = Req.GetCapture(3);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- ProjectStore::Oplog& Oplog = *FoundLog;
-
- if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
- {
- if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value()))
- {
- CbObject& Op = MaybeOp.value();
- if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage)
- {
- CbPackage Package;
- Package.SetObject(Op);
-
- Op.IterateAttachments([&](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash);
-
- // We force this for now as content type is not consistently tracked (will
- // be fixed in CidStore refactor)
- Payload.SetContentType(ZenContentType::kCompressedBinary);
-
- if (Payload)
- {
- switch (Payload.GetContentType())
- {
- case ZenContentType::kCbObject:
- if (CbObject Object = LoadCompactBinaryObject(Payload))
- {
- Package.AddAttachment(CbAttachment(Object));
- }
- else
- {
- // Error - malformed object
-
- ZEN_WARN("malformed object returned for {}", AttachmentHash);
- }
- break;
-
- case ZenContentType::kCompressedBinary:
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload)))
- {
- Package.AddAttachment(CbAttachment(Compressed, AttachmentHash));
- }
- else
- {
- // Error - not compressed!
-
- ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash);
- }
- break;
-
- default:
- Package.AddAttachment(CbAttachment(SharedBuffer(Payload)));
- break;
- }
- }
- });
-
- return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package);
- }
- else
- {
- // Client cannot accept a package, so we only send the core object
- return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op);
- }
- }
- }
-
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
-
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("project {} not found", ProjectId));
- }
- Project->TouchProject();
-
- switch (HttpReq.RequestVerb())
- {
- case HttpVerb::kGet:
- {
- ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
- if (!OplogIt)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("oplog {} not found in project {}", OplogId, ProjectId));
- }
- Project->TouchOplog(OplogId);
-
- ProjectStore::Oplog& Log = *OplogIt;
-
- CbObjectWriter Cb;
- Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str()
- << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount"
- << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::max(), Log);
-
- HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save());
- }
- break;
-
- case HttpVerb::kPost:
- {
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
- std::filesystem::path OplogMarkerPath;
- if (CbObject Params = HttpReq.ReadPayloadObject())
- {
- OplogMarkerPath = Params["gcpath"sv].AsString();
- }
-
- ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
- if (!OplogIt)
- {
- if (!Project->NewOplog(OplogId, OplogMarkerPath))
- {
- // TODO: indicate why the operation failed!
- return HttpReq.WriteResponse(HttpResponseCode::InternalServerError);
- }
- Project->TouchOplog(OplogId);
-
- ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath);
-
- return HttpReq.WriteResponse(HttpResponseCode::Created);
- }
-
- // I guess this should ultimately be used to execute RPCs but for now, it
- // does absolutely nothing
-
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
- break;
-
- case HttpVerb::kDelete:
- {
- ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId);
-
- Project->DeleteOplog(OplogId);
-
- return HttpReq.WriteResponse(HttpResponseCode::OK);
- }
- break;
-
- default:
- break;
- }
- },
- HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/entries",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogEntries");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- CbObjectWriter Response;
-
- if (FoundLog->OplogCount() > 0)
- {
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
-
- if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty())
- {
- Oid OpKeyId = OpKeyStringAsOId(OpKey);
- std::optional<CbObject> Op = FoundLog->GetOpByKey(OpKeyId);
-
- if (Op.has_value())
- {
- Response << "entry"sv << Op.value();
- }
- else
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- }
- else
- {
- Response.BeginArray("entries"sv);
-
- FoundLog->IterateOplog([&Response](CbObject Op) { Response << Op; });
-
- Response.EndArray();
- }
- }
-
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const std::string ProjectId = Req.GetCapture(1);
-
- switch (HttpReq.RequestVerb())
- {
- case HttpVerb::kPost:
- {
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
-
- IoBuffer Payload = HttpReq.ReadPayload();
- CbObject Params = LoadCompactBinaryObject(Payload);
- std::string_view Id = Params["id"sv].AsString();
- std::string_view Root = Params["root"sv].AsString();
- std::string_view EngineRoot = Params["engine"sv].AsString();
- std::string_view ProjectRoot = Params["project"sv].AsString();
- std::string_view ProjectFilePath = Params["projectfile"sv].AsString();
-
- const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId;
- m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath);
-
- ZEN_INFO("established project - {} (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
- ProjectId,
- Id,
- Root,
- EngineRoot,
- ProjectRoot,
- ProjectFilePath,
- ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
-
- HttpReq.WriteResponse(HttpResponseCode::Created);
- }
- break;
-
- case HttpVerb::kGet:
- {
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("project {} not found", ProjectId));
- }
- Project->TouchProject();
-
- std::vector<std::string> OpLogs = Project->ScanForOplogs();
-
- CbObjectWriter Response;
- Response << "id"sv << Project->Identifier;
- Response << "root"sv << PathToUtf8(Project->RootDir);
- Response << "engine"sv << PathToUtf8(Project->EngineRootDir);
- Response << "project"sv << PathToUtf8(Project->ProjectRootDir);
- Response << "projectfile"sv << PathToUtf8(Project->ProjectFilePath);
-
- Response.BeginArray("oplogs"sv);
- for (const std::string& OplogId : OpLogs)
- {
- Response.BeginObject();
- Response << "id"sv << OplogId;
- Response.EndObject();
- }
- Response.EndArray(); // oplogs
-
- HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
- }
- break;
-
- case HttpVerb::kDelete:
- {
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("project {} not found", ProjectId));
- }
-
- ZEN_INFO("deleting project '{}'", ProjectId);
- if (!m_ProjectStore->DeleteProject(ProjectId))
- {
- return HttpReq.WriteResponse(HttpResponseCode::Locked,
- HttpContentType::kText,
- fmt::format("project {} is in use", ProjectId));
- }
-
- return HttpReq.WriteResponse(HttpResponseCode::NoContent);
- }
- break;
-
- default:
- break;
- }
- },
- HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete);
-
- // Push a oplog container
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/save",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogSave");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- if (HttpReq.RequestContentType() != HttpContentType::kCbObject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type");
- }
- IoBuffer Payload = HttpReq.ReadPayload();
-
- CbObject Response;
- std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->WriteOplog(ProjectId, OplogId, std::move(Payload), Response);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kPost);
-
- // Pull a oplog container
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/load",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogLoad");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- if (HttpReq.AcceptContentType() != HttpContentType::kCbObject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type");
- }
- IoBuffer Payload = HttpReq.ReadPayload();
-
- CbObject Response;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->ReadOplog(ProjectId, OplogId, HttpReq.GetQueryParams(), Response);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet);
-
- // Do an rpc style operation on project/oplog
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/rpc",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- IoBuffer Payload = HttpReq.ReadPayload();
-
- m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "details\\$",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- Project.IterateOplogs([&](ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
- CSVWriteOp(m_CidStore,
- Project.Identifier,
- Oplog.OplogId(),
- Details,
- AttachmentDetails,
- LSN,
- Key,
- Op,
- CSVWriter);
- });
- });
- });
-
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- Cbo.BeginArray("projects");
- {
- m_ProjectStore->DiscoverProjects();
-
- m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- std::vector<std::string> OpLogs = Project.ScanForOplogs();
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
- });
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "details\\$/{project}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
- if (!FoundProject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- ProjectStore::Project& Project = *FoundProject.Get();
-
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- FoundProject->IterateOplogs([&](ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN,
- const Oid& Key,
- CbObject Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
- });
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
- Cbo.BeginArray("projects");
- {
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "details\\$/{project}/{log}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
- if (!FoundProject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- Cbo.BeginArray("oplogs");
- {
- CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "details\\$/{project}/{log}/{chunk}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& ChunkId = Req.GetCapture(3);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
- if (!FoundProject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
- {
- return HttpReq.WriteResponse(
- HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId));
- }
-
- const Oid ObjId = Oid::FromHexString(ChunkId);
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
-
- int LSN = Oplog.GetOpIndexByKey(ObjId);
- if (LSN == -1)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- std::optional<CbObject> Op = Oplog.GetOpByIndex(LSN);
- if (!Op.has_value())
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter);
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- Cbo.BeginArray("ops");
- {
- CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo);
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-}
-
-HttpProjectService::~HttpProjectService()
-{
- m_StatsService.UnregisterHandler("prj", *this);
-}
-
-const char*
-HttpProjectService::BaseUri() const
-{
- return "/prj/";
-}
-
-void
-HttpProjectService::HandleRequest(HttpServerRequest& Request)
-{
- if (m_Router.HandleRequest(Request) == false)
- {
- ZEN_WARN("No route found for {0}", Request.RelativeUri());
- }
-}
-
-void
-HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
-{
- const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
- const CidStoreSize CidSize = m_CidStore.TotalSize();
-
- CbObjectWriter Cbo;
- Cbo.BeginObject("store");
- {
- Cbo.BeginObject("size");
- {
- Cbo << "disk" << StoreSize.DiskSize;
- Cbo << "memory" << StoreSize.MemorySize;
- }
- Cbo.EndObject();
- }
- Cbo.EndObject();
-
- Cbo.BeginObject("cid");
- {
- Cbo.BeginObject("size");
- {
- Cbo << "tiny" << CidSize.TinySize;
- Cbo << "small" << CidSize.SmallSize;
- Cbo << "large" << CidSize.LargeSize;
- Cbo << "total" << CidSize.TotalSize;
- }
- Cbo.EndObject();
- }
- Cbo.EndObject();
-
- return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
-}
-
-//////////////////////////////////////////////////////////////////////////
-
#if ZEN_WITH_TESTS
namespace testutils {