aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-05-17 17:00:35 +0200
committerGitHub <[email protected]>2023-05-17 17:00:35 +0200
commitd7ce2295395e23ca6fa822fa6171d16226889566 (patch)
tree43ac05cdcd9fddfb402969a7aff237f5e286ebfa /src
parentSb/oplog export fixes (#315) (diff)
downloadzen-d7ce2295395e23ca6fa822fa6171d16226889566.tar.xz
zen-d7ce2295395e23ca6fa822fa6171d16226889566.zip
project store refactor (#316)
moved HttpProjectService into own file to improve maintainability
Diffstat (limited to 'src')
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp1499
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h55
-rw-r--r--src/zenserver/projectstore/projectstore.cpp1476
-rw-r--r--src/zenserver/projectstore/projectstore.h47
-rw-r--r--src/zenserver/zenserver.cpp1
5 files changed, 1560 insertions, 1518 deletions
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
new file mode 100644
index 000000000..b0efb6bec
--- /dev/null
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -0,0 +1,1499 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httpprojectstore.h"
+
+#include "projectstore.h"
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/stream.h>
+#include <zencore/trace.h>
+
+namespace zen {
+
+Oid
+OpKeyStringAsOId(std::string_view OpKey)
+{
+ using namespace std::literals;
+
+ CbObjectWriter Writer;
+ Writer << "key"sv << OpKey;
+
+ XXH3_128Stream KeyHasher;
+ Writer.Save()["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
+ XXH3_128 KeyHash = KeyHasher.GetHash();
+
+ Oid OpId;
+ memcpy(OpId.OidBits, &KeyHash, sizeof(OpId.OidBits));
+
+ return OpId;
+}
+
+void
+CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter)
+{
+ if (AttachmentDetails)
+ {
+ CSVWriter << "Project, Oplog, LSN, Key, Cid, Size";
+ }
+ else if (Details)
+ {
+ CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize";
+ }
+ else
+ {
+ CSVWriter << "Project, Oplog, Key";
+ }
+}
+
+void
+CSVWriteOp(CidStore& CidStore,
+ std::string_view ProjectId,
+ std::string_view OplogId,
+ bool Details,
+ bool AttachmentDetails,
+ int LSN,
+ const Oid& Key,
+ CbObject Op,
+ StringBuilderBase& CSVWriter)
+{
+ StringBuilder<32> KeyStringBuilder;
+ Key.ToString(KeyStringBuilder);
+ const std::string_view KeyString = KeyStringBuilder.ToView();
+
+ SharedBuffer Buffer = Op.GetBuffer();
+ if (AttachmentDetails)
+ {
+ Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) {
+ const IoHash AttachmentHash = FieldView.AsAttachment();
+ IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+ CSVWriter << "\r\n"
+ << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << AttachmentHash.ToHexString() << ", "
+ << gsl::narrow<uint64_t>(Attachment.GetSize());
+ });
+ }
+ else if (Details)
+ {
+ uint64_t AttachmentCount = 0;
+ size_t AttachmentsSize = 0;
+ Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
+ const IoHash AttachmentHash = FieldView.AsAttachment();
+ AttachmentCount++;
+ IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+ AttachmentsSize += Attachment.GetSize();
+ });
+ CSVWriter << "\r\n"
+ << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Buffer.GetSize())
+ << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
+ }
+ else
+ {
+ CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString;
+ }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace {
+
+ void CbWriteOp(CidStore& CidStore,
+ bool Details,
+ bool OpDetails,
+ bool AttachmentDetails,
+ int LSN,
+ const Oid& Key,
+ CbObject Op,
+ CbObjectWriter& CbWriter)
+ {
+ CbWriter.BeginObject();
+ {
+ SharedBuffer Buffer = Op.GetBuffer();
+ CbWriter.AddObjectId("key", Key);
+ if (Details)
+ {
+ CbWriter.AddInteger("lsn", LSN);
+ CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Buffer.GetSize()));
+ }
+ if (AttachmentDetails)
+ {
+ CbWriter.BeginArray("attachments");
+ Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) {
+ const IoHash AttachmentHash = FieldView.AsAttachment();
+ CbWriter.BeginObject();
+ {
+ IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+ CbWriter.AddString("cid", AttachmentHash.ToHexString());
+ CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Attachment.GetSize()));
+ }
+ CbWriter.EndObject();
+ });
+ CbWriter.EndArray();
+ }
+ else if (Details)
+ {
+ uint64_t AttachmentCount = 0;
+ size_t AttachmentsSize = 0;
+ Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
+ const IoHash AttachmentHash = FieldView.AsAttachment();
+ AttachmentCount++;
+ IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+ AttachmentsSize += Attachment.GetSize();
+ });
+ if (AttachmentCount > 0)
+ {
+ CbWriter.AddInteger("attachments", AttachmentCount);
+ CbWriter.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
+ }
+ }
+ if (OpDetails)
+ {
+ CbWriter.BeginObject("op");
+ for (const CbFieldView& Field : Op)
+ {
+ if (!Field.HasName())
+ {
+ CbWriter.AddField(Field);
+ continue;
+ }
+ std::string_view FieldName = Field.GetName();
+ CbWriter.AddField(FieldName, Field);
+ }
+ CbWriter.EndObject();
+ }
+ }
+ CbWriter.EndObject();
+ };
+
+ void CbWriteOplogOps(CidStore& CidStore,
+ ProjectStore::Oplog& Oplog,
+ bool Details,
+ bool OpDetails,
+ bool AttachmentDetails,
+ CbObjectWriter& Cbo)
+ {
+ Cbo.BeginArray("ops");
+ {
+ Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+ CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
+ });
+ }
+ Cbo.EndArray();
+ }
+
+ void CbWriteOplog(CidStore& CidStore,
+ ProjectStore::Oplog& Oplog,
+ bool Details,
+ bool OpDetails,
+ bool AttachmentDetails,
+ CbObjectWriter& Cbo)
+ {
+ Cbo.BeginObject();
+ {
+ Cbo.AddString("name", Oplog.OplogId());
+ CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+ }
+ Cbo.EndObject();
+ }
+
+ void CbWriteOplogs(CidStore& CidStore,
+ ProjectStore::Project& Project,
+ std::vector<std::string> OpLogs,
+ bool Details,
+ bool OpDetails,
+ bool AttachmentDetails,
+ CbObjectWriter& Cbo)
+ {
+ Cbo.BeginArray("oplogs");
+ {
+ for (const std::string& OpLogId : OpLogs)
+ {
+ ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId);
+ if (Oplog != nullptr)
+ {
+ CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+ }
+ }
+ }
+ Cbo.EndArray();
+ }
+
+ void CbWriteProject(CidStore& CidStore,
+ ProjectStore::Project& Project,
+ std::vector<std::string> OpLogs,
+ bool Details,
+ bool OpDetails,
+ bool AttachmentDetails,
+ CbObjectWriter& Cbo)
+ {
+ Cbo.BeginObject();
+ {
+ Cbo.AddString("name", Project.Identifier);
+ CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ }
+ Cbo.EndObject();
+ }
+
+} // namespace
+
+//////////////////////////////////////////////////////////////////////////
+
+HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
+: m_Log(logging::Get("project"))
+, m_CidStore(Store)
+, m_ProjectStore(Projects)
+, m_StatsService(StatsService)
+, m_AuthMgr(AuthMgr)
+{
+ using namespace std::literals;
+
+ m_StatsService.RegisterHandler("prj", *this);
+
+ m_Router.AddPattern("project", "([[:alnum:]_.]+)");
+ m_Router.AddPattern("log", "([[:alnum:]_.]+)");
+ m_Router.AddPattern("op", "([[:digit:]]+?)");
+ m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
+ m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
+
+ m_Router.RegisterRoute(
+ "",
+ [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_ProjectStore->GetProjectsList()); },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "list",
+ [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_ProjectStore->GetProjectsList()); },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/batch",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchOplog(OplogId);
+
+ // Parse Request
+
+ IoBuffer Payload = HttpReq.ReadPayload();
+ BinaryReader Reader(Payload);
+
+ struct RequestHeader
+ {
+ enum
+ {
+ kMagic = 0xAAAA'77AC
+ };
+ uint32_t Magic;
+ uint32_t ChunkCount;
+ uint32_t Reserved1;
+ uint32_t Reserved2;
+ };
+
+ struct RequestChunkEntry
+ {
+ Oid ChunkId;
+ uint32_t CorrelationId;
+ uint64_t Offset;
+ uint64_t RequestBytes;
+ };
+
+ if (Payload.Size() <= sizeof(RequestHeader))
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest);
+ }
+
+ RequestHeader RequestHdr;
+ Reader.Read(&RequestHdr, sizeof RequestHdr);
+
+ if (RequestHdr.Magic != RequestHeader::kMagic)
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest);
+ }
+
+ std::vector<RequestChunkEntry> RequestedChunks;
+ RequestedChunks.resize(RequestHdr.ChunkCount);
+ Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount);
+
+ // Make Response
+
+ struct ResponseHeader
+ {
+ uint32_t Magic = 0xbada'b00f;
+ uint32_t ChunkCount;
+ uint32_t Reserved1 = 0;
+ uint32_t Reserved2 = 0;
+ };
+
+ struct ResponseChunkEntry
+ {
+ uint32_t CorrelationId;
+ uint32_t Flags = 0;
+ uint64_t ChunkSize;
+ };
+
+ std::vector<IoBuffer> OutBlobs;
+ OutBlobs.emplace_back(sizeof(ResponseHeader) + RequestHdr.ChunkCount * sizeof(ResponseChunkEntry));
+ for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
+ {
+ const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
+ IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId);
+ if (FoundChunk)
+ {
+ if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1))
+ {
+ uint64_t Offset = RequestedChunk.Offset;
+ if (Offset > FoundChunk.Size())
+ {
+ Offset = FoundChunk.Size();
+ }
+ uint64_t Size = RequestedChunk.RequestBytes;
+ if ((Offset + Size) > FoundChunk.Size())
+ {
+ Size = FoundChunk.Size() - Offset;
+ }
+ FoundChunk = IoBuffer(FoundChunk, Offset, Size);
+ }
+ }
+ OutBlobs.emplace_back(std::move(FoundChunk));
+ }
+ uint8_t* ResponsePtr = reinterpret_cast<uint8_t*>(OutBlobs[0].MutableData());
+ ResponseHeader ResponseHdr;
+ ResponseHdr.ChunkCount = RequestHdr.ChunkCount;
+ memcpy(ResponsePtr, &ResponseHdr, sizeof(ResponseHdr));
+ ResponsePtr += sizeof(ResponseHdr);
+ for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
+ {
+ // const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
+ const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]);
+ ResponseChunkEntry ResponseChunk;
+ ResponseChunk.CorrelationId = ChunkIndex;
+ if (FoundChunk)
+ {
+ ResponseChunk.ChunkSize = FoundChunk.Size();
+ }
+ else
+ {
+ ResponseChunk.ChunkSize = uint64_t(-1);
+ }
+ memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk));
+ ResponsePtr += sizeof(ResponseChunk);
+ }
+ return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs);
+ },
+ HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/files",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ // File manifest fetch, returns the client file list
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+
+ HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
+
+ const bool FilterClient = Params.GetValue("filter"sv) == "client"sv;
+
+ CbObject ResponsePayload;
+ std::pair<HttpResponseCode, std::string> Result =
+ m_ProjectStore->GetProjectFiles(ProjectId, OplogId, FilterClient, ResponsePayload);
+ if (Result.first == HttpResponseCode::OK)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
+ }
+ else
+ {
+ ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+ ToString(HttpReq.RequestVerb()),
+ HttpReq.QueryString(),
+ static_cast<int>(Result.first),
+ Result.second);
+ }
+ if (Result.second.empty())
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/{chunk}/info",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+ const auto& ChunkId = Req.GetCapture(3);
+
+ CbObject ResponsePayload;
+ std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunkInfo(ProjectId, OplogId, ChunkId, ResponsePayload);
+ if (Result.first == HttpResponseCode::OK)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
+ }
+ else if (Result.first == HttpResponseCode::NotFound)
+ {
+ ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
+ }
+ else
+ {
+ ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+ ToString(HttpReq.RequestVerb()),
+ HttpReq.QueryString(),
+ static_cast<int>(Result.first),
+ Result.second);
+ }
+ if (Result.second.empty())
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/{chunk}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+ const auto& ChunkId = Req.GetCapture(3);
+
+ uint64_t Offset = 0;
+ uint64_t Size = ~(0ull);
+
+ auto QueryParms = HttpReq.GetQueryParams();
+
+ if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false)
+ {
+ if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm))
+ {
+ Offset = OffsetVal.value();
+ }
+ else
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
+ }
+ }
+
+ if (auto SizeParm = QueryParms.GetValue("size"); SizeParm.empty() == false)
+ {
+ if (auto SizeVal = ParseInt<uint64_t>(SizeParm))
+ {
+ Size = SizeVal.value();
+ }
+ else
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
+ }
+ }
+
+ HttpContentType AcceptType = HttpReq.AcceptContentType();
+
+ IoBuffer Chunk;
+ std::pair<HttpResponseCode, std::string> Result =
+ m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk);
+ if (Result.first == HttpResponseCode::OK)
+ {
+ ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Chunk.GetContentType()));
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Chunk.GetContentType(), Chunk);
+ }
+ else if (Result.first == HttpResponseCode::NotFound)
+ {
+ ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
+ }
+ else
+ {
+ ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+ ToString(HttpReq.RequestVerb()),
+ HttpReq.QueryString(),
+ static_cast<int>(Result.first),
+ Result.second);
+ }
+ if (Result.second.empty())
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+ },
+ HttpVerb::kGet | HttpVerb::kHead);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/{hash}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+ const auto& Cid = Req.GetCapture(3);
+ HttpContentType AcceptType = HttpReq.AcceptContentType();
+ HttpContentType RequestType = HttpReq.RequestContentType();
+
+ switch (HttpReq.RequestVerb())
+ {
+ case HttpVerb::kGet:
+ {
+ IoBuffer Value;
+ std::pair<HttpResponseCode, std::string> Result =
+ m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value);
+
+ if (Result.first == HttpResponseCode::OK)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
+ }
+ else if (Result.first == HttpResponseCode::NotFound)
+ {
+ ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid);
+ }
+ else
+ {
+ ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+ ToString(HttpReq.RequestVerb()),
+ HttpReq.QueryString(),
+ static_cast<int>(Result.first),
+ Result.second);
+ }
+ if (Result.second.empty())
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+ }
+ case HttpVerb::kPost:
+ {
+ if (!m_ProjectStore->AreDiskWritesAllowed())
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+ }
+ std::pair<HttpResponseCode, std::string> Result =
+ m_ProjectStore->PutChunk(ProjectId, OplogId, Cid, RequestType, HttpReq.ReadPayload());
+ if (Result.first == HttpResponseCode::OK || Result.first == HttpResponseCode::Created)
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ else
+ {
+ ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+ ToString(HttpReq.RequestVerb()),
+ HttpReq.QueryString(),
+ static_cast<int>(Result.first),
+ Result.second);
+ }
+ if (Result.second.empty())
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+ }
+ break;
+ }
+ },
+ HttpVerb::kGet | HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/prep",
+ [this](HttpRouterRequest& Req) {
+ ZEN_TRACE_CPU("ProjectService::OplogPrep");
+
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchOplog(OplogId);
+
+ // This operation takes a list of referenced hashes and decides which
+ // chunks are not present on this server. This list is then returned in
+ // the "need" list in the response
+
+ IoBuffer Payload = HttpReq.ReadPayload();
+ CbObject RequestObject = LoadCompactBinaryObject(Payload);
+
+ std::vector<IoHash> NeedList;
+
+ for (auto Entry : RequestObject["have"sv])
+ {
+ const IoHash FileHash = Entry.AsHash();
+
+ if (!m_CidStore.ContainsChunk(FileHash))
+ {
+ ZEN_DEBUG("prep - NEED: {}", FileHash);
+
+ NeedList.push_back(FileHash);
+ }
+ }
+
+ CbObjectWriter Cbo;
+ Cbo.BeginArray("need");
+
+ for (const IoHash& Hash : NeedList)
+ {
+ Cbo << Hash;
+ }
+
+ Cbo.EndArray();
+ CbObject Response = Cbo.Save();
+
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
+ },
+ HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/new",
+ [this](HttpRouterRequest& Req) {
+ ZEN_TRACE_CPU("ProjectService::OplogNew");
+
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ if (!m_ProjectStore->AreDiskWritesAllowed())
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+ }
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+
+ HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
+
+ bool IsUsingSalt = false;
+ IoHash SaltHash = IoHash::Zero;
+
+ if (std::string_view SaltParam = Params.GetValue("salt"); SaltParam.empty() == false)
+ {
+ const uint32_t Salt = std::stoi(std::string(SaltParam));
+ SaltHash = IoHash::HashBuffer(&Salt, sizeof Salt);
+ IsUsingSalt = true;
+ }
+
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchOplog(OplogId);
+
+ ProjectStore::Oplog& Oplog = *FoundLog;
+
+ IoBuffer Payload = HttpReq.ReadPayload();
+
+ // This will attempt to open files which may not exist for the case where
+ // the prep step rejected the chunk. This should be fixed since there's
+ // a performance cost associated with any file system activity
+
+ bool IsValid = true;
+ std::vector<IoHash> MissingChunks;
+
+ CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
+ if (m_CidStore.ContainsChunk(Hash))
+ {
+ // Return null attachment as we already have it, no point in reading it and storing it again
+ return {};
+ }
+
+ IoHash AttachmentId;
+ if (IsUsingSalt)
+ {
+ IoHash AttachmentSpec[]{SaltHash, Hash};
+ AttachmentId = IoHash::HashBuffer(MakeMemoryView(AttachmentSpec));
+ }
+ else
+ {
+ AttachmentId = Hash;
+ }
+
+ std::filesystem::path AttachmentPath = Oplog.TempPath() / AttachmentId.ToHexString();
+ if (IoBuffer Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath))
+ {
+ return SharedBuffer(std::move(Data));
+ }
+ else
+ {
+ IsValid = false;
+ MissingChunks.push_back(Hash);
+
+ return {};
+ }
+ };
+
+ CbPackage Package;
+
+ if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver))
+ {
+ std::filesystem::path BadPackagePath =
+ Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId());
+
+ ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath);
+
+ WriteFile(BadPackagePath, Payload);
+
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package");
+ }
+
+ if (!IsValid)
+ {
+ // TODO: emit diagnostics identifying missing chunks
+
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing chunk reference");
+ }
+
+ CbObject Core = Package.GetObject();
+
+ if (!Core["key"sv])
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
+ }
+
+ // Write core to oplog
+
+ const uint32_t OpLsn = Oplog.AppendNewOplogEntry(Package);
+
+ if (OpLsn == ProjectStore::Oplog::kInvalidOp)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
+ }
+
+ ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString());
+
+ HttpReq.WriteResponse(HttpResponseCode::Created);
+ },
+ HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/{op}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const std::string& ProjectId = Req.GetCapture(1);
+ const std::string& OplogId = Req.GetCapture(2);
+ const std::string& OpIdString = Req.GetCapture(3);
+
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchOplog(OplogId);
+
+ ProjectStore::Oplog& Oplog = *FoundLog;
+
+ if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
+ {
+ if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value()))
+ {
+ CbObject& Op = MaybeOp.value();
+ if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage)
+ {
+ CbPackage Package;
+ Package.SetObject(Op);
+
+ Op.IterateAttachments([&](CbFieldView FieldView) {
+ const IoHash AttachmentHash = FieldView.AsAttachment();
+ IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash);
+
+ // We force this for now as content type is not consistently tracked (will
+ // be fixed in CidStore refactor)
+ Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+ if (Payload)
+ {
+ switch (Payload.GetContentType())
+ {
+ case ZenContentType::kCbObject:
+ if (CbObject Object = LoadCompactBinaryObject(Payload))
+ {
+ Package.AddAttachment(CbAttachment(Object));
+ }
+ else
+ {
+ // Error - malformed object
+
+ ZEN_WARN("malformed object returned for {}", AttachmentHash);
+ }
+ break;
+
+ case ZenContentType::kCompressedBinary:
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload)))
+ {
+ Package.AddAttachment(CbAttachment(Compressed, AttachmentHash));
+ }
+ else
+ {
+ // Error - not compressed!
+
+ ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash);
+ }
+ break;
+
+ default:
+ Package.AddAttachment(CbAttachment(SharedBuffer(Payload)));
+ break;
+ }
+ }
+ });
+
+ return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package);
+ }
+ else
+ {
+ // Client cannot accept a package, so we only send the core object
+ return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op);
+ }
+ }
+ }
+
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+ HttpContentType::kText,
+ fmt::format("project {} not found", ProjectId));
+ }
+ Project->TouchProject();
+
+ switch (HttpReq.RequestVerb())
+ {
+ case HttpVerb::kGet:
+ {
+ ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
+ if (!OplogIt)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+ HttpContentType::kText,
+ fmt::format("oplog {} not found in project {}", OplogId, ProjectId));
+ }
+ Project->TouchOplog(OplogId);
+
+ ProjectStore::Oplog& Log = *OplogIt;
+
+ CbObjectWriter Cb;
+ Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str()
+ << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount"
+ << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::max(), Log);
+
+ HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save());
+ }
+ break;
+
+ case HttpVerb::kPost:
+ {
+ if (!m_ProjectStore->AreDiskWritesAllowed())
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+ }
+ std::filesystem::path OplogMarkerPath;
+ if (CbObject Params = HttpReq.ReadPayloadObject())
+ {
+ OplogMarkerPath = Params["gcpath"sv].AsString();
+ }
+
+ ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
+ if (!OplogIt)
+ {
+ if (!Project->NewOplog(OplogId, OplogMarkerPath))
+ {
+ // TODO: indicate why the operation failed!
+ return HttpReq.WriteResponse(HttpResponseCode::InternalServerError);
+ }
+ Project->TouchOplog(OplogId);
+
+ ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath);
+
+ return HttpReq.WriteResponse(HttpResponseCode::Created);
+ }
+
+ // I guess this should ultimately be used to execute RPCs but for now, it
+ // does absolutely nothing
+
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
+ }
+ break;
+
+ case HttpVerb::kDelete:
+ {
+ ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId);
+
+ Project->DeleteOplog(OplogId);
+
+ return HttpReq.WriteResponse(HttpResponseCode::OK);
+ }
+ break;
+
+ default:
+ break;
+ }
+ },
+ HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete);
+
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/entries",
+ [this](HttpRouterRequest& Req) {
+ ZEN_TRACE_CPU("ProjectService::OplogEntries");
+
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ Project->TouchOplog(OplogId);
+
+ CbObjectWriter Response;
+
+ if (FoundLog->OplogCount() > 0)
+ {
+ HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
+
+ if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty())
+ {
+ Oid OpKeyId = OpKeyStringAsOId(OpKey);
+ std::optional<CbObject> Op = FoundLog->GetOpByKey(OpKeyId);
+
+ if (Op.has_value())
+ {
+ Response << "entry"sv << Op.value();
+ }
+ else
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ }
+ else
+ {
+ Response.BeginArray("entries"sv);
+
+ FoundLog->IterateOplog([&Response](CbObject Op) { Response << Op; });
+
+ Response.EndArray();
+ }
+ }
+
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "{project}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const std::string ProjectId = Req.GetCapture(1);
+
+ switch (HttpReq.RequestVerb())
+ {
+ case HttpVerb::kPost:
+ {
+ if (!m_ProjectStore->AreDiskWritesAllowed())
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+ }
+
+ IoBuffer Payload = HttpReq.ReadPayload();
+ CbObject Params = LoadCompactBinaryObject(Payload);
+ std::string_view Id = Params["id"sv].AsString();
+ std::string_view Root = Params["root"sv].AsString();
+ std::string_view EngineRoot = Params["engine"sv].AsString();
+ std::string_view ProjectRoot = Params["project"sv].AsString();
+ std::string_view ProjectFilePath = Params["projectfile"sv].AsString();
+
+ const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId;
+ m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath);
+
+ ZEN_INFO("established project - {} (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
+ ProjectId,
+ Id,
+ Root,
+ EngineRoot,
+ ProjectRoot,
+ ProjectFilePath,
+ ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
+
+ HttpReq.WriteResponse(HttpResponseCode::Created);
+ }
+ break;
+
+ case HttpVerb::kGet:
+ {
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+ HttpContentType::kText,
+ fmt::format("project {} not found", ProjectId));
+ }
+ Project->TouchProject();
+
+ std::vector<std::string> OpLogs = Project->ScanForOplogs();
+
+ CbObjectWriter Response;
+ Response << "id"sv << Project->Identifier;
+ Response << "root"sv << PathToUtf8(Project->RootDir);
+ Response << "engine"sv << PathToUtf8(Project->EngineRootDir);
+ Response << "project"sv << PathToUtf8(Project->ProjectRootDir);
+ Response << "projectfile"sv << PathToUtf8(Project->ProjectFilePath);
+
+ Response.BeginArray("oplogs"sv);
+ for (const std::string& OplogId : OpLogs)
+ {
+ Response.BeginObject();
+ Response << "id"sv << OplogId;
+ Response.EndObject();
+ }
+ Response.EndArray(); // oplogs
+
+ HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
+ }
+ break;
+
+ case HttpVerb::kDelete:
+ {
+ Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+ if (!Project)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+ HttpContentType::kText,
+ fmt::format("project {} not found", ProjectId));
+ }
+
+ ZEN_INFO("deleting project '{}'", ProjectId);
+ if (!m_ProjectStore->DeleteProject(ProjectId))
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::Locked,
+ HttpContentType::kText,
+ fmt::format("project {} is in use", ProjectId));
+ }
+
+ return HttpReq.WriteResponse(HttpResponseCode::NoContent);
+ }
+ break;
+
+ default:
+ break;
+ }
+ },
+ HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete);
+
+ // Push a oplog container
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/save",
+ [this](HttpRouterRequest& Req) {
+ ZEN_TRACE_CPU("ProjectService::OplogSave");
+
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ if (!m_ProjectStore->AreDiskWritesAllowed())
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+ }
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+ if (HttpReq.RequestContentType() != HttpContentType::kCbObject)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type");
+ }
+ IoBuffer Payload = HttpReq.ReadPayload();
+
+ CbObject Response;
+ std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->WriteOplog(ProjectId, OplogId, std::move(Payload), Response);
+ if (Result.first == HttpResponseCode::OK)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
+ }
+ if (Result.second.empty())
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+ },
+ HttpVerb::kPost);
+
+ // Pull a oplog container
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/load",
+ [this](HttpRouterRequest& Req) {
+ ZEN_TRACE_CPU("ProjectService::OplogLoad");
+
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+ if (HttpReq.AcceptContentType() != HttpContentType::kCbObject)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type");
+ }
+ IoBuffer Payload = HttpReq.ReadPayload();
+
+ CbObject Response;
+ std::pair<HttpResponseCode, std::string> Result =
+ m_ProjectStore->ReadOplog(ProjectId, OplogId, HttpReq.GetQueryParams(), Response);
+ if (Result.first == HttpResponseCode::OK)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
+ }
+ if (Result.second.empty())
+ {
+ return HttpReq.WriteResponse(Result.first);
+ }
+ return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+ },
+ HttpVerb::kGet);
+
+ // Do an rpc style operation on project/oplog
+ m_Router.RegisterRoute(
+ "{project}/oplog/{log}/rpc",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+ IoBuffer Payload = HttpReq.ReadPayload();
+
+ m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
+ },
+ HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "details\\$",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+
+ HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
+ bool CSV = Params.GetValue("csv") == "true";
+ bool Details = Params.GetValue("details") == "true";
+ bool OpDetails = Params.GetValue("opdetails") == "true";
+ bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+ if (CSV)
+ {
+ ExtendableStringBuilder<4096> CSVWriter;
+ CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+ m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+ Project.IterateOplogs([&](ProjectStore::Oplog& Oplog) {
+ Oplog.IterateOplogWithKey(
+ [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+ CSVWriteOp(m_CidStore,
+ Project.Identifier,
+ Oplog.OplogId(),
+ Details,
+ AttachmentDetails,
+ LSN,
+ Key,
+ Op,
+ CSVWriter);
+ });
+ });
+ });
+
+ HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+ }
+ else
+ {
+ CbObjectWriter Cbo;
+ Cbo.BeginArray("projects");
+ {
+ m_ProjectStore->DiscoverProjects();
+
+ m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+ std::vector<std::string> OpLogs = Project.ScanForOplogs();
+ CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ });
+ }
+ Cbo.EndArray();
+ HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ }
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "details\\$/{project}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const auto& ProjectId = Req.GetCapture(1);
+
+ HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
+ bool CSV = Params.GetValue("csv") == "true";
+ bool Details = Params.GetValue("details") == "true";
+ bool OpDetails = Params.GetValue("opdetails") == "true";
+ bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+ Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+ if (!FoundProject)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ ProjectStore::Project& Project = *FoundProject.Get();
+
+ if (CSV)
+ {
+ ExtendableStringBuilder<4096> CSVWriter;
+ CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+ FoundProject->IterateOplogs([&](ProjectStore::Oplog& Oplog) {
+ Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN,
+ const Oid& Key,
+ CbObject Op) {
+ CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
+ });
+ HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+ }
+ else
+ {
+ CbObjectWriter Cbo;
+ std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
+ Cbo.BeginArray("projects");
+ {
+ CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ }
+ Cbo.EndArray();
+ HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ }
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "details\\$/{project}/{log}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+
+ HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
+ bool CSV = Params.GetValue("csv") == "true";
+ bool Details = Params.GetValue("details") == "true";
+ bool OpDetails = Params.GetValue("opdetails") == "true";
+ bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+ Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+ if (!FoundProject)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+
+ ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+
+ ProjectStore::Project& Project = *FoundProject.Get();
+ ProjectStore::Oplog& Oplog = *FoundLog;
+ if (CSV)
+ {
+ ExtendableStringBuilder<4096> CSVWriter;
+ CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+ Oplog.IterateOplogWithKey(
+ [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+ CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
+ HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+ }
+ else
+ {
+ CbObjectWriter Cbo;
+ Cbo.BeginArray("oplogs");
+ {
+ CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+ }
+ Cbo.EndArray();
+ HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ }
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "details\\$/{project}/{log}/{chunk}",
+ [this](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const auto& ProjectId = Req.GetCapture(1);
+ const auto& OplogId = Req.GetCapture(2);
+ const auto& ChunkId = Req.GetCapture(3);
+
+ HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
+ bool CSV = Params.GetValue("csv") == "true";
+ bool Details = Params.GetValue("details") == "true";
+ bool OpDetails = Params.GetValue("opdetails") == "true";
+ bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+ Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+ if (!FoundProject)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+
+ ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+
+ if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
+ {
+ return HttpReq.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId));
+ }
+
+ const Oid ObjId = Oid::FromHexString(ChunkId);
+ ProjectStore::Project& Project = *FoundProject.Get();
+ ProjectStore::Oplog& Oplog = *FoundLog;
+
+ int LSN = Oplog.GetOpIndexByKey(ObjId);
+ if (LSN == -1)
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+ std::optional<CbObject> Op = Oplog.GetOpByIndex(LSN);
+ if (!Op.has_value())
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+ }
+
+ if (CSV)
+ {
+ ExtendableStringBuilder<4096> CSVWriter;
+ CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+ CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter);
+ HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+ }
+ else
+ {
+ CbObjectWriter Cbo;
+ Cbo.BeginArray("ops");
+ {
+ CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo);
+ }
+ Cbo.EndArray();
+ HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ }
+ },
+ HttpVerb::kGet);
+}
+
+HttpProjectService::~HttpProjectService()
+{
+ m_StatsService.UnregisterHandler("prj", *this);
+}
+
+const char*
+HttpProjectService::BaseUri() const
+{
+ return "/prj/";
+}
+
+void
+HttpProjectService::HandleRequest(HttpServerRequest& Request)
+{
+ if (m_Router.HandleRequest(Request) == false)
+ {
+ ZEN_WARN("No route found for {0}", Request.RelativeUri());
+ }
+}
+
+void
+HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+ const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+ const CidStoreSize CidSize = m_CidStore.TotalSize();
+
+ CbObjectWriter Cbo;
+ Cbo.BeginObject("store");
+ {
+ Cbo.BeginObject("size");
+ {
+ Cbo << "disk" << StoreSize.DiskSize;
+ Cbo << "memory" << StoreSize.MemorySize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+
+ Cbo.BeginObject("cid");
+ {
+ Cbo.BeginObject("size");
+ {
+ Cbo << "tiny" << CidSize.TinySize;
+ Cbo << "small" << CidSize.SmallSize;
+ Cbo << "large" << CidSize.LargeSize;
+ Cbo << "total" << CidSize.TotalSize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+} // namespace zen \ No newline at end of file
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
new file mode 100644
index 000000000..feb7ca82f
--- /dev/null
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -0,0 +1,55 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenhttp/auth/authmgr.h>
+#include <zenhttp/httpserver.h>
+#include <zenhttp/httpstats.h>
+#include <zenstore/cidstore.h>
+
+namespace zen {
+
+class ProjectStore;
+
+//////////////////////////////////////////////////////////////////////////
+//
+// {project} a project identifier
+// {target} a variation of the project, typically a build target
+// {lsn} oplog entry sequence number
+//
+// /prj/{project}
+// /prj/{project}/oplog/{target}
+// /prj/{project}/oplog/{target}/{lsn}
+//
+// oplog entry
+//
+// id: {id}
+// key: {}
+// meta: {}
+// data: []
+// refs:
+//
+
+class HttpProjectService : public HttpService, public IHttpStatsProvider
+{
+public:
+ HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr);
+ ~HttpProjectService();
+
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(HttpServerRequest& Request) override;
+
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+
+private:
+ inline spdlog::logger& Log() { return m_Log; }
+
+ spdlog::logger& m_Log;
+ CidStore& m_CidStore;
+ HttpRequestRouter m_Router;
+ Ref<ProjectStore> m_ProjectStore;
+ HttpStatsService& m_StatsService;
+ AuthMgr& m_AuthMgr;
+};
+
+} // namespace zen \ No newline at end of file
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index dd4c7597c..08b4a6db5 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -201,228 +201,10 @@ namespace {
: fmt::format("{}. Reason: '{}'", Result.Text, Result.Reason)};
}
- void CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter)
- {
- if (AttachmentDetails)
- {
- CSVWriter << "Project, Oplog, LSN, Key, Cid, Size";
- }
- else if (Details)
- {
- CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize";
- }
- else
- {
- CSVWriter << "Project, Oplog, Key";
- }
- }
-
- void CSVWriteOp(CidStore& CidStore,
- std::string_view ProjectId,
- std::string_view OplogId,
- bool Details,
- bool AttachmentDetails,
- int LSN,
- const Oid& Key,
- CbObject Op,
- StringBuilderBase& CSVWriter)
- {
- StringBuilder<32> KeyStringBuilder;
- Key.ToString(KeyStringBuilder);
- const std::string_view KeyString = KeyStringBuilder.ToView();
-
- SharedBuffer Buffer = Op.GetBuffer();
- if (AttachmentDetails)
- {
- Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- CSVWriter << "\r\n"
- << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << AttachmentHash.ToHexString()
- << ", " << gsl::narrow<uint64_t>(Attachment.GetSize());
- });
- }
- else if (Details)
- {
- uint64_t AttachmentCount = 0;
- size_t AttachmentsSize = 0;
- Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- AttachmentCount++;
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- AttachmentsSize += Attachment.GetSize();
- });
- CSVWriter << "\r\n"
- << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Buffer.GetSize())
- << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
- }
- else
- {
- CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString;
- }
- };
-
- void CbWriteOp(CidStore& CidStore,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- int LSN,
- const Oid& Key,
- CbObject Op,
- CbObjectWriter& CbWriter)
- {
- CbWriter.BeginObject();
- {
- SharedBuffer Buffer = Op.GetBuffer();
- CbWriter.AddObjectId("key", Key);
- if (Details)
- {
- CbWriter.AddInteger("lsn", LSN);
- CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Buffer.GetSize()));
- }
- if (AttachmentDetails)
- {
- CbWriter.BeginArray("attachments");
- Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- CbWriter.BeginObject();
- {
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- CbWriter.AddString("cid", AttachmentHash.ToHexString());
- CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Attachment.GetSize()));
- }
- CbWriter.EndObject();
- });
- CbWriter.EndArray();
- }
- else if (Details)
- {
- uint64_t AttachmentCount = 0;
- size_t AttachmentsSize = 0;
- Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- AttachmentCount++;
- IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
- AttachmentsSize += Attachment.GetSize();
- });
- if (AttachmentCount > 0)
- {
- CbWriter.AddInteger("attachments", AttachmentCount);
- CbWriter.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
- }
- }
- if (OpDetails)
- {
- CbWriter.BeginObject("op");
- for (const CbFieldView& Field : Op)
- {
- if (!Field.HasName())
- {
- CbWriter.AddField(Field);
- continue;
- }
- std::string_view FieldName = Field.GetName();
- CbWriter.AddField(FieldName, Field);
- }
- CbWriter.EndObject();
- }
- }
- CbWriter.EndObject();
- };
-
- void CbWriteOplogOps(CidStore& CidStore,
- ProjectStore::Oplog& Oplog,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginArray("ops");
- {
- Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
- CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
- });
- }
- Cbo.EndArray();
- }
-
- void CbWriteOplog(CidStore& CidStore,
- ProjectStore::Oplog& Oplog,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginObject();
- {
- Cbo.AddString("name", Oplog.OplogId());
- CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndObject();
- }
-
- void CbWriteOplogs(CidStore& CidStore,
- ProjectStore::Project& Project,
- std::vector<std::string> OpLogs,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginArray("oplogs");
- {
- for (const std::string& OpLogId : OpLogs)
- {
- ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId);
- if (Oplog != nullptr)
- {
- CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
- }
- }
- }
- Cbo.EndArray();
- }
-
- void CbWriteProject(CidStore& CidStore,
- ProjectStore::Project& Project,
- std::vector<std::string> OpLogs,
- bool Details,
- bool OpDetails,
- bool AttachmentDetails,
- CbObjectWriter& Cbo)
- {
- Cbo.BeginObject();
- {
- Cbo.AddString("name", Project.Identifier);
- CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndObject();
- }
-
} // namespace
//////////////////////////////////////////////////////////////////////////
-Oid
-OpKeyStringAsOId(std::string_view OpKey)
-{
- using namespace std::literals;
-
- CbObjectWriter Writer;
- Writer << "key"sv << OpKey;
-
- XXH3_128Stream KeyHasher;
- Writer.Save()["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash = KeyHasher.GetHash();
-
- Oid OpId;
- memcpy(OpId.OidBits, &KeyHash, sizeof(OpId.OidBits));
-
- return OpId;
-}
-
-//////////////////////////////////////////////////////////////////////////
-
struct ProjectStore::OplogStorage : public RefCounted
{
OplogStorage(ProjectStore::Oplog* OwnerOplog, std::filesystem::path BasePath) : m_OwnerOplog(OwnerOplog), m_OplogStoragePath(BasePath)
@@ -2728,1264 +2510,6 @@ ProjectStore::AreDiskWritesAllowed() const
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
-: m_Log(logging::Get("project"))
-, m_CidStore(Store)
-, m_ProjectStore(Projects)
-, m_StatsService(StatsService)
-, m_AuthMgr(AuthMgr)
-{
- using namespace std::literals;
-
- m_StatsService.RegisterHandler("prj", *this);
-
- m_Router.AddPattern("project", "([[:alnum:]_.]+)");
- m_Router.AddPattern("log", "([[:alnum:]_.]+)");
- m_Router.AddPattern("op", "([[:digit:]]+?)");
- m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
- m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
-
- m_Router.RegisterRoute(
- "",
- [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_ProjectStore->GetProjectsList()); },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "list",
- [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_ProjectStore->GetProjectsList()); },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/batch",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- // Parse Request
-
- IoBuffer Payload = HttpReq.ReadPayload();
- BinaryReader Reader(Payload);
-
- struct RequestHeader
- {
- enum
- {
- kMagic = 0xAAAA'77AC
- };
- uint32_t Magic;
- uint32_t ChunkCount;
- uint32_t Reserved1;
- uint32_t Reserved2;
- };
-
- struct RequestChunkEntry
- {
- Oid ChunkId;
- uint32_t CorrelationId;
- uint64_t Offset;
- uint64_t RequestBytes;
- };
-
- if (Payload.Size() <= sizeof(RequestHeader))
- {
- HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
-
- RequestHeader RequestHdr;
- Reader.Read(&RequestHdr, sizeof RequestHdr);
-
- if (RequestHdr.Magic != RequestHeader::kMagic)
- {
- HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
-
- std::vector<RequestChunkEntry> RequestedChunks;
- RequestedChunks.resize(RequestHdr.ChunkCount);
- Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount);
-
- // Make Response
-
- struct ResponseHeader
- {
- uint32_t Magic = 0xbada'b00f;
- uint32_t ChunkCount;
- uint32_t Reserved1 = 0;
- uint32_t Reserved2 = 0;
- };
-
- struct ResponseChunkEntry
- {
- uint32_t CorrelationId;
- uint32_t Flags = 0;
- uint64_t ChunkSize;
- };
-
- std::vector<IoBuffer> OutBlobs;
- OutBlobs.emplace_back(sizeof(ResponseHeader) + RequestHdr.ChunkCount * sizeof(ResponseChunkEntry));
- for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
- {
- const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
- IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId);
- if (FoundChunk)
- {
- if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1))
- {
- uint64_t Offset = RequestedChunk.Offset;
- if (Offset > FoundChunk.Size())
- {
- Offset = FoundChunk.Size();
- }
- uint64_t Size = RequestedChunk.RequestBytes;
- if ((Offset + Size) > FoundChunk.Size())
- {
- Size = FoundChunk.Size() - Offset;
- }
- FoundChunk = IoBuffer(FoundChunk, Offset, Size);
- }
- }
- OutBlobs.emplace_back(std::move(FoundChunk));
- }
- uint8_t* ResponsePtr = reinterpret_cast<uint8_t*>(OutBlobs[0].MutableData());
- ResponseHeader ResponseHdr;
- ResponseHdr.ChunkCount = RequestHdr.ChunkCount;
- memcpy(ResponsePtr, &ResponseHdr, sizeof(ResponseHdr));
- ResponsePtr += sizeof(ResponseHdr);
- for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
- {
- // const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
- const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]);
- ResponseChunkEntry ResponseChunk;
- ResponseChunk.CorrelationId = ChunkIndex;
- if (FoundChunk)
- {
- ResponseChunk.ChunkSize = FoundChunk.Size();
- }
- else
- {
- ResponseChunk.ChunkSize = uint64_t(-1);
- }
- memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk));
- ResponsePtr += sizeof(ResponseChunk);
- }
- return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/files",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- // File manifest fetch, returns the client file list
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
-
- const bool FilterClient = Params.GetValue("filter"sv) == "client"sv;
-
- CbObject ResponsePayload;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetProjectFiles(ProjectId, OplogId, FilterClient, ResponsePayload);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{chunk}/info",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& ChunkId = Req.GetCapture(3);
-
- CbObject ResponsePayload;
- std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunkInfo(ProjectId, OplogId, ChunkId, ResponsePayload);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
- }
- else if (Result.first == HttpResponseCode::NotFound)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{chunk}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& ChunkId = Req.GetCapture(3);
-
- uint64_t Offset = 0;
- uint64_t Size = ~(0ull);
-
- auto QueryParms = HttpReq.GetQueryParams();
-
- if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false)
- {
- if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm))
- {
- Offset = OffsetVal.value();
- }
- else
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
- }
-
- if (auto SizeParm = QueryParms.GetValue("size"); SizeParm.empty() == false)
- {
- if (auto SizeVal = ParseInt<uint64_t>(SizeParm))
- {
- Size = SizeVal.value();
- }
- else
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
- }
-
- HttpContentType AcceptType = HttpReq.AcceptContentType();
-
- IoBuffer Chunk;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk);
- if (Result.first == HttpResponseCode::OK)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Chunk.GetContentType()));
- return HttpReq.WriteResponse(HttpResponseCode::OK, Chunk.GetContentType(), Chunk);
- }
- else if (Result.first == HttpResponseCode::NotFound)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet | HttpVerb::kHead);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{hash}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& Cid = Req.GetCapture(3);
- HttpContentType AcceptType = HttpReq.AcceptContentType();
- HttpContentType RequestType = HttpReq.RequestContentType();
-
- switch (HttpReq.RequestVerb())
- {
- case HttpVerb::kGet:
- {
- IoBuffer Value;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value);
-
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
- }
- else if (Result.first == HttpResponseCode::NotFound)
- {
- ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- }
- case HttpVerb::kPost:
- {
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->PutChunk(ProjectId, OplogId, Cid, RequestType, HttpReq.ReadPayload());
- if (Result.first == HttpResponseCode::OK || Result.first == HttpResponseCode::Created)
- {
- return HttpReq.WriteResponse(Result.first);
- }
- else
- {
- ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
- ToString(HttpReq.RequestVerb()),
- HttpReq.QueryString(),
- static_cast<int>(Result.first),
- Result.second);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- }
- break;
- }
- },
- HttpVerb::kGet | HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/prep",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogPrep");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- // This operation takes a list of referenced hashes and decides which
- // chunks are not present on this server. This list is then returned in
- // the "need" list in the response
-
- IoBuffer Payload = HttpReq.ReadPayload();
- CbObject RequestObject = LoadCompactBinaryObject(Payload);
-
- std::vector<IoHash> NeedList;
-
- for (auto Entry : RequestObject["have"sv])
- {
- const IoHash FileHash = Entry.AsHash();
-
- if (!m_CidStore.ContainsChunk(FileHash))
- {
- ZEN_DEBUG("prep - NEED: {}", FileHash);
-
- NeedList.push_back(FileHash);
- }
- }
-
- CbObjectWriter Cbo;
- Cbo.BeginArray("need");
-
- for (const IoHash& Hash : NeedList)
- {
- Cbo << Hash;
- }
-
- Cbo.EndArray();
- CbObject Response = Cbo.Save();
-
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/new",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogNew");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
-
- bool IsUsingSalt = false;
- IoHash SaltHash = IoHash::Zero;
-
- if (std::string_view SaltParam = Params.GetValue("salt"); SaltParam.empty() == false)
- {
- const uint32_t Salt = std::stoi(std::string(SaltParam));
- SaltHash = IoHash::HashBuffer(&Salt, sizeof Salt);
- IsUsingSalt = true;
- }
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- ProjectStore::Oplog& Oplog = *FoundLog;
-
- IoBuffer Payload = HttpReq.ReadPayload();
-
- // This will attempt to open files which may not exist for the case where
- // the prep step rejected the chunk. This should be fixed since there's
- // a performance cost associated with any file system activity
-
- bool IsValid = true;
- std::vector<IoHash> MissingChunks;
-
- CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
- if (m_CidStore.ContainsChunk(Hash))
- {
- // Return null attachment as we already have it, no point in reading it and storing it again
- return {};
- }
-
- IoHash AttachmentId;
- if (IsUsingSalt)
- {
- IoHash AttachmentSpec[]{SaltHash, Hash};
- AttachmentId = IoHash::HashBuffer(MakeMemoryView(AttachmentSpec));
- }
- else
- {
- AttachmentId = Hash;
- }
-
- std::filesystem::path AttachmentPath = Oplog.TempPath() / AttachmentId.ToHexString();
- if (IoBuffer Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath))
- {
- return SharedBuffer(std::move(Data));
- }
- else
- {
- IsValid = false;
- MissingChunks.push_back(Hash);
-
- return {};
- }
- };
-
- CbPackage Package;
-
- if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver))
- {
- std::filesystem::path BadPackagePath =
- Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId());
-
- ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath);
-
- WriteFile(BadPackagePath, Payload);
-
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package");
- }
-
- if (!IsValid)
- {
- // TODO: emit diagnostics identifying missing chunks
-
- return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing chunk reference");
- }
-
- CbObject Core = Package.GetObject();
-
- if (!Core["key"sv])
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
- }
-
- // Write core to oplog
-
- const uint32_t OpLsn = Oplog.AppendNewOplogEntry(Package);
-
- if (OpLsn == ProjectStore::Oplog::kInvalidOp)
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
-
- ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString());
-
- HttpReq.WriteResponse(HttpResponseCode::Created);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/{op}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const std::string& ProjectId = Req.GetCapture(1);
- const std::string& OplogId = Req.GetCapture(2);
- const std::string& OpIdString = Req.GetCapture(3);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- ProjectStore::Oplog& Oplog = *FoundLog;
-
- if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
- {
- if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value()))
- {
- CbObject& Op = MaybeOp.value();
- if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage)
- {
- CbPackage Package;
- Package.SetObject(Op);
-
- Op.IterateAttachments([&](CbFieldView FieldView) {
- const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash);
-
- // We force this for now as content type is not consistently tracked (will
- // be fixed in CidStore refactor)
- Payload.SetContentType(ZenContentType::kCompressedBinary);
-
- if (Payload)
- {
- switch (Payload.GetContentType())
- {
- case ZenContentType::kCbObject:
- if (CbObject Object = LoadCompactBinaryObject(Payload))
- {
- Package.AddAttachment(CbAttachment(Object));
- }
- else
- {
- // Error - malformed object
-
- ZEN_WARN("malformed object returned for {}", AttachmentHash);
- }
- break;
-
- case ZenContentType::kCompressedBinary:
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload)))
- {
- Package.AddAttachment(CbAttachment(Compressed, AttachmentHash));
- }
- else
- {
- // Error - not compressed!
-
- ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash);
- }
- break;
-
- default:
- Package.AddAttachment(CbAttachment(SharedBuffer(Payload)));
- break;
- }
- }
- });
-
- return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package);
- }
- else
- {
- // Client cannot accept a package, so we only send the core object
- return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op);
- }
- }
- }
-
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
-
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("project {} not found", ProjectId));
- }
- Project->TouchProject();
-
- switch (HttpReq.RequestVerb())
- {
- case HttpVerb::kGet:
- {
- ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
- if (!OplogIt)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("oplog {} not found in project {}", OplogId, ProjectId));
- }
- Project->TouchOplog(OplogId);
-
- ProjectStore::Oplog& Log = *OplogIt;
-
- CbObjectWriter Cb;
- Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str()
- << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount"
- << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::max(), Log);
-
- HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save());
- }
- break;
-
- case HttpVerb::kPost:
- {
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
- std::filesystem::path OplogMarkerPath;
- if (CbObject Params = HttpReq.ReadPayloadObject())
- {
- OplogMarkerPath = Params["gcpath"sv].AsString();
- }
-
- ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
- if (!OplogIt)
- {
- if (!Project->NewOplog(OplogId, OplogMarkerPath))
- {
- // TODO: indicate why the operation failed!
- return HttpReq.WriteResponse(HttpResponseCode::InternalServerError);
- }
- Project->TouchOplog(OplogId);
-
- ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath);
-
- return HttpReq.WriteResponse(HttpResponseCode::Created);
- }
-
- // I guess this should ultimately be used to execute RPCs but for now, it
- // does absolutely nothing
-
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
- }
- break;
-
- case HttpVerb::kDelete:
- {
- ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId);
-
- Project->DeleteOplog(OplogId);
-
- return HttpReq.WriteResponse(HttpResponseCode::OK);
- }
- break;
-
- default:
- break;
- }
- },
- HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete);
-
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/entries",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogEntries");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- Project->TouchOplog(OplogId);
-
- CbObjectWriter Response;
-
- if (FoundLog->OplogCount() > 0)
- {
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
-
- if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty())
- {
- Oid OpKeyId = OpKeyStringAsOId(OpKey);
- std::optional<CbObject> Op = FoundLog->GetOpByKey(OpKeyId);
-
- if (Op.has_value())
- {
- Response << "entry"sv << Op.value();
- }
- else
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- }
- else
- {
- Response.BeginArray("entries"sv);
-
- FoundLog->IterateOplog([&Response](CbObject Op) { Response << Op; });
-
- Response.EndArray();
- }
- }
-
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "{project}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const std::string ProjectId = Req.GetCapture(1);
-
- switch (HttpReq.RequestVerb())
- {
- case HttpVerb::kPost:
- {
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
-
- IoBuffer Payload = HttpReq.ReadPayload();
- CbObject Params = LoadCompactBinaryObject(Payload);
- std::string_view Id = Params["id"sv].AsString();
- std::string_view Root = Params["root"sv].AsString();
- std::string_view EngineRoot = Params["engine"sv].AsString();
- std::string_view ProjectRoot = Params["project"sv].AsString();
- std::string_view ProjectFilePath = Params["projectfile"sv].AsString();
-
- const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId;
- m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath);
-
- ZEN_INFO("established project - {} (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
- ProjectId,
- Id,
- Root,
- EngineRoot,
- ProjectRoot,
- ProjectFilePath,
- ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
-
- HttpReq.WriteResponse(HttpResponseCode::Created);
- }
- break;
-
- case HttpVerb::kGet:
- {
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("project {} not found", ProjectId));
- }
- Project->TouchProject();
-
- std::vector<std::string> OpLogs = Project->ScanForOplogs();
-
- CbObjectWriter Response;
- Response << "id"sv << Project->Identifier;
- Response << "root"sv << PathToUtf8(Project->RootDir);
- Response << "engine"sv << PathToUtf8(Project->EngineRootDir);
- Response << "project"sv << PathToUtf8(Project->ProjectRootDir);
- Response << "projectfile"sv << PathToUtf8(Project->ProjectFilePath);
-
- Response.BeginArray("oplogs"sv);
- for (const std::string& OplogId : OpLogs)
- {
- Response.BeginObject();
- Response << "id"sv << OplogId;
- Response.EndObject();
- }
- Response.EndArray(); // oplogs
-
- HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
- }
- break;
-
- case HttpVerb::kDelete:
- {
- Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
- if (!Project)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("project {} not found", ProjectId));
- }
-
- ZEN_INFO("deleting project '{}'", ProjectId);
- if (!m_ProjectStore->DeleteProject(ProjectId))
- {
- return HttpReq.WriteResponse(HttpResponseCode::Locked,
- HttpContentType::kText,
- fmt::format("project {} is in use", ProjectId));
- }
-
- return HttpReq.WriteResponse(HttpResponseCode::NoContent);
- }
- break;
-
- default:
- break;
- }
- },
- HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete);
-
- // Push a oplog container
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/save",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogSave");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- if (!m_ProjectStore->AreDiskWritesAllowed())
- {
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- if (HttpReq.RequestContentType() != HttpContentType::kCbObject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type");
- }
- IoBuffer Payload = HttpReq.ReadPayload();
-
- CbObject Response;
- std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->WriteOplog(ProjectId, OplogId, std::move(Payload), Response);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kPost);
-
- // Pull a oplog container
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/load",
- [this](HttpRouterRequest& Req) {
- ZEN_TRACE_CPU("ProjectService::OplogLoad");
-
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- if (HttpReq.AcceptContentType() != HttpContentType::kCbObject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type");
- }
- IoBuffer Payload = HttpReq.ReadPayload();
-
- CbObject Response;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->ReadOplog(ProjectId, OplogId, HttpReq.GetQueryParams(), Response);
- if (Result.first == HttpResponseCode::OK)
- {
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
- }
- if (Result.second.empty())
- {
- return HttpReq.WriteResponse(Result.first);
- }
- return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
- },
- HttpVerb::kGet);
-
- // Do an rpc style operation on project/oplog
- m_Router.RegisterRoute(
- "{project}/oplog/{log}/rpc",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- IoBuffer Payload = HttpReq.ReadPayload();
-
- m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
- },
- HttpVerb::kPost);
-
- m_Router.RegisterRoute(
- "details\\$",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- Project.IterateOplogs([&](ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
- CSVWriteOp(m_CidStore,
- Project.Identifier,
- Oplog.OplogId(),
- Details,
- AttachmentDetails,
- LSN,
- Key,
- Op,
- CSVWriter);
- });
- });
- });
-
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- Cbo.BeginArray("projects");
- {
- m_ProjectStore->DiscoverProjects();
-
- m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- std::vector<std::string> OpLogs = Project.ScanForOplogs();
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
- });
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "details\\$/{project}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
- if (!FoundProject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- ProjectStore::Project& Project = *FoundProject.Get();
-
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- FoundProject->IterateOplogs([&](ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN,
- const Oid& Key,
- CbObject Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
- });
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
- Cbo.BeginArray("projects");
- {
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "details\\$/{project}/{log}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
- if (!FoundProject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- Cbo.BeginArray("oplogs");
- {
- CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "details\\$/{project}/{log}/{chunk}",
- [this](HttpRouterRequest& Req) {
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const auto& ProjectId = Req.GetCapture(1);
- const auto& OplogId = Req.GetCapture(2);
- const auto& ChunkId = Req.GetCapture(3);
-
- HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams();
- bool CSV = Params.GetValue("csv") == "true";
- bool Details = Params.GetValue("details") == "true";
- bool OpDetails = Params.GetValue("opdetails") == "true";
- bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
-
- Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
- if (!FoundProject)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
- if (!FoundLog)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
- {
- return HttpReq.WriteResponse(
- HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId));
- }
-
- const Oid ObjId = Oid::FromHexString(ChunkId);
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
-
- int LSN = Oplog.GetOpIndexByKey(ObjId);
- if (LSN == -1)
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
- std::optional<CbObject> Op = Oplog.GetOpByIndex(LSN);
- if (!Op.has_value())
- {
- return HttpReq.WriteResponse(HttpResponseCode::NotFound);
- }
-
- if (CSV)
- {
- ExtendableStringBuilder<4096> CSVWriter;
- CSVHeader(Details, AttachmentDetails, CSVWriter);
-
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter);
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
- }
- else
- {
- CbObjectWriter Cbo;
- Cbo.BeginArray("ops");
- {
- CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo);
- }
- Cbo.EndArray();
- HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
- }
- },
- HttpVerb::kGet);
-}
-
-HttpProjectService::~HttpProjectService()
-{
- m_StatsService.UnregisterHandler("prj", *this);
-}
-
-const char*
-HttpProjectService::BaseUri() const
-{
- return "/prj/";
-}
-
-void
-HttpProjectService::HandleRequest(HttpServerRequest& Request)
-{
- if (m_Router.HandleRequest(Request) == false)
- {
- ZEN_WARN("No route found for {0}", Request.RelativeUri());
- }
-}
-
-void
-HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
-{
- const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
- const CidStoreSize CidSize = m_CidStore.TotalSize();
-
- CbObjectWriter Cbo;
- Cbo.BeginObject("store");
- {
- Cbo.BeginObject("size");
- {
- Cbo << "disk" << StoreSize.DiskSize;
- Cbo << "memory" << StoreSize.MemorySize;
- }
- Cbo.EndObject();
- }
- Cbo.EndObject();
-
- Cbo.BeginObject("cid");
- {
- Cbo.BeginObject("size");
- {
- Cbo << "tiny" << CidSize.TinySize;
- Cbo << "small" << CidSize.SmallSize;
- Cbo << "large" << CidSize.LargeSize;
- Cbo << "total" << CidSize.TotalSize;
- }
- Cbo.EndObject();
- }
- Cbo.EndObject();
-
- return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
-}
-
-//////////////////////////////////////////////////////////////////////////
-
#if ZEN_WITH_TESTS
namespace testutils {
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 27e06a495..8b5369e98 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -2,16 +2,18 @@
#pragma once
+#include <zencore/compactbinary.h>
#include <zencore/uid.h>
#include <zencore/xxhash.h>
#include <zenhttp/httpserver.h>
-#include <zenhttp/httpstats.h>
#include <zenstore/gc.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
ZEN_THIRD_PARTY_INCLUDES_END
+#include <map>
+
namespace zen {
class CbPackage;
@@ -19,6 +21,8 @@ class CidStore;
class AuthMgr;
class ScrubContext;
+enum class HttpResponseCode;
+
struct OplogEntry
{
uint32_t OpLsn;
@@ -335,47 +339,6 @@ private:
std::filesystem::path BasePathForProject(std::string_view ProjectId);
};
-//////////////////////////////////////////////////////////////////////////
-//
-// {project} a project identifier
-// {target} a variation of the project, typically a build target
-// {lsn} oplog entry sequence number
-//
-// /prj/{project}
-// /prj/{project}/oplog/{target}
-// /prj/{project}/oplog/{target}/{lsn}
-//
-// oplog entry
-//
-// id: {id}
-// key: {}
-// meta: {}
-// data: []
-// refs:
-//
-
-class HttpProjectService : public HttpService, public IHttpStatsProvider
-{
-public:
- HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr);
- ~HttpProjectService();
-
- virtual const char* BaseUri() const override;
- virtual void HandleRequest(HttpServerRequest& Request) override;
-
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
-
-private:
- inline spdlog::logger& Log() { return m_Log; }
-
- spdlog::logger& m_Log;
- CidStore& m_CidStore;
- HttpRequestRouter m_Router;
- Ref<ProjectStore> m_ProjectStore;
- HttpStatsService& m_StatsService;
- AuthMgr& m_AuthMgr;
-};
-
void prj_forcelink();
} // namespace zen
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index e57c217d1..1f96781fd 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -121,6 +121,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
#include "frontend/frontend.h"
#include "httpcidstore.h"
#include "objectstore/objectstore.h"
+#include "projectstore/httpprojectstore.h"
#include "projectstore/projectstore.h"
#include "upstream/upstream.h"