aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver
diff options
context:
space:
mode:
authorFlorent Devillechabrol <[email protected]>2025-04-02 10:38:02 -0700
committerGitHub Enterprise <[email protected]>2025-04-02 10:38:02 -0700
commit486a22ad2c61bc1616d8745e0077eb320089bfec (patch)
tree665d5c9002cd97c04ddffeaf211fcf8e55d01dce /src/zenserver
parentFixed missing trailing quote when converting binary data from compact binary ... (diff)
parentadded --find-max-block-count option to builds upload (#337) (diff)
downloadzen-486a22ad2c61bc1616d8745e0077eb320089bfec.tar.xz
zen-486a22ad2c61bc1616d8745e0077eb320089bfec.zip
Merge branch 'main' into fd-fix-binary-json
Diffstat (limited to 'src/zenserver')
-rw-r--r--src/zenserver/admin/admin.cpp18
-rw-r--r--src/zenserver/admin/admin.h3
-rw-r--r--src/zenserver/buildstore/httpbuildstore.cpp539
-rw-r--r--src/zenserver/buildstore/httpbuildstore.h68
-rw-r--r--src/zenserver/config.cpp22
-rw-r--r--src/zenserver/config.h12
-rw-r--r--src/zenserver/frontend/frontend.cpp19
-rw-r--r--src/zenserver/frontend/frontend.h7
-rw-r--r--src/zenserver/frontend/html/pages/entry.js69
-rw-r--r--src/zenserver/main.cpp6
-rw-r--r--src/zenserver/objectstore/objectstore.cpp24
-rw-r--r--src/zenserver/objectstore/objectstore.h7
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp2
-rw-r--r--src/zenserver/projectstore/fileremoteprojectstore.cpp10
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp80
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h11
-rw-r--r--src/zenserver/projectstore/projectstore.cpp132
-rw-r--r--src/zenserver/projectstore/projectstore.h1
-rw-r--r--src/zenserver/projectstore/remoteprojectstore.cpp6
-rw-r--r--src/zenserver/vfs/vfsservice.cpp15
-rw-r--r--src/zenserver/vfs/vfsservice.h9
-rw-r--r--src/zenserver/workspaces/httpworkspaces.cpp25
-rw-r--r--src/zenserver/workspaces/httpworkspaces.h10
-rw-r--r--src/zenserver/zenserver.cpp36
-rw-r--r--src/zenserver/zenserver.h4
25 files changed, 1001 insertions, 134 deletions
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 2888f5450..73166e608 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -20,6 +20,7 @@
#include <zenstore/cidstore.h>
#include <zenstore/gc.h>
+#include <zenstore/buildstore/buildstore.h>
#include <zenstore/cache/structuredcachestore.h>
#include <zenutil/workerpools.h>
#include "config.h"
@@ -39,7 +40,7 @@ struct DirStats
DirStats
GetStatsForDirectory(std::filesystem::path Dir)
{
- if (!std::filesystem::exists(Dir))
+ if (!IsDir(Dir))
return {};
struct StatsTraversal : public GetDirectoryContentVisitor
@@ -105,6 +106,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
ZenCacheStore* CacheStore,
CidStore* CidStore,
ProjectStore* ProjectStore,
+ BuildStore* BuildStore,
const LogPaths& LogPaths,
const ZenServerOptions& ServerOptions)
: m_GcScheduler(Scheduler)
@@ -112,6 +114,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
, m_CacheStore(CacheStore)
, m_CidStore(CidStore)
, m_ProjectStore(ProjectStore)
+, m_BuildStore(BuildStore)
, m_LogPaths(LogPaths)
, m_ServerOptions(ServerOptions)
{
@@ -306,6 +309,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
Response << "Interval" << ToTimeSpan(State.Config.Interval);
Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration);
Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration);
+ Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration);
Response << "CollectSmallObjects" << State.Config.CollectSmallObjects;
Response << "Enabled" << State.Config.Enabled;
Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize);
@@ -401,6 +405,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
}
}
+ if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false)
+ {
+ if (auto Value = ParseInt<uint64_t>(Param))
+ {
+ GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value());
+ }
+ }
+
if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false)
{
if (auto Value = ParseInt<uint64_t>(Param))
@@ -782,6 +794,10 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
{
m_ProjectStore->Flush();
}
+ if (m_BuildStore)
+ {
+ m_BuildStore->Flush();
+ }
HttpReq.WriteResponse(HttpResponseCode::OK);
},
HttpVerb::kPost);
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 563c4f536..e7821dead 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -12,6 +12,7 @@ class JobQueue;
class ZenCacheStore;
class CidStore;
class ProjectStore;
+class BuildStore;
struct ZenServerOptions;
class HttpAdminService : public zen::HttpService
@@ -28,6 +29,7 @@ public:
ZenCacheStore* CacheStore,
CidStore* CidStore,
ProjectStore* ProjectStore,
+ BuildStore* BuildStore,
const LogPaths& LogPaths,
const ZenServerOptions& ServerOptions);
~HttpAdminService();
@@ -42,6 +44,7 @@ private:
ZenCacheStore* m_CacheStore;
CidStore* m_CidStore;
ProjectStore* m_ProjectStore;
+ BuildStore* m_BuildStore;
LogPaths m_LogPaths;
const ZenServerOptions& m_ServerOptions;
};
diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp
new file mode 100644
index 000000000..c918f5683
--- /dev/null
+++ b/src/zenserver/buildstore/httpbuildstore.cpp
@@ -0,0 +1,539 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httpbuildstore.h"
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/compactbinaryvalue.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/trace.h>
+#include <zenhttp/packageformat.h>
+#include <zenstore/buildstore/buildstore.h>
+#include <zenutil/workerpools.h>
+
+#include <numeric>
+
+namespace zen {
+using namespace std::literals;
+
+ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv);
+
+HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store)
+: m_Log(logging::Get("builds"))
+, m_StatusService(StatusService)
+, m_StatsService(StatsService)
+, m_BuildStore(Store)
+{
+ Initialize();
+
+ m_StatusService.RegisterHandler("builds", *this);
+ m_StatsService.RegisterHandler("builds", *this);
+}
+
+HttpBuildStoreService::~HttpBuildStoreService()
+{
+ m_StatsService.UnregisterHandler("builds", *this);
+ m_StatusService.UnregisterHandler("builds", *this);
+}
+
+const char*
+HttpBuildStoreService::BaseUri() const
+{
+ return "/builds/";
+}
+
+void
+HttpBuildStoreService::Initialize()
+{
+ ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service");
+
+ m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)");
+ m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)");
+ m_Router.AddPattern("buildid", "([[:xdigit:]]{24})");
+ m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
+
+ m_Router.RegisterRoute(
+ "{namespace}/{bucket}/{buildid}/blobs/{hash}",
+ [this](HttpRouterRequest& Req) { PutBlobRequest(Req); },
+ HttpVerb::kPut);
+
+ m_Router.RegisterRoute(
+ "{namespace}/{bucket}/{buildid}/blobs/{hash}",
+ [this](HttpRouterRequest& Req) { GetBlobRequest(Req); },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata",
+ [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); },
+ HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata",
+ [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); },
+ HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "{namespace}/{bucket}/{buildid}/blobs/exists",
+ [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); },
+ HttpVerb::kPost);
+}
+
+void
+HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest");
+ metrics::OperationTiming::Scope $(m_HttpRequests);
+
+ m_BuildStoreStats.RequestCount++;
+ if (m_Router.HandleRequest(Request) == false)
+ {
+ ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri());
+ return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
+ }
+}
+
+void
+HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest");
+ HttpServerRequest& ServerRequest = Req.ServerRequest();
+ const std::string_view Namespace = Req.GetCapture(1);
+ const std::string_view Bucket = Req.GetCapture(2);
+ const std::string_view BuildId = Req.GetCapture(3);
+ const std::string_view Hash = Req.GetCapture(4);
+ ZEN_UNUSED(Namespace, Bucket, BuildId);
+ IoHash BlobHash;
+ if (!IoHash::TryParse(Hash, BlobHash))
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Invalid blob hash '{}'", Hash));
+ }
+ m_BuildStoreStats.BlobWriteCount++;
+ IoBuffer Payload = ServerRequest.ReadPayload();
+ if (!Payload)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Payload blob {} is empty", Hash));
+ }
+ if (Payload.GetContentType() != HttpContentType::kCompressedBinary)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType())));
+ }
+ m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload());
+ // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize());
+ return ServerRequest.WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest");
+ HttpServerRequest& ServerRequest = Req.ServerRequest();
+ std::string_view Namespace = Req.GetCapture(1);
+ std::string_view Bucket = Req.GetCapture(2);
+ std::string_view BuildId = Req.GetCapture(3);
+ std::string_view Hash = Req.GetCapture(4);
+ ZEN_UNUSED(Namespace, Bucket, BuildId);
+ IoHash BlobHash;
+ if (!IoHash::TryParse(Hash, BlobHash))
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Invalid blob hash '{}'", Hash));
+ }
+ zen::HttpRanges Ranges;
+ bool HasRange = ServerRequest.TryGetRanges(Ranges);
+ if (Ranges.size() > 1)
+ {
+ // Only a single range is supported
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Multiple ranges in blob request is not supported");
+ }
+
+ m_BuildStoreStats.BlobReadCount++;
+ IoBuffer Blob = m_BuildStore.GetBlob(BlobHash);
+ if (!Blob)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::NotFound,
+ HttpContentType::kText,
+ fmt::format("Blob with hash '{}' could not be found", Hash));
+ }
+ // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize());
+ m_BuildStoreStats.BlobHitCount++;
+ if (HasRange)
+ {
+ const HttpRange& Range = Ranges.front();
+ const uint64_t BlobSize = Blob.GetSize();
+ const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0;
+ const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize);
+ if (Range.Start + RangeSize >= BlobSize)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+ }
+ Blob = IoBuffer(Blob, Range.Start, RangeSize);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
+ }
+ else
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+ }
+}
+
+void
+HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest");
+ HttpServerRequest& ServerRequest = Req.ServerRequest();
+ std::string_view Namespace = Req.GetCapture(1);
+ std::string_view Bucket = Req.GetCapture(2);
+ std::string_view BuildId = Req.GetCapture(3);
+
+ IoBuffer MetaPayload = ServerRequest.ReadPayload();
+ if (MetaPayload.GetContentType() != ZenContentType::kCbPackage)
+ {
+ throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'",
+ ToString(MetaPayload.GetContentType()),
+ ToString(ZenContentType::kCbPackage)));
+ }
+ CbPackage Message = ParsePackageMessage(MetaPayload);
+
+ CbObjectView MessageObject = Message.GetObject();
+ if (!MessageObject)
+ {
+ throw std::runtime_error("PutMetadataRequest payload object is missing");
+ }
+ CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView();
+ CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView();
+
+ const uint64_t BlobCount = BlobsArray.Num();
+ if (BlobCount == 0)
+ {
+ throw std::runtime_error("PutMetadataRequest blobs array is empty");
+ }
+ if (BlobCount != MetadataArray.Num())
+ {
+ throw std::runtime_error(
+ fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount));
+ }
+
+ std::vector<IoHash> BlobHashes;
+ std::vector<IoBuffer> MetadataPayloads;
+
+ BlobHashes.reserve(BlobCount);
+ MetadataPayloads.reserve(BlobCount);
+
+ auto BlobsArrayIt = begin(BlobsArray);
+ auto MetadataArrayIt = begin(MetadataArray);
+ while (BlobsArrayIt != end(BlobsArray))
+ {
+ const IoHash BlobHash = (*BlobsArrayIt).AsHash();
+ const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment();
+
+ const CbAttachment* Attachment = Message.FindAttachment(MetadataHash);
+ if (Attachment == nullptr)
+ {
+ throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash));
+ }
+ BlobHashes.push_back(BlobHash);
+ if (Attachment->IsObject())
+ {
+ MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer());
+ MetadataPayloads.back().SetContentType(ZenContentType::kCbObject);
+ }
+ else if (Attachment->IsCompressedBinary())
+ {
+ MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer());
+ MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary);
+ }
+ else
+ {
+ ZEN_ASSERT(Attachment->IsBinary());
+ MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer());
+ MetadataPayloads.back().SetContentType(ZenContentType::kBinary);
+ }
+
+ BlobsArrayIt++;
+ MetadataArrayIt++;
+ }
+ m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest");
+ HttpServerRequest& ServerRequest = Req.ServerRequest();
+ std::string_view Namespace = Req.GetCapture(1);
+ std::string_view Bucket = Req.GetCapture(2);
+ std::string_view BuildId = Req.GetCapture(3);
+ ZEN_UNUSED(Namespace, Bucket, BuildId);
+ IoBuffer RequestPayload = ServerRequest.ReadPayload();
+ if (!RequestPayload)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Expected compact binary body for metadata request, body is missing");
+ }
+ if (RequestPayload.GetContentType() != HttpContentType::kCbObject)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType())));
+ }
+ if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default);
+ ValidateError != CbValidateError::None)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError)));
+ }
+ CbObject RequestObject = LoadCompactBinaryObject(RequestPayload);
+ CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView();
+ if (!BlobsArray)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Compact binary body for metadata request is missing 'blobHashes' array");
+ }
+ const uint64_t BlobCount = BlobsArray.Num();
+
+ std::vector<IoHash> BlobRawHashes;
+ BlobRawHashes.reserve(BlobCount);
+ for (CbFieldView BlockHashView : BlobsArray)
+ {
+ BlobRawHashes.push_back(BlockHashView.AsHash());
+ if (BlobRawHashes.back() == IoHash::Zero)
+ {
+ const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType();
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type));
+ }
+ }
+ m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size();
+ std::vector<IoBuffer> BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst));
+
+ CbPackage ResponsePackage;
+ std::vector<CbAttachment> Attachments;
+ tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes;
+ Attachments.reserve(BlobCount);
+ AttachmentHashes.reserve(BlobCount);
+ {
+ CbObjectWriter ResponseWriter;
+
+ ResponseWriter.BeginArray("blobHashes");
+ for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+ {
+ if (BlockMetadatas[BlockHashIndex])
+ {
+ const IoHash& BlockHash = BlobRawHashes[BlockHashIndex];
+ ResponseWriter.AddHash(BlockHash);
+ }
+ }
+ ResponseWriter.EndArray(); // blobHashes
+
+ ResponseWriter.BeginArray("metadatas");
+
+ for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+ {
+ if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata)
+ {
+ switch (Metadata.GetContentType())
+ {
+ case ZenContentType::kCbObject:
+ {
+ CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned());
+ const IoHash ObjectHash = Object.GetHash();
+ ResponseWriter.AddBinaryAttachment(ObjectHash);
+ if (!AttachmentHashes.contains(ObjectHash))
+ {
+ Attachments.push_back(CbAttachment(Object, ObjectHash));
+ AttachmentHashes.insert(ObjectHash);
+ }
+ }
+ break;
+ case ZenContentType::kCompressedBinary:
+ {
+ IoHash RawHash;
+ uint64_t _;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _);
+ ResponseWriter.AddBinaryAttachment(RawHash);
+ if (!AttachmentHashes.contains(RawHash))
+ {
+ Attachments.push_back(CbAttachment(Compressed, RawHash));
+ AttachmentHashes.insert(RawHash);
+ }
+ }
+ break;
+ default:
+ {
+ const IoHash RawHash = IoHash::HashBuffer(Metadata);
+ ResponseWriter.AddBinaryAttachment(RawHash);
+ if (!AttachmentHashes.contains(RawHash))
+ {
+ Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash));
+ AttachmentHashes.insert(RawHash);
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ ResponseWriter.EndArray(); // metadatas
+
+ ResponsePackage.SetObject(ResponseWriter.Save());
+ }
+ ResponsePackage.AddAttachments(Attachments);
+
+ CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
+ ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
+}
+
+void
+HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest");
+ HttpServerRequest& ServerRequest = Req.ServerRequest();
+ std::string_view Namespace = Req.GetCapture(1);
+ std::string_view Bucket = Req.GetCapture(2);
+ std::string_view BuildId = Req.GetCapture(3);
+ ZEN_UNUSED(Namespace, Bucket, BuildId);
+ IoBuffer RequestPayload = ServerRequest.ReadPayload();
+ if (!RequestPayload)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Expected compact binary body for blob exists request, body is missing");
+ }
+ if (RequestPayload.GetContentType() != HttpContentType::kCbObject)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType())));
+ }
+ if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default);
+ ValidateError != CbValidateError::None)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError)));
+ }
+ CbObject RequestObject = LoadCompactBinaryObject(RequestPayload);
+ CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView();
+ if (!BlobsArray)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Compact binary body for blob exists request is missing 'blobHashes' array");
+ }
+
+ std::vector<IoHash> BlobRawHashes;
+ BlobRawHashes.reserve(BlobsArray.Num());
+ for (CbFieldView BlockHashView : BlobsArray)
+ {
+ BlobRawHashes.push_back(BlockHashView.AsHash());
+ if (BlobRawHashes.back() == IoHash::Zero)
+ {
+ const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType();
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type));
+ }
+ }
+
+ m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size();
+ std::vector<BuildStore::BlobExistsResult> BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes);
+ CbObjectWriter ResponseWriter(9 * BlobsExists.size());
+ ResponseWriter.BeginArray("blobExists"sv);
+ for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists)
+ {
+ ResponseWriter.AddBool(BlobExists.HasBody);
+ if (BlobExists.HasBody)
+ {
+ m_BuildStoreStats.BlobExistsBodyHitCount++;
+ }
+ }
+ ResponseWriter.EndArray(); // blobExist
+ ResponseWriter.BeginArray("metadataExists"sv);
+ for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists)
+ {
+ ResponseWriter.AddBool(BlobExists.HasBody);
+ if (BlobExists.HasMetadata)
+ {
+ m_BuildStoreStats.BlobExistsMetaHitCount++;
+ }
+ }
+ ResponseWriter.EndArray(); // metadataExists
+ CbObject ResponseObject = ResponseWriter.Save();
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject);
+}
+
+void
+HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::Stats");
+ CbObjectWriter Cbo;
+
+ EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+ Cbo.BeginObject("builds");
+ {
+ Cbo.BeginObject("blobs");
+ {
+ Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount"
+ << m_BuildStoreStats.BlobHitCount;
+ }
+ Cbo.EndObject();
+
+ Cbo.BeginObject("metadata");
+ {
+ Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount"
+ << m_BuildStoreStats.BlobMetaHitCount;
+ }
+ Cbo.EndObject();
+
+ Cbo << "requestcount" << m_BuildStoreStats.RequestCount;
+ Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount;
+ }
+ Cbo.EndObject();
+
+ return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
+HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+ ZEN_TRACE_CPU("HttpBuildStoreService::Status");
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+} // namespace zen
diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h
new file mode 100644
index 000000000..50cb5db12
--- /dev/null
+++ b/src/zenserver/buildstore/httpbuildstore.h
@@ -0,0 +1,68 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/stats.h>
+#include <zenhttp/httpserver.h>
+#include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
+
+#include <filesystem>
+
+namespace zen {
+
+class BuildStore;
+
+class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider
+{
+public:
+ HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store);
+ virtual ~HttpBuildStoreService();
+
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(zen::HttpServerRequest& Request) override;
+
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+
+private:
+ struct BuildStoreStats
+ {
+ std::atomic_uint64_t BlobReadCount{};
+ std::atomic_uint64_t BlobHitCount{};
+ std::atomic_uint64_t BlobWriteCount{};
+ std::atomic_uint64_t BlobMetaReadCount{};
+ std::atomic_uint64_t BlobMetaHitCount{};
+ std::atomic_uint64_t BlobMetaWriteCount{};
+ std::atomic_uint64_t BlobExistsCount{};
+ std::atomic_uint64_t BlobExistsBodyHitCount{};
+ std::atomic_uint64_t BlobExistsMetaHitCount{};
+ std::atomic_uint64_t RequestCount{};
+ std::atomic_uint64_t BadRequestCount{};
+ };
+
+ void Initialize();
+
+ inline LoggerRef Log() { return m_Log; }
+
+ LoggerRef m_Log;
+
+ void PutBlobRequest(HttpRouterRequest& Req);
+ void GetBlobRequest(HttpRouterRequest& Req);
+
+ void PutMetadataRequest(HttpRouterRequest& Req);
+ void GetMetadatasRequest(HttpRouterRequest& Req);
+
+ void BlobsExistsRequest(HttpRouterRequest& Req);
+
+ HttpRequestRouter m_Router;
+
+ HttpStatusService& m_StatusService;
+ HttpStatsService& m_StatsService;
+
+ BuildStore& m_BuildStore;
+ BuildStoreStats m_BuildStoreStats;
+ metrics::OperationTiming m_HttpRequests;
+};
+
+} // namespace zen
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 809092378..52f539dcd 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -377,6 +377,9 @@ ParseConfigFile(const std::filesystem::path& Path,
LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv);
LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig);
+ ////// buildsstore
+ LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv);
+
////// network
LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv);
LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv);
@@ -492,6 +495,9 @@ ParseConfigFile(const std::filesystem::path& Path,
LuaOptions.AddOption("gc.projectstore.duration.seconds"sv,
ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds,
"gc-projectstore-duration-seconds");
+ LuaOptions.AddOption("gc.buildstore.duration.seconds"sv,
+ ServerOptions.GcConfig.BuildStore.MaxDurationSeconds,
+ "gc-buildstore-duration-seconds");
////// security
LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv);
@@ -960,6 +966,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
options.add_option("gc",
"",
+ "gc-buildstore-duration-seconds",
+ "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)",
+ cxxopts::value<int32_t>(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"),
+ "");
+
+ options.add_option("gc",
+ "",
"disk-reserve-size",
"Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.",
cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"),
@@ -1031,6 +1044,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
cxxopts::value<std::vector<std::string>>(BucketConfigs),
"");
+ options.add_option("buildstore",
+ "",
+ "buildstore-enabled",
+ "Whether the builds store is enabled or not.",
+ cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"),
+ "");
+
options.add_option("stats",
"",
"statsd",
@@ -1096,7 +1116,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
if (DataDir.empty())
throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot");
- if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir))
+ if (!IsDir(ServerOptions.BaseSnapshotDir))
throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir));
}
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index c7781aada..a87b6f8b3 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -59,11 +59,17 @@ struct ZenProjectStoreEvictionPolicy
int32_t MaxDurationSeconds = 7 * 24 * 60 * 60;
};
+struct ZenBuildStoreEvictionPolicy
+{
+ int32_t MaxDurationSeconds = 3 * 24 * 60 * 60;
+};
+
struct ZenGcConfig
{
// ZenCasEvictionPolicy Cas;
ZenCacheEvictionPolicy Cache;
ZenProjectStoreEvictionPolicy ProjectStore;
+ ZenBuildStoreEvictionPolicy BuildStore;
int32_t MonitorIntervalSeconds = 30;
int32_t IntervalSeconds = 0;
bool CollectSmallObjects = true;
@@ -130,6 +136,11 @@ struct ZenProjectStoreConfig
bool StoreProjectAttachmentMetaData = false;
};
+struct ZenBuildStoreConfig
+{
+ bool Enabled = false;
+};
+
struct ZenWorkspacesConfig
{
bool Enabled = false;
@@ -145,6 +156,7 @@ struct ZenServerOptions
zen::HttpServerConfig HttpServerConfig;
ZenStructuredCacheConfig StructuredCacheConfig;
ZenProjectStoreConfig ProjectStoreConfig;
+ ZenBuildStoreConfig BuildStoreConfig;
ZenStatsConfig StatsConfig;
ZenWorkspacesConfig WorksSpacesConfig;
std::filesystem::path SystemRootDir; // System root directory (used for machine level config)
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 31d9e1c94..dfa710ae0 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -2,6 +2,7 @@
#include "frontend.h"
+#include <zencore/compactbinarybuilder.h>
#include <zencore/endian.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
@@ -26,7 +27,9 @@ static unsigned char gHtmlZipData[] = {
namespace zen {
////////////////////////////////////////////////////////////////////////////////
-HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory)
+HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService)
+: m_Directory(Directory)
+, m_StatusService(StatusService)
{
std::filesystem::path SelfPath = GetRunningExecutablePath();
@@ -50,7 +53,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
{
break;
}
- if (std::filesystem::is_regular_file(ParentPath / "xmake.lua", ErrorCode))
+ if (IsFile(ParentPath / "xmake.lua", ErrorCode))
{
if (ErrorCode)
{
@@ -59,7 +62,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
std::filesystem::path HtmlDir = ParentPath / "src" / "zenserver" / "frontend" / "html";
- if (std::filesystem::is_directory(HtmlDir, ErrorCode))
+ if (IsDir(HtmlDir, ErrorCode))
{
m_Directory = HtmlDir;
}
@@ -81,10 +84,12 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
{
ZEN_INFO("front-end is NOT AVAILABLE");
}
+ m_StatusService.RegisterHandler("dashboard", *this);
}
HttpFrontendService::~HttpFrontendService()
{
+ m_StatusService.UnregisterHandler("dashboard", *this);
}
const char*
@@ -95,6 +100,14 @@ HttpFrontendService::BaseUri() const
////////////////////////////////////////////////////////////////////////////////
void
+HttpFrontendService::HandleStatusRequest(zen::HttpServerRequest& Request)
+{
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
{
using namespace std::literals;
diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h
index 6eac20620..84ffaac42 100644
--- a/src/zenserver/frontend/frontend.h
+++ b/src/zenserver/frontend/frontend.h
@@ -3,23 +3,26 @@
#pragma once
#include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
#include "zipfs.h"
#include <filesystem>
namespace zen {
-class HttpFrontendService final : public zen::HttpService
+class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider
{
public:
- HttpFrontendService(std::filesystem::path Directory);
+ HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService);
virtual ~HttpFrontendService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(zen::HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
ZipFs m_ZipFs;
std::filesystem::path m_Directory;
+ HttpStatusService& m_StatusService;
};
} // namespace zen
diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js
index 65a3ef39b..f127cb0a3 100644
--- a/src/zenserver/frontend/html/pages/entry.js
+++ b/src/zenserver/frontend/html/pages/entry.js
@@ -59,6 +59,62 @@ export class Page extends ZenPage
}
}
+ _find_iohash_field(container, name)
+ {
+ const found_field = container.find(name);
+ if (found_field != undefined)
+ {
+ var found_value = found_field.as_value();
+ if (found_value instanceof Uint8Array)
+ {
+ var ret = "";
+ for (var x of found_value)
+ ret += x.toString(16).padStart(2, "0");
+ return ret;
+ }
+ }
+ return null;
+ }
+
+ async _build_meta(section, entry)
+ {
+ var tree = {}
+ const cookart = this._find_iohash_field(entry, "CookPackageArtifacts");
+ if (cookart != null)
+ {
+ tree["cook"] = { CookPackageArtifacts: cookart};
+ }
+
+ if (Object.keys(tree).length == 0)
+ return;
+
+ const sub_section = section.add_section("meta");
+
+ for (const cat_name in tree)
+ {
+ const cat_section = sub_section.add_section(cat_name);
+ const table = cat_section.add_widget(
+ Table,
+ ["name", "actions"], Table.Flag_PackRight
+ );
+ Object.entries(tree[cat_name]).forEach(([key, value]) =>
+ {
+ const row = table.add_row(key);
+
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
+ const link = row.get_cell(0).link(
+ "/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
+ );
+
+ const action_tb = new Toolbar(row.get_cell(-1), true);
+ action_tb.left().add("copy-hash").on_click(async (v) => {
+ await navigator.clipboard.writeText(v);
+ }, value);
+ });
+ }
+ }
+
async _build_page()
{
var entry = await this._entry;
@@ -78,8 +134,16 @@ export class Page extends ZenPage
delete tree["$id"];
- const sub_section = section.add_section("deps");
- this._build_deps(sub_section, tree);
+ if (Object.keys(tree).length != 0)
+ {
+ const sub_section = section.add_section("deps");
+ this._build_deps(sub_section, tree);
+ }
+ }
+
+ // meta
+ {
+ this._build_meta(section, entry);
}
// data
@@ -128,7 +192,6 @@ export class Page extends ZenPage
);
link.first_child().attr("download", `${io_hash}_${base_name}`);
- const do_nothing = () => void(0);
const action_tb = new Toolbar(row.get_cell(-1), true);
action_tb.left().add("copy-hash").on_click(async (v) => {
await navigator.clipboard.writeText(v);
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index d5419d342..78ddd39a0 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -406,17 +406,17 @@ main(int argc, char* argv[])
if (!DeleteReason.empty())
{
- if (std::filesystem::exists(ServerOptions.DataDir))
+ if (IsDir(ServerOptions.DataDir))
{
ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason);
DeleteDirectories(ServerOptions.DataDir);
}
}
- if (!std::filesystem::exists(ServerOptions.DataDir))
+ if (!IsDir(ServerOptions.DataDir))
{
ServerOptions.IsFirstRun = true;
- std::filesystem::create_directories(ServerOptions.DataDir);
+ CreateDirectories(ServerOptions.DataDir);
}
if (!ServerOptions.BaseSnapshotDir.empty())
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index e757ef84e..8faf12165 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -219,13 +219,17 @@ private:
StringBuilderBase& Builder;
};
-HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg))
+HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg)
+: m_StatusService(StatusService)
+, m_Cfg(std::move(Cfg))
{
Inititalize();
+ m_StatusService.RegisterHandler("obj", *this);
}
HttpObjectStoreService::~HttpObjectStoreService()
{
+ m_StatusService.UnregisterHandler("obj", *this);
}
const char*
@@ -245,13 +249,21 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
}
void
+HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
HttpObjectStoreService::Inititalize()
{
namespace fs = std::filesystem;
ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory);
const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets";
- if (!fs::exists(BucketsPath))
+ if (!IsDir(BucketsPath))
{
CreateDirectories(BucketsPath);
}
@@ -324,7 +336,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
{
std::lock_guard _(BucketsMutex);
- if (!fs::exists(BucketPath))
+ if (!IsDir(BucketPath))
{
CreateDirectories(BucketPath);
ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName);
@@ -406,7 +418,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath);
FileSystemTraversal Traversal;
- if (std::filesystem::exists(FullPath))
+ if (IsDir(FullPath))
{
std::lock_guard _(BucketsMutex);
Traversal.TraverseFileSystem(FullPath, FileVisitor);
@@ -475,7 +487,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
}
const fs::path FilePath = BucketDir / RelativeBucketPath;
- if (!fs::exists(FilePath))
+ if (!IsFile(FilePath))
{
ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath);
return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
@@ -576,7 +588,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
{
std::lock_guard _(BucketsMutex);
- if (!fs::exists(FileDirectory))
+ if (!IsDir(FileDirectory))
{
CreateDirectories(FileDirectory);
}
diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h
index dae979c4c..44e50e208 100644
--- a/src/zenserver/objectstore/objectstore.h
+++ b/src/zenserver/objectstore/objectstore.h
@@ -3,6 +3,7 @@
#pragma once
#include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
#include <atomic>
#include <filesystem>
#include <mutex>
@@ -23,14 +24,15 @@ struct ObjectStoreConfig
std::vector<BucketConfig> Buckets;
};
-class HttpObjectStoreService final : public zen::HttpService
+class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider
{
public:
- HttpObjectStoreService(ObjectStoreConfig Cfg);
+ HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg);
virtual ~HttpObjectStoreService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(zen::HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
void Inititalize();
@@ -41,6 +43,7 @@ private:
void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path);
void PutObject(zen::HttpRouterRequest& Request);
+ HttpStatusService& m_StatusService;
ObjectStoreConfig m_Cfg;
std::mutex BucketsMutex;
HttpRequestRouter m_Router;
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index fbb9bc344..a6583b722 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -308,7 +308,7 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
- JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId);
+ JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1);
AddStats(FindResult);
GetKnownBlocksResult Result{ConvertResult(FindResult)};
if (Result.ErrorCode)
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index 98e292d91..375e44e59 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -73,7 +73,7 @@ public:
ContainerObject.IterateAttachments([&](CbFieldView FieldView) {
IoHash AttachmentHash = FieldView.AsBinaryAttachment();
std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash);
- if (!std::filesystem::exists(AttachmentPath))
+ if (!IsFile(AttachmentPath))
{
Result.Needs.insert(AttachmentHash);
}
@@ -111,7 +111,7 @@ public:
Stopwatch Timer;
SaveAttachmentResult Result;
std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
- if (!std::filesystem::exists(ChunkPath))
+ if (!IsFile(ChunkPath))
{
try
{
@@ -182,7 +182,7 @@ public:
for (const IoHash& RawHash : BlockHashes)
{
std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
- if (std::filesystem::is_regular_file(ChunkPath))
+ if (IsFile(ChunkPath))
{
ExistingBlockHashes.push_back(RawHash);
}
@@ -203,7 +203,7 @@ public:
Stopwatch Timer;
LoadAttachmentResult Result;
std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
- if (!std::filesystem::is_regular_file(ChunkPath))
+ if (!IsFile(ChunkPath))
{
Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string());
@@ -246,7 +246,7 @@ private:
LoadContainerResult Result;
std::filesystem::path SourcePath = m_OutputPath;
SourcePath.append(Name);
- if (!std::filesystem::is_regular_file(SourcePath))
+ if (!IsFile(SourcePath))
{
Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string());
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 47748dd90..317a419eb 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -8,6 +8,7 @@
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
#include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalidation.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
@@ -234,10 +235,15 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
+HttpProjectService::HttpProjectService(CidStore& Store,
+ ProjectStore* Projects,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr)
: m_Log(logging::Get("project"))
, m_CidStore(Store)
, m_ProjectStore(Projects)
+, m_StatusService(StatusService)
, m_StatsService(StatsService)
, m_AuthMgr(AuthMgr)
{
@@ -245,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
using namespace std::literals;
- m_StatsService.RegisterHandler("prj", *this);
-
m_Router.AddPattern("project", "([[:alnum:]_.]+)");
m_Router.AddPattern("log", "([[:alnum:]_.]+)");
m_Router.AddPattern("op", "([[:digit:]]+?)");
@@ -365,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
"details\\$/{project}/{log}/{chunk}",
[this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); },
HttpVerb::kGet);
+
+ m_StatusService.RegisterHandler("prj", *this);
+ m_StatsService.RegisterHandler("prj", *this);
}
HttpProjectService::~HttpProjectService()
{
m_StatsService.UnregisterHandler("prj", *this);
+ m_StatusService.UnregisterHandler("prj", *this);
}
const char*
@@ -465,6 +473,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
void
+HttpProjectService::HandleStatusRequest(HttpServerRequest& Request)
+{
+ ZEN_TRACE_CPU("HttpProjectService::Status");
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req)
{
ZEN_TRACE_CPU("ProjectService::ProjectList");
@@ -885,10 +902,63 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req)
case HttpVerb::kGet:
{
IoBuffer Value;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr);
+ std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr);
if (Result.first == HttpResponseCode::OK)
{
+ if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary ||
+ AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+ AcceptType == ZenContentType::kCbObject)
+ {
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value));
+ IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer();
+
+ if (DecompressedBuffer)
+ {
+ if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+ AcceptType == ZenContentType::kCbObject)
+ {
+ CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default);
+ if (!!CbErr)
+ {
+ m_ProjectStats.BadRequestCount++;
+ ZEN_DEBUG(
+ "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`",
+ ProjectId,
+ OplogId,
+ Cid,
+ ToString(AcceptType));
+ return HttpReq.WriteResponse(
+ HttpResponseCode::NotAcceptable,
+ HttpContentType::kText,
+ fmt::format("Content format not supported, requested {} format, but could not convert to object",
+ ToString(AcceptType)));
+ }
+
+ m_ProjectStats.ChunkHitCount++;
+ CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject);
+ }
+ else
+ {
+ Value = DecompressedBuffer;
+ Value.SetContentType(ZenContentType::kBinary);
+ }
+ }
+ else
+ {
+ m_ProjectStats.BadRequestCount++;
+ ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`",
+ ProjectId,
+ OplogId,
+ Cid,
+ ToString(AcceptType));
+ return HttpReq.WriteResponse(
+ HttpResponseCode::NotAcceptable,
+ HttpContentType::kText,
+ fmt::format("Content format not supported, requested {} format, but could not decompress stored data",
+ ToString(AcceptType)));
+ }
+ }
m_ProjectStats.ChunkHitCount++;
return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
}
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 8e74c57a5..295defa5c 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -5,6 +5,7 @@
#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
#include <zenstore/cidstore.h>
namespace zen {
@@ -31,16 +32,21 @@ class ProjectStore;
// refs:
//
-class HttpProjectService : public HttpService, public IHttpStatsProvider
+class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr);
+ HttpProjectService(CidStore& Store,
+ ProjectStore* InProjectStore,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr);
~HttpProjectService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct ProjectStats
@@ -89,6 +95,7 @@ private:
CidStore& m_CidStore;
HttpRequestRouter m_Router;
Ref<ProjectStore> m_ProjectStore;
+ HttpStatusService& m_StatusService;
HttpStatsService& m_StatsService;
AuthMgr& m_AuthMgr;
ProjectStats m_ProjectStats;
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 86791e29a..1966eeef9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -58,7 +58,7 @@ namespace {
std::filesystem::path DroppedBucketPath;
do
{
- if (!std::filesystem::exists(Dir))
+ if (!IsDir(Dir))
{
return true;
}
@@ -68,7 +68,7 @@ namespace {
std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), MovedId);
DroppedBucketPath = Dir.parent_path() / DroppedName;
- if (std::filesystem::exists(DroppedBucketPath))
+ if (IsDir(DroppedBucketPath))
{
if (!DeleteDirectories(DroppedBucketPath))
{
@@ -77,7 +77,7 @@ namespace {
Dir);
continue;
}
- if (std::filesystem::exists(DroppedBucketPath))
+ if (IsDir(DroppedBucketPath))
{
ZEN_INFO("Drop directory '{}' for '{}' still exists after remove, attempting different name.", DroppedBucketPath, Dir);
continue;
@@ -88,13 +88,13 @@ namespace {
do
{
std::error_code Ec;
- std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+ RenameDirectory(Dir, DroppedBucketPath, Ec);
if (!Ec)
{
OutDeleteDir = DroppedBucketPath;
return true;
}
- if (std::filesystem::exists(DroppedBucketPath))
+ if (IsDir(DroppedBucketPath))
{
ZEN_INFO("Can't rename '{}' to still existing drop directory '{}'. Reason: '{}'. Attempting different name.",
Dir,
@@ -486,7 +486,7 @@ struct ProjectStore::OplogStorage : public RefCounted
[[nodiscard]] bool Exists() const { return Exists(m_OplogStoragePath); }
[[nodiscard]] static bool Exists(const std::filesystem::path& BasePath)
{
- return std::filesystem::exists(GetLogPath(BasePath)) && std::filesystem::exists(GetBlobsPath(BasePath));
+ return IsFile(GetLogPath(BasePath)) && IsFile(GetBlobsPath(BasePath));
}
[[nodiscard]] bool IsValid() const { return IsValid(m_OplogStoragePath); }
[[nodiscard]] static bool IsValid(const std::filesystem::path& BasePath)
@@ -496,13 +496,13 @@ struct ProjectStore::OplogStorage : public RefCounted
void WipeState() const
{
std::error_code Ec;
- std::filesystem::remove(GetLogPath(), Ec);
- std::filesystem::remove(GetBlobsPath(), Ec);
+ RemoveFile(GetLogPath(), Ec);
+ RemoveFile(GetBlobsPath(), Ec);
}
static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); }
- uint64_t OpBlobsSize() const { return std::filesystem::file_size(GetBlobsPath()); }
+ uint64_t OpBlobsSize() const { return FileSizeFromPath(GetBlobsPath()); }
uint64_t OpsSize() const { return OpsSize(m_OplogStoragePath); }
static uint64_t OpsSize(const std::filesystem::path& BasePath)
@@ -510,7 +510,7 @@ struct ProjectStore::OplogStorage : public RefCounted
if (Exists(BasePath))
{
std::error_code DummyEc;
- return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath));
+ return FileSizeFromPath(GetLogPath(BasePath)) + FileSizeFromPath(GetBlobsPath(BasePath));
}
return 0;
}
@@ -689,7 +689,7 @@ struct ProjectStore::OplogStorage : public RefCounted
m_OpBlobs.Close();
Oplog.Close();
- std::filesystem::rename(OplogPath, GetLogPath(), Ec);
+ RenameFile(OplogPath, GetLogPath(), Ec);
if (Ec)
{
throw std::system_error(
@@ -702,9 +702,9 @@ struct ProjectStore::OplogStorage : public RefCounted
if (Ec)
{
// We failed late - clean everything up as best we can
- std::filesystem::remove(OpBlobs.GetPath(), Ec);
- std::filesystem::remove(GetLogPath(), Ec);
- std::filesystem::remove(GetBlobsPath(), Ec);
+ RemoveFile(OpBlobs.GetPath(), Ec);
+ RemoveFile(GetLogPath(), Ec);
+ RemoveFile(GetBlobsPath(), Ec);
throw std::system_error(Ec,
fmt::format("Oplog::Compact failed to rename temporary oplog file from '{}' to '{}'",
OpBlobs.GetPath(),
@@ -739,7 +739,7 @@ struct ProjectStore::OplogStorage : public RefCounted
}
catch (const std::exception& /*Ex*/)
{
- std::filesystem::remove(OpBlobs.GetPath(), Ec);
+ RemoveFile(OpBlobs.GetPath(), Ec);
throw;
}
}
@@ -1108,7 +1108,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id,
ZEN_WARN("Invalid oplog found at '{}'. Wiping state for oplog.", m_BasePath);
m_Storage->WipeState();
std::error_code DummyEc;
- std::filesystem::remove(m_MetaPath, DummyEc);
+ RemoveFile(m_MetaPath, DummyEc);
}
}
m_Storage->Open(/* IsCreate */ !StoreExists);
@@ -1116,7 +1116,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id,
m_MetaPath = m_BasePath / "ops.meta"sv;
m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath());
- CleanDirectory(m_TempPath);
+ CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false);
}
ProjectStore::Oplog::~Oplog()
@@ -1142,7 +1142,7 @@ ProjectStore::Oplog::Flush()
if (!m_MetaValid)
{
std::error_code DummyEc;
- std::filesystem::remove(m_MetaPath, DummyEc);
+ RemoveFile(m_MetaPath, DummyEc);
}
uint64_t LogCount = m_Storage->LogCount();
@@ -1238,19 +1238,19 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath)
uint64_t Size = OplogStorage::OpsSize(BasePath);
std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
- if (std::filesystem::exists(StateFilePath))
+ if (IsFile(StateFilePath))
{
- Size += std::filesystem::file_size(StateFilePath);
+ Size += FileSizeFromPath(StateFilePath);
}
std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv;
- if (std::filesystem::exists(MetaFilePath))
+ if (IsFile(MetaFilePath))
{
- Size += std::filesystem::file_size(MetaFilePath);
+ Size += FileSizeFromPath(MetaFilePath);
}
std::filesystem::path IndexFilePath = BasePath / "ops.zidx"sv;
- if (std::filesystem::exists(IndexFilePath))
+ if (IsFile(IndexFilePath))
{
- Size += std::filesystem::file_size(IndexFilePath);
+ Size += FileSizeFromPath(IndexFilePath);
}
return Size;
@@ -1303,7 +1303,7 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath)
using namespace std::literals;
std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
- return std::filesystem::is_regular_file(StateFilePath);
+ return IsFile(StateFilePath);
}
bool
@@ -1337,7 +1337,7 @@ ProjectStore::Oplog::Read()
if (!m_MetaValid)
{
std::error_code DummyEc;
- std::filesystem::remove(m_MetaPath, DummyEc);
+ RemoveFile(m_MetaPath, DummyEc);
}
ReadIndexSnapshot();
@@ -1438,7 +1438,7 @@ ProjectStore::Oplog::Reset()
m_Storage = new OplogStorage(this, m_BasePath);
m_Storage->Open(true);
m_MetaValid = false;
- CleanDirectory(m_TempPath);
+ CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false);
Write();
}
// Erase content on disk
@@ -1457,7 +1457,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f
using namespace std::literals;
std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
- if (std::filesystem::is_regular_file(StateFilePath))
+ if (IsFile(StateFilePath))
{
// ZEN_INFO("oplog '{}/{}': config read from '{}'", m_OuterProject->Identifier, m_OplogId, StateFilePath);
@@ -1536,7 +1536,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo
if (File.Hash == IoHash::Zero)
{
std::filesystem::path FilePath = m_OuterProject->RootDir / File.ServerPath;
- if (!std::filesystem::is_regular_file(FilePath))
+ if (!IsFile(FilePath))
{
ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); });
HasMissingEntries = true;
@@ -1625,10 +1625,10 @@ ProjectStore::Oplog::WriteIndexSnapshot()
fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp";
// Move index away, we keep it if something goes wrong
- if (fs::is_regular_file(TempIndexPath))
+ if (IsFile(TempIndexPath))
{
std::error_code Ec;
- if (!fs::remove(TempIndexPath, Ec) || Ec)
+ if (!RemoveFile(TempIndexPath, Ec) || Ec)
{
ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'",
GetOuterProject()->Identifier,
@@ -1641,9 +1641,9 @@ ProjectStore::Oplog::WriteIndexSnapshot()
try
{
- if (fs::is_regular_file(IndexPath))
+ if (IsFile(IndexPath))
{
- fs::rename(IndexPath, TempIndexPath);
+ RenameFile(IndexPath, TempIndexPath);
}
// Write the current state of the location map to a new index state
@@ -1778,11 +1778,11 @@ ProjectStore::Oplog::WriteIndexSnapshot()
// Restore any previous snapshot
- if (fs::is_regular_file(TempIndexPath))
+ if (IsFile(TempIndexPath))
{
std::error_code Ec;
- fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless
- fs::rename(TempIndexPath, IndexPath, Ec);
+ RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless
+ RenameFile(TempIndexPath, IndexPath, Ec);
if (Ec)
{
ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'",
@@ -1793,10 +1793,10 @@ ProjectStore::Oplog::WriteIndexSnapshot()
}
}
}
- if (fs::is_regular_file(TempIndexPath))
+ if (IsFile(TempIndexPath))
{
std::error_code Ec;
- if (!fs::remove(TempIndexPath, Ec) || Ec)
+ if (!RemoveFile(TempIndexPath, Ec) || Ec)
{
ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'",
m_OuterProject->Identifier,
@@ -1814,7 +1814,7 @@ ProjectStore::Oplog::ReadIndexSnapshot()
ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot");
std::filesystem::path IndexPath = m_BasePath / "ops.zidx";
- if (std::filesystem::is_regular_file(IndexPath))
+ if (IsFile(IndexPath))
{
uint64_t EntryCount = 0;
Stopwatch Timer;
@@ -3133,7 +3133,7 @@ ProjectStore::Project::~Project()
bool
ProjectStore::Project::Exists(const std::filesystem::path& BasePath)
{
- return std::filesystem::exists(BasePath / "Project.zcb");
+ return IsFile(BasePath / "Project.zcb");
}
void
@@ -3207,7 +3207,7 @@ ProjectStore::Project::ReadAccessTimes()
using namespace std::literals;
std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv;
- if (!std::filesystem::exists(ProjectAccessTimesFilePath))
+ if (!IsFile(ProjectAccessTimesFilePath))
{
return;
}
@@ -3598,14 +3598,14 @@ ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath)
uint64_t Size = 0;
std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv;
- if (std::filesystem::exists(AccessTimesFilePath))
+ if (IsFile(AccessTimesFilePath))
{
- Size += std::filesystem::file_size(AccessTimesFilePath);
+ Size += FileSizeFromPath(AccessTimesFilePath);
}
std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv;
- if (std::filesystem::exists(ProjectFilePath))
+ if (IsFile(ProjectFilePath))
{
- Size += std::filesystem::file_size(ProjectFilePath);
+ Size += FileSizeFromPath(ProjectFilePath);
}
return Size;
@@ -3717,7 +3717,7 @@ ProjectStore::Project::IsExpired(const std::string& EntryName,
if (!MarkerPath.empty())
{
std::error_code Ec;
- if (std::filesystem::exists(MarkerPath, Ec))
+ if (IsFile(MarkerPath, Ec))
{
if (Ec)
{
@@ -3870,7 +3870,7 @@ void
ProjectStore::DiscoverProjects()
{
ZEN_MEMSCOPE(GetProjectstoreTag());
- if (!std::filesystem::exists(m_ProjectBasePath))
+ if (!IsDir(m_ProjectBasePath))
{
return;
}
@@ -3979,7 +3979,7 @@ ProjectStore::StorageSize() const
GcStorageSize Result;
{
- if (std::filesystem::exists(m_ProjectBasePath))
+ if (IsDir(m_ProjectBasePath))
{
DirectoryContent ProjectsFolderContent;
GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent);
@@ -3987,7 +3987,7 @@ ProjectStore::StorageSize() const
for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories)
{
std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv;
- if (std::filesystem::exists(ProjectStateFilePath))
+ if (IsFile(ProjectStateFilePath))
{
Result.DiskSize += Project::TotalSize(ProjectBasePath);
DirectoryContent DirContent;
@@ -4770,7 +4770,6 @@ std::pair<HttpResponseCode, std::string>
ProjectStore::GetChunk(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view Cid,
- ZenContentType AcceptType,
IoBuffer& OutChunk,
uint64_t* OptionalInOutModificationTag)
{
@@ -4812,16 +4811,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
}
}
- if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary)
- {
- CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk));
- OutChunk = Compressed.Decompress().AsIoBuffer();
- OutChunk.SetContentType(ZenContentType::kBinary);
- }
- else
- {
- OutChunk.SetContentType(ZenContentType::kCompressedBinary);
- }
+ OutChunk.SetContentType(ZenContentType::kCompressedBinary);
return {HttpResponseCode::OK, {}};
}
@@ -7253,7 +7243,7 @@ TEST_CASE("project.store.gc")
CHECK(ProjectStore.OpenProject("proj2"sv));
}
- std::filesystem::remove(Project1FilePath);
+ RemoveFile(Project1FilePath);
{
GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24),
@@ -7282,7 +7272,7 @@ TEST_CASE("project.store.gc")
CHECK(ProjectStore.OpenProject("proj2"sv));
}
- std::filesystem::remove(Project2Oplog1Path);
+ RemoveFile(Project2Oplog1Path);
{
GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24),
.ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
@@ -7311,7 +7301,7 @@ TEST_CASE("project.store.gc")
CHECK(ProjectStore.OpenProject("proj2"sv));
}
- std::filesystem::remove(Project2FilePath);
+ RemoveFile(Project2FilePath);
{
GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24),
.ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24),
@@ -8035,7 +8025,7 @@ TEST_CASE("project.store.rpc.getchunks")
CompositeBuffer Buffer = Attachment->AsCompositeBinary();
CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)),
IoHash::HashBuffer(Buffer));
- CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second));
+ CHECK_EQ(Chunk["Size"sv].AsUInt64(), FileSizeFromPath(FilesOpIdAttachments[0].second));
CHECK(!Chunk.FindView("RawSize"));
}
{
@@ -8516,12 +8506,7 @@ TEST_CASE("project.store.partial.read")
uint64_t ModificationTag = 0;
IoBuffer Chunk;
CHECK(ProjectStore
- .GetChunk("proj1"sv,
- "oplog1"sv,
- Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
- HttpContentType::kCompressedBinary,
- Chunk,
- &ModificationTag)
+ .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
.first == HttpResponseCode::OK);
IoHash RawHash;
uint64_t RawSize;
@@ -8530,12 +8515,7 @@ TEST_CASE("project.store.partial.read")
CHECK(ModificationTag != 0);
CHECK(ProjectStore
- .GetChunk("proj1"sv,
- "oplog1"sv,
- Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
- HttpContentType::kCompressedBinary,
- Chunk,
- &ModificationTag)
+ .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
.first == HttpResponseCode::NotModified);
}
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 8f2d3ce0d..368da5ea4 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -449,7 +449,6 @@ public:
std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view Cid,
- ZenContentType AcceptType,
IoBuffer& OutChunk,
uint64_t* OptionalInOutModificationTag);
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index a7263da83..f96b3e185 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -1212,7 +1212,7 @@ BuildContainer(CidStore& ChunkStore,
{
std::string_view ServerPath = View["serverpath"sv].AsString();
std::filesystem::path FilePath = Project.RootDir / ServerPath;
- if (!std::filesystem::is_regular_file(FilePath))
+ if (!IsFile(FilePath))
{
remotestore_impl::ReportMessage(
OptionalContext,
@@ -3083,9 +3083,9 @@ LoadOplog(CidStore& ChunkStore,
OptionalContext]() {
auto _ = MakeGuard([&DechunkLatch, &TempFileName] {
std::error_code Ec;
- if (std::filesystem::exists(TempFileName, Ec))
+ if (IsFile(TempFileName, Ec))
{
- std::filesystem::remove(TempFileName, Ec);
+ RemoveFile(TempFileName, Ec);
if (Ec)
{
ZEN_INFO("Failed to remove temporary file '{}'. Reason: {}", TempFileName, Ec.message());
diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp
index d302a10ec..bf761f8d1 100644
--- a/src/zenserver/vfs/vfsservice.cpp
+++ b/src/zenserver/vfs/vfsservice.cpp
@@ -61,7 +61,7 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb)
// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @-
// echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @-
-VfsService::VfsService()
+VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
{
m_Impl = new Impl;
@@ -136,10 +136,12 @@ VfsService::VfsService()
}
},
HttpVerb::kPost);
+ m_StatusService.RegisterHandler("vfs", *this);
}
VfsService::~VfsService()
{
+ m_StatusService.UnregisterHandler("vfs", *this);
delete m_Impl;
}
@@ -169,8 +171,9 @@ VfsService::AddService(Ref<ZenCacheStore>&& Z$)
#else
-VfsService::VfsService()
+VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
{
+ ZEN_UNUSED(StatusService);
}
VfsService::~VfsService()
@@ -209,6 +212,14 @@ VfsService::BaseUri() const
}
void
+VfsService::HandleStatusRequest(HttpServerRequest& Request)
+{
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest)
{
m_Router.HandleRequest(HttpServiceRequest);
diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h
index dcdc71e81..0d0168e23 100644
--- a/src/zenserver/vfs/vfsservice.h
+++ b/src/zenserver/vfs/vfsservice.h
@@ -4,6 +4,7 @@
#include <zenbase/refcount.h>
#include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
#include <zenvfs/vfs.h>
#include <memory>
@@ -24,10 +25,10 @@ class ZenCacheStore;
*/
-class VfsService : public HttpService
+class VfsService : public HttpService, public IHttpStatusProvider
{
public:
- VfsService();
+ explicit VfsService(HttpStatusService& StatusService);
~VfsService();
void Mount(std::string_view MountPoint);
@@ -39,12 +40,14 @@ public:
protected:
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct Impl;
Impl* m_Impl = nullptr;
- HttpRequestRouter m_Router;
+ HttpStatusService& m_StatusService;
+ HttpRequestRouter m_Router;
friend struct VfsServiceDataSource;
};
diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp
index 8a4b977ad..7ef84743e 100644
--- a/src/zenserver/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/workspaces/httpworkspaces.cpp
@@ -73,8 +73,12 @@ namespace {
} // namespace
-HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces)
+HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ const WorkspacesServeConfig& Cfg,
+ Workspaces& Workspaces)
: m_Log(logging::Get("workspaces"))
+, m_StatusService(StatusService)
, m_StatsService(StatsService)
, m_Config(Cfg)
, m_Workspaces(Workspaces)
@@ -84,7 +88,8 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con
HttpWorkspacesService::~HttpWorkspacesService()
{
- m_StatsService.UnregisterHandler("prj", *this);
+ m_StatsService.UnregisterHandler("ws", *this);
+ m_StatusService.UnregisterHandler("ws", *this);
}
const char*
@@ -149,14 +154,21 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
void
+HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request)
+{
+ ZEN_TRACE_CPU("HttpWorkspacesService::Status");
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
HttpWorkspacesService::Initialize()
{
using namespace std::literals;
ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service");
- m_StatsService.RegisterHandler("ws", *this);
-
m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})");
m_Router.AddPattern("share_id", "([[:xdigit:]]{24})");
m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
@@ -238,6 +250,9 @@ HttpWorkspacesService::Initialize()
HttpVerb::kGet);
RefreshState();
+
+ m_StatsService.RegisterHandler("ws", *this);
+ m_StatusService.RegisterHandler("ws", *this);
}
std::filesystem::path
@@ -1100,7 +1115,7 @@ HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& Workspace
}
}
- if (!std::filesystem::is_directory(Workspace.RootPath / NewConfig.SharePath))
+ if (!IsDir(Workspace.RootPath / NewConfig.SharePath))
{
return ServerRequest.WriteResponse(HttpResponseCode::NotFound,
HttpContentType::kText,
diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h
index f01f58b86..89a8e8bdc 100644
--- a/src/zenserver/workspaces/httpworkspaces.h
+++ b/src/zenserver/workspaces/httpworkspaces.h
@@ -5,6 +5,7 @@
#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
namespace zen {
@@ -16,16 +17,20 @@ struct WorkspacesServeConfig
bool AllowConfigurationChanges = false;
};
-class HttpWorkspacesService final : public HttpService, public IHttpStatsProvider
+class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces);
+ HttpWorkspacesService(HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ const WorkspacesServeConfig& Cfg,
+ Workspaces& Workspaces);
virtual ~HttpWorkspacesService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct WorkspacesStats
@@ -80,6 +85,7 @@ private:
void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId);
void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId);
+ HttpStatusService& m_StatusService;
HttpStatsService& m_StatsService;
const WorkspacesServeConfig m_Config;
HttpRequestRouter m_Router;
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index f84bc0b00..c7cb2ba6e 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -23,6 +23,7 @@
#include <zencore/trace.h>
#include <zencore/workthreadpool.h>
#include <zenhttp/httpserver.h>
+#include <zenstore/buildstore/buildstore.h>
#include <zenstore/cidstore.h>
#include <zenstore/scrubcontext.h>
#include <zenstore/workspaces.h>
@@ -250,18 +251,26 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
*m_JobQueue,
*m_OpenProcessCache,
ProjectStore::Configuration{});
- m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr});
+ m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr});
if (ServerOptions.WorksSpacesConfig.Enabled)
{
m_Workspaces.reset(new Workspaces());
m_HttpWorkspacesService.reset(
- new HttpWorkspacesService(m_StatsService,
+ new HttpWorkspacesService(m_StatusService,
+ m_StatsService,
{.SystemRootDir = ServerOptions.SystemRootDir,
.AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges},
*m_Workspaces));
}
+ if (ServerOptions.BuildStoreConfig.Enabled)
+ {
+ BuildStoreConfig ObjCfg;
+ ObjCfg.RootDirectory = m_DataRoot / "builds";
+ m_BuildStore = std::make_unique<BuildStore>(std::move(ObjCfg), m_GcManager);
+ }
+
if (ServerOptions.StructuredCacheConfig.Enabled)
{
InitializeStructuredCache(ServerOptions);
@@ -287,7 +296,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
m_Http->RegisterService(*m_HttpWorkspacesService);
}
- m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot);
+ m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
if (m_FrontendService)
{
@@ -306,12 +315,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
ObjCfg.Buckets.push_back(std::move(NewBucket));
}
- m_ObjStoreService = std::make_unique<HttpObjectStoreService>(std::move(ObjCfg));
+ m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg));
m_Http->RegisterService(*m_ObjStoreService);
}
+ if (ServerOptions.BuildStoreConfig.Enabled)
+ {
+ m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore);
+ m_Http->RegisterService(*m_BuildStoreService);
+ }
+
#if ZEN_WITH_VFS
- m_VfsService = std::make_unique<VfsService>();
+ m_VfsService = std::make_unique<VfsService>(m_StatusService);
m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore));
m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore));
m_Http->RegisterService(*m_VfsService);
@@ -327,6 +342,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
.Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds),
.MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds),
.MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds),
+ .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds),
.CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects,
.Enabled = ServerOptions.GcConfig.Enabled,
.DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize,
@@ -347,6 +363,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
m_CacheStore.Get(),
m_CidStore.get(),
m_ProjectStore,
+ m_BuildStore.get(),
HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile,
.HttpLogPath = ServerOptions.DataDir / "logs" / "http.log",
.CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
@@ -418,7 +435,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION)
{
std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch";
- if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath))
+ if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath))
{
ZEN_INFO(
"Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating "
@@ -467,7 +484,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
{
ZEN_INFO("Deleting '{}'", DirEntry.path());
- std::filesystem::remove_all(DirEntry.path(), Ec);
+ DeleteDirectories(DirEntry.path(), Ec);
if (Ec)
{
@@ -801,6 +818,9 @@ ZenServer::Cleanup()
m_ObjStoreService.reset();
m_FrontendService.reset();
+ m_BuildStoreService.reset();
+ m_BuildStore = {};
+
m_StructuredCacheService.reset();
m_UpstreamService.reset();
m_UpstreamCache.reset();
@@ -895,7 +915,7 @@ ZenServer::CheckStateMarker()
std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker";
try
{
- if (!std::filesystem::exists(StateMarkerPath))
+ if (!IsFile(StateMarkerPath))
{
ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath);
RequestExit(1);
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 80054dc35..5cfa04ba1 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -25,6 +25,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
#include <zenstore/cache/structuredcachestore.h>
#include <zenstore/gc.h>
#include "admin/admin.h"
+#include "buildstore/httpbuildstore.h"
#include "cache/httpstructuredcache.h"
#include "diag/diagsvcs.h"
#include "frontend/frontend.h"
@@ -127,6 +128,8 @@ private:
Ref<ZenCacheStore> m_CacheStore;
std::unique_ptr<OpenProcessCache> m_OpenProcessCache;
HttpTestService m_TestService;
+ std::unique_ptr<BuildStore> m_BuildStore;
+
#if ZEN_WITH_TESTS
HttpTestingService m_TestingService;
#endif
@@ -140,6 +143,7 @@ private:
HttpHealthService m_HealthService;
std::unique_ptr<HttpFrontendService> m_FrontendService;
std::unique_ptr<HttpObjectStoreService> m_ObjStoreService;
+ std::unique_ptr<HttpBuildStoreService> m_BuildStoreService;
std::unique_ptr<VfsService> m_VfsService;
std::unique_ptr<JobQueue> m_JobQueue;
std::unique_ptr<HttpAdminService> m_AdminService;