aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/storage
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/storage')
-rw-r--r--src/zenserver/storage/admin/admin.cpp10
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.cpp153
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.h7
-rw-r--r--src/zenserver/storage/cache/httpstructuredcache.cpp141
-rw-r--r--src/zenserver/storage/cache/httpstructuredcache.h11
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.cpp382
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.h25
-rw-r--r--src/zenserver/storage/storageconfig.cpp26
-rw-r--r--src/zenserver/storage/storageconfig.h5
-rw-r--r--src/zenserver/storage/upstream/zen.cpp2
-rw-r--r--src/zenserver/storage/workspaces/httpworkspaces.cpp12
-rw-r--r--src/zenserver/storage/workspaces/httpworkspaces.h5
-rw-r--r--src/zenserver/storage/zenstorageserver.cpp54
-rw-r--r--src/zenserver/storage/zenstorageserver.h8
14 files changed, 670 insertions, 171 deletions
diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp
index 19155e02b..6e78a6179 100644
--- a/src/zenserver/storage/admin/admin.cpp
+++ b/src/zenserver/storage/admin/admin.cpp
@@ -184,6 +184,10 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
State.TotalCount > 0
? gsl::narrow<uint32_t>((100 * (State.TotalCount - State.RemainingCount)) / State.TotalCount)
: 0);
+ if (State.ProgressElapsedTimeMs != (uint64_t)-1)
+ {
+ Obj.AddInteger("ProgressElapsedTimeMs"sv, gsl::narrow<uint64_t>(State.ProgressElapsedTimeMs));
+ }
}
if (!State.Messages.empty())
{
@@ -716,7 +720,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
"logs",
[this](HttpRouterRequest& Req) {
CbObjectWriter Obj;
- auto LogLevel = logging::level::ToStringView(logging::GetLogLevel());
+ auto LogLevel = logging::ToStringView(logging::GetLogLevel());
Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size()));
Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath));
Obj.BeginObject("cache");
@@ -767,8 +771,8 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
}
if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false)
{
- logging::level::LogLevel NewLevel = logging::level::ParseLogLevelString(Param);
- std::string_view LogLevel = logging::level::ToStringView(NewLevel);
+ logging::LogLevel NewLevel = logging::ParseLogLevelString(Param);
+ std::string_view LogLevel = logging::ToStringView(NewLevel);
if (LogLevel != Param)
{
return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index f5ba30616..de9589078 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -71,7 +71,7 @@ HttpBuildStoreService::Initialize()
m_Router.RegisterRoute(
"{namespace}/{bucket}/{buildid}/blobs/{hash}",
[this](HttpRouterRequest& Req) { GetBlobRequest(Req); },
- HttpVerb::kGet);
+ HttpVerb::kGet | HttpVerb::kPost);
m_Router.RegisterRoute(
"{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata",
@@ -161,14 +161,57 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
HttpContentType::kText,
fmt::format("Invalid blob hash '{}'", Hash));
}
- zen::HttpRanges Ranges;
- bool HasRange = ServerRequest.TryGetRanges(Ranges);
- if (Ranges.size() > 1)
+
+ std::vector<std::pair<uint64_t, uint64_t>> OffsetAndLengthPairs;
+ if (ServerRequest.RequestVerb() == HttpVerb::kPost)
{
- // Only a single range is supported
- return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
- HttpContentType::kText,
- "Multiple ranges in blob request is not supported");
+ CbObject RangePayload = ServerRequest.ReadPayloadObject();
+ if (RangePayload)
+ {
+ CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView();
+ OffsetAndLengthPairs.reserve(RangesArray.Num());
+ for (CbFieldView FieldView : RangesArray)
+ {
+ CbObjectView RangeView = FieldView.AsObjectView();
+ uint64_t RangeOffset = RangeView["offset"sv].AsUInt64();
+ uint64_t RangeLength = RangeView["length"sv].AsUInt64();
+ OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength));
+ }
+ if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}",
+ OffsetAndLengthPairs.size(),
+ MaxRangeCountPerRequestSupported));
+ }
+ }
+ if (OffsetAndLengthPairs.empty())
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Fetching blob without ranges must be done with the GET verb");
+ }
+ }
+ else
+ {
+ HttpRanges Ranges;
+ bool HasRange = ServerRequest.TryGetRanges(Ranges);
+ if (HasRange)
+ {
+ if (Ranges.size() > 1)
+ {
+ // Only a single http range is supported, we have limited support for http multirange responses
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Multiple ranges in blob request is only supported for {} accept type",
+ ToString(HttpContentType::kCbPackage)));
+ }
+ const HttpRange& FirstRange = Ranges.front();
+ OffsetAndLengthPairs.push_back(std::make_pair<uint64_t, uint64_t>(FirstRange.Start, FirstRange.End - FirstRange.Start + 1));
+ }
}
m_BuildStoreStats.BlobReadCount++;
@@ -179,24 +222,79 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
HttpContentType::kText,
fmt::format("Blob with hash '{}' could not be found", Hash));
}
- // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize());
m_BuildStoreStats.BlobHitCount++;
- if (HasRange)
+
+ if (OffsetAndLengthPairs.empty())
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+ }
+
+ if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage)
{
- const HttpRange& Range = Ranges.front();
- const uint64_t BlobSize = Blob.GetSize();
- const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0;
- const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize);
- if (Range.Start + RangeSize > BlobSize)
+ const uint64_t BlobSize = Blob.GetSize();
+
+ CbPackage ResponsePackage;
+ std::vector<IoBuffer> RangeBuffers;
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ for (const std::pair<uint64_t, uint64_t>& Range : OffsetAndLengthPairs)
{
- return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+ const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0;
+ const uint64_t RangeSize = Min(Range.second, MaxBlobSize);
+ Writer.BeginObject();
+ {
+ if (Range.first + RangeSize <= BlobSize)
+ {
+ RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize));
+ Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("length"sv, RangeSize);
+ }
+ else
+ {
+ Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("length"sv, 0);
+ }
+ }
+ Writer.EndObject();
}
- Blob = IoBuffer(Blob, Range.Start, RangeSize);
- return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
+ Writer.EndArray();
+
+ CompositeBuffer Ranges(RangeBuffers);
+ CbAttachment PayloadAttachment(std::move(Ranges), BlobHash);
+ Writer.AddAttachment("payload", PayloadAttachment);
+
+ CbObject HeaderObject = Writer.Save();
+
+ ResponsePackage.AddAttachment(PayloadAttachment);
+ ResponsePackage.SetObject(HeaderObject);
+
+ CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
+ uint64_t ResponseSize = RpcResponseBuffer.GetSize();
+ ZEN_UNUSED(ResponseSize);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
}
else
{
- return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+ if (OffsetAndLengthPairs.size() != 1)
+ {
+ // Only a single http range is supported, we have limited support for http multirange responses
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage)));
+ }
+
+ const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front();
+ const uint64_t BlobSize = Blob.GetSize();
+ const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0;
+ const uint64_t RangeSize = Min(OffsetAndLength.second, MaxBlobSize);
+ if (OffsetAndLength.first + RangeSize > BlobSize)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+ }
+ Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
}
}
@@ -507,8 +605,8 @@ HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req)
return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject);
}
-void
-HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
+CbObject
+HttpBuildStoreService::CollectStats()
{
ZEN_TRACE_CPU("HttpBuildStoreService::Stats");
@@ -562,7 +660,13 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
}
Cbo.EndObject();
- return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ return Cbo.Save();
+}
+
+void
+HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+ Request.WriteResponse(HttpResponseCode::OK, CollectStats());
}
void
@@ -571,6 +675,11 @@ HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
ZEN_TRACE_CPU("HttpBuildStoreService::Status");
CbObjectWriter Cbo;
Cbo << "ok" << true;
+ Cbo.BeginObject("capabilities");
+ {
+ Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported;
+ }
+ Cbo.EndObject(); // capabilities
Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
}
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h
index e10986411..2a09b71cf 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.h
+++ b/src/zenserver/storage/buildstore/httpbuildstore.h
@@ -22,8 +22,9 @@ public:
virtual const char* BaseUri() const override;
virtual void HandleRequest(zen::HttpServerRequest& Request) override;
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct BuildStoreStats
@@ -45,6 +46,8 @@ private:
inline LoggerRef Log() { return m_Log; }
+ static constexpr uint32_t MaxRangeCountPerRequestSupported = 256u;
+
LoggerRef m_Log;
void PutBlobRequest(HttpRouterRequest& Req);
diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp
index 72f29d14e..bbdb03ba4 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.cpp
+++ b/src/zenserver/storage/cache/httpstructuredcache.cpp
@@ -654,7 +654,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
auto NewEnd = std::unique(AllAttachments.begin(), AllAttachments.end());
AllAttachments.erase(NewEnd, AllAttachments.end());
- uint64_t AttachmentsSize = 0;
+ std::atomic<uint64_t> AttachmentsSize = 0;
m_CidStore.IterateChunks(
AllAttachments,
@@ -746,7 +746,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
ResponseWriter << "Size" << ValuesSize;
ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size();
- uint64_t AttachmentsSize = 0;
+ std::atomic<uint64_t> AttachmentsSize = 0;
WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background);
@@ -1827,8 +1827,8 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::st
}
}
-void
-HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
+CbObject
+HttpStructuredCacheService::CollectStats()
{
ZEN_MEMSCOPE(GetCacheHttpTag());
@@ -1858,13 +1858,130 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
const CidStoreSize CidSize = m_CidStore.TotalSize();
const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
+ Cbo.BeginObject("cache");
+ {
+ Cbo << "badrequestcount" << BadRequestCount;
+ Cbo.BeginObject("rpc");
+ Cbo << "count" << RpcRequests;
+ Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests;
+ Cbo.BeginObject("records");
+ Cbo << "count" << RpcRecordRequests;
+ Cbo << "ops" << RpcRecordBatchRequests;
+ Cbo.EndObject();
+ Cbo.BeginObject("values");
+ Cbo << "count" << RpcValueRequests;
+ Cbo << "ops" << RpcValueBatchRequests;
+ Cbo.EndObject();
+ Cbo.BeginObject("chunks");
+ Cbo << "count" << RpcChunkRequests;
+ Cbo << "ops" << RpcChunkBatchRequests;
+ Cbo.EndObject();
+ Cbo.EndObject();
+
+ Cbo.BeginObject("size");
+ {
+ Cbo << "disk" << CacheSize.DiskSize;
+ Cbo << "memory" << CacheSize.MemorySize;
+ }
+ Cbo.EndObject();
+
+ Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
+ Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
+
+ if (m_UpstreamCache.IsActive())
+ {
+ Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+ Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+ }
+
+ Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+
+ {
+ ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats();
+ Cbo.BeginObject("store");
+ Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount
+ << "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount;
+ const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount;
+ Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0);
+ EmitSnapshot("read", StoreStatsData.GetOps, Cbo);
+ EmitSnapshot("write", StoreStatsData.PutOps, Cbo);
+ Cbo.EndObject();
+ }
+ }
+ Cbo.EndObject();
+
+ if (m_UpstreamCache.IsActive())
+ {
+ EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+ Cbo.BeginObject("upstream");
+ {
+ m_UpstreamCache.GetStatus(Cbo);
+ }
+ Cbo.EndObject();
+ }
+
+ Cbo.BeginObject("cid");
+ {
+ Cbo.BeginObject("size");
+ {
+ Cbo << "tiny" << CidSize.TinySize;
+ Cbo << "small" << CidSize.SmallSize;
+ Cbo << "large" << CidSize.LargeSize;
+ Cbo << "total" << CidSize.TotalSize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+
+ return Cbo.Save();
+}
+
+void
+HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
+{
+ ZEN_MEMSCOPE(GetCacheHttpTag());
+
bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true";
bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
- CidStoreStats CidStoreStats = {};
+ if (!ShowCidStoreStats && !ShowCacheStoreStats)
+ {
+ Request.WriteResponse(HttpResponseCode::OK, CollectStats());
+ return;
+ }
+
+ // Full stats with optional detailed store/cid breakdowns
+
+ CbObjectWriter Cbo;
+
+ EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+ const uint64_t HitCount = m_CacheStats.HitCount;
+ const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+ const uint64_t MissCount = m_CacheStats.MissCount;
+ const uint64_t WriteCount = m_CacheStats.WriteCount;
+ const uint64_t BadRequestCount = m_CacheStats.BadRequestCount;
+ struct CidStoreStats StoreStats = m_CidStore.Stats();
+ const uint64_t ChunkHitCount = StoreStats.HitCount;
+ const uint64_t ChunkMissCount = StoreStats.MissCount;
+ const uint64_t ChunkWriteCount = StoreStats.WriteCount;
+ const uint64_t TotalCount = HitCount + MissCount;
+
+ const uint64_t RpcRequests = m_CacheStats.RpcRequests;
+ const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests;
+ const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
+ const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests;
+ const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests;
+ const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests;
+ const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests;
+
+ const CidStoreSize CidSize = m_CidStore.TotalSize();
+ const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
+
+ CidStoreStats DetailedCidStoreStats = {};
if (ShowCidStoreStats)
{
- CidStoreStats = m_CidStore.Stats();
+ DetailedCidStoreStats = m_CidStore.Stats();
}
ZenCacheStore::CacheStoreStats CacheStoreStats = {};
if (ShowCacheStoreStats)
@@ -1906,8 +2023,6 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
{
Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
- Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
- Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
}
Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
@@ -2002,8 +2117,8 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
}
Cbo.EndObject();
}
- Cbo.EndObject();
}
+ Cbo.EndObject();
if (m_UpstreamCache.IsActive())
{
@@ -2029,10 +2144,10 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
if (ShowCidStoreStats)
{
Cbo.BeginObject("store");
- Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount;
- EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo);
- EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo);
- // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo);
+ Cbo << "hits" << DetailedCidStoreStats.HitCount << "misses" << DetailedCidStoreStats.MissCount << "writes"
+ << DetailedCidStoreStats.WriteCount;
+ EmitSnapshot("read", DetailedCidStoreStats.FindChunkOps, Cbo);
+ EmitSnapshot("write", DetailedCidStoreStats.AddChunkOps, Cbo);
Cbo.EndObject();
}
}
diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h
index 5a795c215..d462415d4 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.h
+++ b/src/zenserver/storage/cache/httpstructuredcache.h
@@ -102,11 +102,12 @@ private:
void HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace);
void HandleDetailsRequest(HttpServerRequest& Request);
- void HandleCacheRequest(HttpServerRequest& Request);
- void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace);
- void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket);
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ void HandleCacheRequest(HttpServerRequest& Request);
+ void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace);
+ void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket);
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
bool AreDiskWritesAllowed() const;
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 416e2ed69..2fa10a292 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -13,7 +13,12 @@
#include <zencore/scopeguard.h>
#include <zencore/stream.h>
#include <zencore/trace.h>
+#include <zenhttp/httpclientauth.h>
#include <zenhttp/packageformat.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/jupiter/jupiterhost.h>
+#include <zenremotestore/operationlogoutput.h>
#include <zenremotestore/projectstore/buildsremoteprojectstore.h>
#include <zenremotestore/projectstore/fileremoteprojectstore.h>
#include <zenremotestore/projectstore/jupiterremoteprojectstore.h>
@@ -23,6 +28,7 @@
#include <zenstore/oplogreferencedset.h>
#include <zenstore/projectstore.h>
#include <zenstore/zenstore.h>
+#include <zenutil/authutils.h>
#include <zenutil/openprocesscache.h>
#include <zenutil/workerpools.h>
@@ -101,7 +107,6 @@ CSVWriteOp(CidStore& CidStore,
//////////////////////////////////////////////////////////////////////////
namespace {
-
void CbWriteOp(CidStore& CidStore,
bool Details,
bool OpDetails,
@@ -244,6 +249,22 @@ namespace {
{
std::shared_ptr<RemoteProjectStore> Store;
std::string Description;
+ double LatencySec = -1.0;
+ uint64_t MaxRangeCountPerRequest = 1;
+
+ struct Cache
+ {
+ std::unique_ptr<HttpClient> Http;
+ std::unique_ptr<BuildStorageCache> Cache;
+ Oid BuildsId = Oid::Zero;
+ std::string Description;
+ double LatencySec = -1.0;
+ uint64_t MaxRangeCountPerRequest = 1;
+ BuildStorageCache::Statistics Stats;
+ bool Populate = false;
+ };
+
+ std::unique_ptr<Cache> OptionalCache;
};
CreateRemoteStoreResult CreateRemoteStore(LoggerRef InLog,
@@ -252,7 +273,9 @@ namespace {
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
size_t MaximumInMemoryDownloadSize,
- const std::filesystem::path& TempFilePath)
+ const std::filesystem::path& TempFilePath,
+ const std::filesystem::path& OidcTokenExePath,
+ bool AllowExternalOidcTokenExe)
{
ZEN_MEMSCOPE(GetProjectHttpTag());
@@ -260,7 +283,7 @@ namespace {
using namespace std::literals;
- std::shared_ptr<RemoteProjectStore> RemoteStore;
+ CreateRemoteStoreResult Result;
if (CbObjectView File = Params["file"sv].AsObjectView(); File)
{
@@ -285,7 +308,9 @@ namespace {
std::string(OptionalBaseName),
ForceDisableBlocks,
ForceEnableTempBlocks};
- RemoteStore = CreateFileRemoteStore(Log(), Options);
+ Result.Store = CreateFileRemoteStore(Log(), Options);
+ Result.LatencySec = 0.5 / 1000.0; // 0.5 ms
+ Result.MaxRangeCountPerRequest = 1024u;
}
if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -317,19 +342,24 @@ namespace {
AccessToken = GetEnvVariable(AccessTokenEnvVariable);
}
}
- std::filesystem::path OidcExePath;
- if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+
+ std::filesystem::path OidcExePath = FindOidcTokenExePath(OidcTokenExePath.string());
+ if (OidcExePath.empty() && AllowExternalOidcTokenExe)
{
- std::filesystem::path OidcExePathMaybe(OidcExePathString);
- if (IsFile(OidcExePathMaybe))
- {
- OidcExePath = std::move(OidcExePathMaybe);
- }
- else
+ if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty())
{
- ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ std::filesystem::path OidcExePathMaybe(OidcExePathString);
+ if (IsFile(OidcExePathMaybe))
+ {
+ OidcExePath = std::move(OidcExePathMaybe);
+ }
+ else
+ {
+ ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ }
}
}
+
std::string_view KeyParam = Cloud["key"sv].AsString();
if (KeyParam.empty())
{
@@ -363,21 +393,32 @@ namespace {
bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false);
- JupiterRemoteStoreOptions Options = {
- RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
- Url,
- std::string(Namespace),
- std::string(Bucket),
- Key,
- BaseKey,
- std::string(OpenIdProvider),
- AccessToken,
- AuthManager,
- OidcExePath,
- ForceDisableBlocks,
- ForceDisableTempBlocks,
- AssumeHttp2};
- RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+ if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(Url, AssumeHttp2, /*Verbose*/ false); TestResult.Success)
+ {
+ Result.LatencySec = TestResult.LatencySeconds;
+ Result.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+
+ JupiterRemoteStoreOptions Options = {
+ RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+ Url,
+ std::string(Namespace),
+ std::string(Bucket),
+ Key,
+ BaseKey,
+ std::string(OpenIdProvider),
+ AccessToken,
+ AuthManager,
+ OidcExePath,
+ ForceDisableBlocks,
+ ForceDisableTempBlocks,
+ AssumeHttp2};
+ Result.Store =
+ CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+ }
+ else
+ {
+ return {nullptr, fmt::format("Unable to connect to jupiter host '{}'", Url)};
+ }
}
if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -393,12 +434,13 @@ namespace {
{
return {nullptr, "Missing oplog"};
}
+
ZenRemoteStoreOptions Options = {
RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
std::string(Url),
std::string(Project),
std::string(Oplog)};
- RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath);
+ Result.Store = CreateZenRemoteStore(Log(), Options, TempFilePath);
}
if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
@@ -435,19 +477,24 @@ namespace {
AccessToken = GetEnvVariable(AccessTokenEnvVariable);
}
}
- std::filesystem::path OidcExePath;
- if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+
+ std::filesystem::path OidcExePath = FindOidcTokenExePath(OidcTokenExePath.string());
+ if (OidcExePath.empty() && AllowExternalOidcTokenExe)
{
- std::filesystem::path OidcExePathMaybe(OidcExePathString);
- if (IsFile(OidcExePathMaybe))
+ if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty())
{
- OidcExePath = std::move(OidcExePathMaybe);
- }
- else
- {
- ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ std::filesystem::path OidcExePathMaybe(OidcExePathString);
+ if (IsFile(OidcExePathMaybe))
+ {
+ OidcExePath = std::move(OidcExePathMaybe);
+ }
+ else
+ {
+ ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ }
}
}
+
std::string_view BuildIdParam = Builds["buildsid"sv].AsString();
if (BuildIdParam.empty())
{
@@ -471,11 +518,76 @@ namespace {
MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
+ auto EnsureHttps = [](const std::string& Host, std::string_view PreferredProtocol) {
+ if (!Host.empty() && Host.find("://"sv) == std::string::npos)
+ {
+ // Assume https URL
+ return fmt::format("{}://{}"sv, PreferredProtocol, Host);
+ }
+ return Host;
+ };
+
+ Host = EnsureHttps(Host, "https");
+ OverrideHost = EnsureHttps(OverrideHost, "https");
+ ZenHost = EnsureHttps(ZenHost, "http");
+
+ std::function<HttpClientAccessToken()> TokenProvider;
+ if (!OpenIdProvider.empty())
+ {
+ TokenProvider = httpclientauth::CreateFromOpenIdProvider(AuthManager, OpenIdProvider);
+ }
+ else if (!AccessToken.empty())
+ {
+ TokenProvider = httpclientauth::CreateFromStaticToken(AccessToken);
+ }
+ else if (!OidcExePath.empty())
+ {
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(OidcExePath,
+ Host.empty() ? OverrideHost : Host,
+ /*Quiet*/ false,
+ /*Unattended*/ false,
+ /*Hidden*/ true);
+ TokenProviderMaybe)
+ {
+ TokenProvider = TokenProviderMaybe.value();
+ }
+ }
+
+ if (!TokenProvider)
+ {
+ TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(AuthManager);
+ }
+
+ BuildStorageResolveResult ResolveResult;
+ {
+ HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
+ .AccessTokenProvider = TokenProvider,
+ .AssumeHttp2 = AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 2};
+
+ std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(Log()));
+
+ try
+ {
+ ResolveResult = ResolveBuildStorage(*Output,
+ ClientSettings,
+ Host,
+ OverrideHost,
+ ZenHost,
+ ZenCacheResolveMode::Discovery,
+ /*Verbose*/ false);
+ }
+ catch (const std::exception& Ex)
+ {
+ return {nullptr, fmt::format("Failed resolving storage host and cache. Reason: '{}'", Ex.what())};
+ }
+ }
+ Result.LatencySec = ResolveResult.Cloud.LatencySec;
+ Result.MaxRangeCountPerRequest = ResolveResult.Cloud.Caps.MaxRangeCountPerRequest;
+
BuildsRemoteStoreOptions Options = {
RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
- Host,
- OverrideHost,
- ZenHost,
std::string(Namespace),
std::string(Bucket),
BuildId,
@@ -485,25 +597,43 @@ namespace {
OidcExePath,
ForceDisableBlocks,
ForceDisableTempBlocks,
- AssumeHttp2,
- PopulateCache,
MetaData,
MaximumInMemoryDownloadSize};
- RemoteStore = CreateJupiterBuildsRemoteStore(Log(),
- Options,
- TempFilePath,
- /*Quiet*/ false,
- /*Unattended*/ false,
- /*Hidden*/ true,
- GetTinyWorkerPool(EWorkloadType::Background));
+ Result.Store = CreateJupiterBuildsRemoteStore(Log(), ResolveResult, std::move(TokenProvider), Options, TempFilePath);
+
+ if (!ResolveResult.Cache.Address.empty())
+ {
+ Result.OptionalCache = std::make_unique<CreateRemoteStoreResult::Cache>();
+
+ HttpClientSettings CacheClientSettings{.LogCategory = "httpcacheclient",
+ .ConnectTimeout = std::chrono::milliseconds{3000},
+ .Timeout = std::chrono::milliseconds{30000},
+ .AssumeHttp2 = ResolveResult.Cache.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 0,
+ .MaximumInMemoryDownloadSize = MaximumInMemoryDownloadSize};
+
+ Result.OptionalCache->Http = std::make_unique<HttpClient>(ResolveResult.Cache.Address, CacheClientSettings);
+ Result.OptionalCache->Cache = CreateZenBuildStorageCache(*Result.OptionalCache->Http,
+ Result.OptionalCache->Stats,
+ Namespace,
+ Bucket,
+ TempFilePath,
+ GetTinyWorkerPool(EWorkloadType::Background));
+ Result.OptionalCache->BuildsId = BuildId;
+ Result.OptionalCache->LatencySec = ResolveResult.Cache.LatencySec;
+ Result.OptionalCache->MaxRangeCountPerRequest = ResolveResult.Cache.Caps.MaxRangeCountPerRequest;
+ Result.OptionalCache->Populate = PopulateCache;
+ Result.OptionalCache->Description =
+ fmt::format("[zenserver] {} namespace {} bucket {}", ResolveResult.Cache.Address, Namespace, Bucket);
+ }
}
-
- if (!RemoteStore)
+ if (!Result.Store)
{
return {nullptr, "Unknown remote store type"};
}
- return {std::move(RemoteStore), ""};
+ return Result;
}
std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result)
@@ -529,13 +659,16 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store,
- ProjectStore* Projects,
- HttpStatusService& StatusService,
- HttpStatsService& StatsService,
- AuthMgr& AuthMgr,
- OpenProcessCache& InOpenProcessCache,
- JobQueue& InJobQueue)
+HttpProjectService::HttpProjectService(CidStore& Store,
+ ProjectStore* Projects,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr,
+ OpenProcessCache& InOpenProcessCache,
+ JobQueue& InJobQueue,
+ bool InRestrictContentTypes,
+ const std::filesystem::path& InOidcTokenExePath,
+ bool InAllowExternalOidcTokenExe)
: m_Log(logging::Get("project"))
, m_CidStore(Store)
, m_ProjectStore(Projects)
@@ -544,6 +677,9 @@ HttpProjectService::HttpProjectService(CidStore& Store,
, m_AuthMgr(AuthMgr)
, m_OpenProcessCache(InOpenProcessCache)
, m_JobQueue(InJobQueue)
+, m_RestrictContentTypes(InRestrictContentTypes)
+, m_OidcTokenExePath(InOidcTokenExePath)
+, m_AllowExternalOidcTokenExe(InAllowExternalOidcTokenExe)
{
ZEN_MEMSCOPE(GetProjectHttpTag());
@@ -712,8 +848,8 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
}
}
-void
-HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+CbObject
+HttpProjectService::CollectStats()
{
ZEN_TRACE_CPU("ProjectService::Stats");
@@ -779,7 +915,13 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
Cbo.EndObject();
- return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ return Cbo.Save();
+}
+
+void
+HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+ HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
}
void
@@ -1866,6 +2008,14 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
{
return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
}
+
+ if (m_RestrictContentTypes && (HttpReq.RequestContentType() == HttpContentType::kText ||
+ HttpReq.RequestContentType() == HttpContentType::kUnknownContentType))
+ {
+ m_ProjectStats.BadRequestCount++;
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ }
+
std::filesystem::path OplogMarkerPath;
if (CbObject Params = HttpReq.ReadPayloadObject())
{
@@ -2156,6 +2306,13 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
}
+ if (m_RestrictContentTypes && (HttpReq.RequestContentType() == HttpContentType::kText ||
+ HttpReq.RequestContentType() == HttpContentType::kUnknownContentType))
+ {
+ m_ProjectStats.BadRequestCount++;
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ }
+
CbValidateError ValidateResult;
if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult);
ValidateResult == CbValidateError::None)
@@ -2356,15 +2513,19 @@ HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req)
tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); };
- auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
+ auto OnNeedBlock = [&AttachmentsLock, &Attachments](ThinChunkBlockDescription&& ThinBlockDescription,
+ std::vector<uint32_t>&& NeededChunkIndexes) {
RwLock::ExclusiveLockScope _(AttachmentsLock);
- if (BlockHash != IoHash::Zero)
+ if (ThinBlockDescription.BlockHash != IoHash::Zero)
{
- Attachments.insert(BlockHash);
+ Attachments.insert(ThinBlockDescription.BlockHash);
}
else
{
- Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
+ for (uint32_t ChunkIndex : NeededChunkIndexes)
+ {
+ Attachments.insert(ThinBlockDescription.ChunkRawHashes[ChunkIndex]);
+ }
}
};
auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
@@ -2567,10 +2728,17 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
CbObject Cb;
switch (PayloadContentType)
{
- case HttpContentType::kJSON:
- case HttpContentType::kUnknownContentType:
case HttpContentType::kText:
+ case HttpContentType::kUnknownContentType:
+ case HttpContentType::kJSON:
{
+ if (m_RestrictContentTypes &&
+ (PayloadContentType == HttpContentType::kText || PayloadContentType == HttpContentType::kUnknownContentType))
+ {
+ m_ProjectStats.BadRequestCount++;
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ }
+
std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize());
Cb = LoadCompactBinaryFromJson(JsonText).AsObject();
if (!Cb)
@@ -2663,36 +2831,40 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
bool CleanOplog = Params["clean"].AsBool(false);
bool BoostWorkerCount = Params["boostworkercount"].AsBool(false);
bool BoostWorkerMemory = Params["boostworkermemory"sv].AsBool(false);
-
- CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(),
- Params,
- m_AuthMgr,
- MaxBlockSize,
- MaxChunkEmbedSize,
- GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
- Oplog->TempPath());
-
- if (RemoteStoreResult.Store == nullptr)
+ EPartialBlockRequestMode PartialBlockRequestMode =
+ PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true"));
+
+ std::shared_ptr<CreateRemoteStoreResult> RemoteStoreResult =
+ std::make_shared<CreateRemoteStoreResult>(CreateRemoteStore(Log(),
+ Params,
+ m_AuthMgr,
+ MaxBlockSize,
+ MaxChunkEmbedSize,
+ GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
+ Oplog->TempPath(),
+ m_OidcTokenExePath,
+ m_AllowExternalOidcTokenExe));
+ if (RemoteStoreResult->Store == nullptr)
{
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult->Description);
}
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
- RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
JobId JobId = m_JobQueue.QueueJob(
fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()),
[this,
- ChunkStore = &m_CidStore,
- ActualRemoteStore = std::move(RemoteStore),
+ RemoteStoreResult = std::move(RemoteStoreResult),
Oplog,
Force,
IgnoreMissingAttachments,
CleanOplog,
+ PartialBlockRequestMode,
BoostWorkerCount](JobContext& Context) {
- Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}",
- Oplog->GetOuterProjectIdentifier(),
- Oplog->OplogId(),
- ActualRemoteStore->GetInfo().Description));
+ Context.ReportMessage(
+ fmt::format("Loading oplog '{}/{}'\n Host: {}\n Cache: {}",
+ Oplog->GetOuterProjectIdentifier(),
+ Oplog->OplogId(),
+ RemoteStoreResult->Store->GetInfo().Description,
+ RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Description : "<none>"));
Ref<TransferThreadWorkers> Workers = GetThreadWorkers(BoostWorkerCount, /*SingleThreaded*/ false);
@@ -2700,16 +2872,26 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
WorkerThreadPool& NetworkWorkerPool = Workers->GetNetworkPool();
Context.ReportMessage(fmt::format("{}", Workers->GetWorkersInfo()));
-
- RemoteProjectStore::Result Result = LoadOplog(m_CidStore,
- *ActualRemoteStore,
- *Oplog,
- NetworkWorkerPool,
- WorkerPool,
- Force,
- IgnoreMissingAttachments,
- CleanOplog,
- &Context);
+ RemoteProjectStore::Result Result = LoadOplog(LoadOplogContext{
+ .ChunkStore = m_CidStore,
+ .RemoteStore = *RemoteStoreResult->Store,
+ .OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr,
+ .CacheBuildId = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->BuildsId : Oid::Zero,
+ .OptionalCacheStats = RemoteStoreResult->OptionalCache ? &RemoteStoreResult->OptionalCache->Stats : nullptr,
+ .Oplog = *Oplog,
+ .NetworkWorkerPool = NetworkWorkerPool,
+ .WorkerPool = WorkerPool,
+ .ForceDownload = Force,
+ .IgnoreMissingAttachments = IgnoreMissingAttachments,
+ .CleanOplog = CleanOplog,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .PopulateCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Populate : false,
+ .StoreLatencySec = RemoteStoreResult->LatencySec,
+ .StoreMaxRangeCountPerRequest = RemoteStoreResult->MaxRangeCountPerRequest,
+ .CacheLatencySec = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->LatencySec : -1.0,
+ .CacheMaxRangeCountPerRequest =
+ RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->MaxRangeCountPerRequest : 0,
+ .OptionalJobContext = &Context});
auto Response = ConvertResult(Result);
ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
@@ -2740,7 +2922,9 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
MaxBlockSize,
MaxChunkEmbedSize,
GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
- Oplog->TempPath());
+ Oplog->TempPath(),
+ m_OidcTokenExePath,
+ m_AllowExternalOidcTokenExe);
if (RemoteStoreResult.Store == nullptr)
{
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h
index b742102a5..917337324 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.h
+++ b/src/zenserver/storage/projectstore/httpprojectstore.h
@@ -38,20 +38,24 @@ class TransferThreadWorkers;
class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(CidStore& Store,
- ProjectStore* InProjectStore,
- HttpStatusService& StatusService,
- HttpStatsService& StatsService,
- AuthMgr& AuthMgr,
- OpenProcessCache& InOpenProcessCache,
- JobQueue& InJobQueue);
+ HttpProjectService(CidStore& Store,
+ ProjectStore* InProjectStore,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr,
+ OpenProcessCache& InOpenProcessCache,
+ JobQueue& InJobQueue,
+ bool InRestrictContentTypes,
+ const std::filesystem::path& InOidcTokenExePath,
+ bool AllowExternalOidcTokenExe);
~HttpProjectService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct ProjectStats
@@ -109,6 +113,9 @@ private:
metrics::OperationTiming m_HttpRequests;
RwLock m_ThreadWorkersLock;
Ref<TransferThreadWorkers> m_ThreadWorkers;
+ bool m_RestrictContentTypes;
+ std::filesystem::path m_OidcTokenExePath;
+ bool m_AllowExternalOidcTokenExe;
Ref<TransferThreadWorkers> GetThreadWorkers(bool BoostWorkers, bool SingleThreaded);
};
diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp
index 089b6b572..e8ccb9097 100644
--- a/src/zenserver/storage/storageconfig.cpp
+++ b/src/zenserver/storage/storageconfig.cpp
@@ -496,6 +496,11 @@ ZenStorageServerConfigurator::AddConfigOptions(LuaConfig::Options& LuaOptions)
LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv);
LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv);
LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig);
+ LuaOptions.AddOption("security.restrictcontenttypes"sv, ServerOptions.RestrictContentTypes, "restrict-content-types"sv);
+ LuaOptions.AddOption("security.oidctokenexecutable"sv, ServerOptions.OidcTokenExecutable, "oidctoken-exe-path"sv);
+ LuaOptions.AddOption("security.allowexternaloidctokenexecutable"sv,
+ ServerOptions.AllowExternalOidcTokenExe,
+ "allow-external-oidctoken-exe"sv);
////// workspaces
LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv);
@@ -649,6 +654,24 @@ ZenStorageServerCmdLineOptions::AddSecurityOptions(cxxopts::Options& options, Ze
options.add_option("security", "", "openid-provider-url", "Open ID provider URL", cxxopts::value<std::string>(OpenIdProviderUrl), "");
options.add_option("security", "", "openid-client-id", "Open ID client ID", cxxopts::value<std::string>(OpenIdClientId), "");
+ options.add_option("security",
+ "",
+ "restrict-content-types",
+ "Restrict content-type in requests to content-types that are not allowed in CORS simple requests",
+ cxxopts::value<bool>(ServerOptions.RestrictContentTypes),
+ "");
+ options.add_option("security",
+ "",
+ "oidctoken-exe-path",
+ "Path to OidcToken executable",
+ cxxopts::value<std::string>(OidcTokenExecutable),
+ "");
+ options.add_option("security",
+ "",
+ "allow-external-oidctoken-exe",
+ "Allow requests to specify a path to an external OidcToken executable",
+ cxxopts::value<bool>(ServerOptions.AllowExternalOidcTokenExe),
+ "");
}
void
@@ -1046,7 +1069,8 @@ ZenStorageServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenStora
{.Name = OpenIdProviderName, .Url = OpenIdProviderUrl, .ClientId = OpenIdClientId});
}
- ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs);
+ ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs);
+ ServerOptions.OidcTokenExecutable = MakeSafeAbsolutePath(OidcTokenExecutable);
}
} // namespace zen
diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h
index b408b0c26..128804d92 100644
--- a/src/zenserver/storage/storageconfig.h
+++ b/src/zenserver/storage/storageconfig.h
@@ -1,4 +1,5 @@
// Copyright Epic Games, Inc. All Rights Reserved.
+#pragma once
#include "config/config.h"
@@ -158,6 +159,9 @@ struct ZenStorageServerConfig : public ZenServerConfig
bool ObjectStoreEnabled = false;
bool ComputeEnabled = true;
std::string ScrubOptions;
+ bool RestrictContentTypes = false;
+ std::filesystem::path OidcTokenExecutable;
+ bool AllowExternalOidcTokenExe = true;
};
struct ZenStorageServerCmdLineOptions
@@ -175,6 +179,7 @@ struct ZenStorageServerCmdLineOptions
std::string OpenIdProviderName;
std::string OpenIdProviderUrl;
std::string OpenIdClientId;
+ std::string OidcTokenExecutable;
void AddSecurityOptions(cxxopts::Options& options, ZenStorageServerConfig& ServerOptions);
diff --git a/src/zenserver/storage/upstream/zen.cpp b/src/zenserver/storage/upstream/zen.cpp
index 423c9039c..a3807675c 100644
--- a/src/zenserver/storage/upstream/zen.cpp
+++ b/src/zenserver/storage/upstream/zen.cpp
@@ -63,7 +63,7 @@ ZenStructuredCacheSession::CheckHealth()
return {.Bytes = Response.DownloadedBytes,
.ElapsedSeconds = Response.ElapsedSeconds,
- .Success = Response.StatusCode == HttpResponseCode::OK};
+ .Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::NoContent};
}
ZenCacheResult
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp
index dc4cc7e69..785dd62f0 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp
@@ -110,8 +110,8 @@ HttpWorkspacesService::HandleRequest(HttpServerRequest& Request)
}
}
-void
-HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
+CbObject
+HttpWorkspacesService::CollectStats()
{
ZEN_TRACE_CPU("WorkspacesService::Stats");
CbObjectWriter Cbo;
@@ -150,7 +150,13 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
Cbo.EndObject();
- return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ return Cbo.Save();
+}
+
+void
+HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+ HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
}
void
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h
index 888a34b4d..7c5ddeff1 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.h
+++ b/src/zenserver/storage/workspaces/httpworkspaces.h
@@ -29,8 +29,9 @@ public:
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct WorkspacesStats
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index ff854b72d..77588bd6c 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -33,6 +33,7 @@
#include <zenutil/service.h>
#include <zenutil/workerpools.h>
#include <zenutil/zenserverprocess.h>
+#include "../sessions/sessions.h"
#if ZEN_PLATFORM_WINDOWS
# include <zencore/windows.h>
@@ -133,7 +134,6 @@ void
ZenStorageServer::RegisterServices()
{
m_Http->RegisterService(*m_AuthService);
- m_Http->RegisterService(m_StatsService);
m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics
#if ZEN_WITH_TESTS
@@ -160,6 +160,11 @@ ZenStorageServer::RegisterServices()
m_Http->RegisterService(*m_HttpWorkspacesService);
}
+ if (m_HttpSessionsService)
+ {
+ m_Http->RegisterService(*m_HttpSessionsService);
+ }
+
m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
if (m_FrontendService)
@@ -183,10 +188,15 @@ ZenStorageServer::RegisterServices()
m_Http->RegisterService(*m_AdminService);
+ if (m_ApiService)
+ {
+ m_Http->RegisterService(*m_ApiService);
+ }
+
#if ZEN_WITH_COMPUTE_SERVICES
- if (m_HttpFunctionService)
+ if (m_HttpComputeService)
{
- m_Http->RegisterService(*m_HttpFunctionService);
+ m_Http->RegisterService(*m_HttpComputeService);
}
#endif
}
@@ -212,8 +222,16 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
m_OpenProcessCache = std::make_unique<OpenProcessCache>();
m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, ProjectStore::Configuration{});
- m_HttpProjectService.reset(
- new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr, *m_OpenProcessCache, *m_JobQueue});
+ m_HttpProjectService.reset(new HttpProjectService{*m_CidStore,
+ m_ProjectStore,
+ m_StatusService,
+ m_StatsService,
+ *m_AuthMgr,
+ *m_OpenProcessCache,
+ *m_JobQueue,
+ ServerOptions.RestrictContentTypes,
+ ServerOptions.OidcTokenExecutable,
+ ServerOptions.AllowExternalOidcTokenExe});
if (ServerOptions.WorksSpacesConfig.Enabled)
{
@@ -228,6 +246,11 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
*m_Workspaces));
}
+ {
+ m_SessionsService = std::make_unique<SessionsService>();
+ m_HttpSessionsService = std::make_unique<HttpSessionsService>(m_StatusService, m_StatsService, *m_SessionsService);
+ }
+
if (ServerOptions.BuildStoreConfig.Enabled)
{
CidStoreConfiguration BuildCidConfig;
@@ -279,8 +302,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
{
ZEN_OTEL_SPAN("InitializeComputeService");
- m_HttpFunctionService =
- std::make_unique<compute::HttpFunctionService>(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions");
+ m_HttpComputeService =
+ std::make_unique<compute::HttpComputeService>(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions");
}
#endif
@@ -316,6 +339,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
.AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount};
m_GcScheduler.Initialize(GcConfig);
+ m_ApiService = std::make_unique<HttpApiService>(*m_Http);
+
// Create and register admin interface last to make sure all is properly initialized
m_AdminService = std::make_unique<HttpAdminService>(
m_GcScheduler,
@@ -700,6 +725,15 @@ ZenStorageServer::Run()
ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId());
+ if (m_FrontendService && !m_NoNetwork)
+ {
+ ZEN_INFO("frontend link: {}", m_Http->GetServiceUri(m_FrontendService.get()));
+ }
+ else if (!m_FrontendService)
+ {
+ ZEN_INFO("frontend service disabled");
+ }
+
#if ZEN_PLATFORM_WINDOWS
if (zen::windows::IsRunningOnWine())
{
@@ -807,6 +841,8 @@ ZenStorageServer::Cleanup()
m_IoRunner.join();
}
+ ShutdownServices();
+
if (m_Http)
{
m_Http->Close();
@@ -823,7 +859,7 @@ ZenStorageServer::Cleanup()
Flush();
#if ZEN_WITH_COMPUTE_SERVICES
- m_HttpFunctionService.reset();
+ m_HttpComputeService.reset();
#endif
m_AdminService.reset();
@@ -841,6 +877,8 @@ ZenStorageServer::Cleanup()
m_UpstreamCache.reset();
m_CacheStore = {};
+ m_HttpSessionsService.reset();
+ m_SessionsService.reset();
m_HttpWorkspacesService.reset();
m_Workspaces.reset();
m_HttpProjectService.reset();
diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h
index 456447a2a..d625f869c 100644
--- a/src/zenserver/storage/zenstorageserver.h
+++ b/src/zenserver/storage/zenstorageserver.h
@@ -12,6 +12,7 @@
#include <zenstore/gc.h>
#include <zenstore/projectstore.h>
+#include "../sessions/httpsessions.h"
#include "admin/admin.h"
#include "buildstore/httpbuildstore.h"
#include "cache/httpstructuredcache.h"
@@ -25,7 +26,7 @@
#include "workspaces/httpworkspaces.h"
#if ZEN_WITH_COMPUTE_SERVICES
-# include <zencompute/httpfunctionservice.h>
+# include <zencompute/httpcomputeservice.h>
#endif
namespace zen {
@@ -62,7 +63,6 @@ private:
void InitializeServices(const ZenStorageServerConfig& ServerOptions);
void RegisterServices();
- HttpStatsService m_StatsService;
std::unique_ptr<JobQueue> m_JobQueue;
GcManager m_GcManager;
GcScheduler m_GcScheduler{m_GcManager};
@@ -82,6 +82,8 @@ private:
std::unique_ptr<HttpProjectService> m_HttpProjectService;
std::unique_ptr<Workspaces> m_Workspaces;
std::unique_ptr<HttpWorkspacesService> m_HttpWorkspacesService;
+ std::unique_ptr<SessionsService> m_SessionsService;
+ std::unique_ptr<HttpSessionsService> m_HttpSessionsService;
std::unique_ptr<UpstreamCache> m_UpstreamCache;
std::unique_ptr<HttpUpstreamService> m_UpstreamService;
std::unique_ptr<HttpStructuredCacheService> m_StructuredCacheService;
@@ -93,7 +95,7 @@ private:
std::unique_ptr<HttpApiService> m_ApiService;
#if ZEN_WITH_COMPUTE_SERVICES
- std::unique_ptr<compute::HttpFunctionService> m_HttpFunctionService;
+ std::unique_ptr<compute::HttpComputeService> m_HttpComputeService;
#endif
};
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-23 16:01:23 +0000