aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/storage/projectstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/storage/projectstore')
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.cpp382
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.h25
2 files changed, 299 insertions, 108 deletions
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 416e2ed69..2fa10a292 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -13,7 +13,12 @@
#include <zencore/scopeguard.h>
#include <zencore/stream.h>
#include <zencore/trace.h>
+#include <zenhttp/httpclientauth.h>
#include <zenhttp/packageformat.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/jupiter/jupiterhost.h>
+#include <zenremotestore/operationlogoutput.h>
#include <zenremotestore/projectstore/buildsremoteprojectstore.h>
#include <zenremotestore/projectstore/fileremoteprojectstore.h>
#include <zenremotestore/projectstore/jupiterremoteprojectstore.h>
@@ -23,6 +28,7 @@
#include <zenstore/oplogreferencedset.h>
#include <zenstore/projectstore.h>
#include <zenstore/zenstore.h>
+#include <zenutil/authutils.h>
#include <zenutil/openprocesscache.h>
#include <zenutil/workerpools.h>
@@ -101,7 +107,6 @@ CSVWriteOp(CidStore& CidStore,
//////////////////////////////////////////////////////////////////////////
namespace {
-
void CbWriteOp(CidStore& CidStore,
bool Details,
bool OpDetails,
@@ -244,6 +249,22 @@ namespace {
{
std::shared_ptr<RemoteProjectStore> Store;
std::string Description;
+ double LatencySec = -1.0;
+ uint64_t MaxRangeCountPerRequest = 1;
+
+ struct Cache
+ {
+ std::unique_ptr<HttpClient> Http;
+ std::unique_ptr<BuildStorageCache> Cache;
+ Oid BuildsId = Oid::Zero;
+ std::string Description;
+ double LatencySec = -1.0;
+ uint64_t MaxRangeCountPerRequest = 1;
+ BuildStorageCache::Statistics Stats;
+ bool Populate = false;
+ };
+
+ std::unique_ptr<Cache> OptionalCache;
};
CreateRemoteStoreResult CreateRemoteStore(LoggerRef InLog,
@@ -252,7 +273,9 @@ namespace {
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
size_t MaximumInMemoryDownloadSize,
- const std::filesystem::path& TempFilePath)
+ const std::filesystem::path& TempFilePath,
+ const std::filesystem::path& OidcTokenExePath,
+ bool AllowExternalOidcTokenExe)
{
ZEN_MEMSCOPE(GetProjectHttpTag());
@@ -260,7 +283,7 @@ namespace {
using namespace std::literals;
- std::shared_ptr<RemoteProjectStore> RemoteStore;
+ CreateRemoteStoreResult Result;
if (CbObjectView File = Params["file"sv].AsObjectView(); File)
{
@@ -285,7 +308,9 @@ namespace {
std::string(OptionalBaseName),
ForceDisableBlocks,
ForceEnableTempBlocks};
- RemoteStore = CreateFileRemoteStore(Log(), Options);
+ Result.Store = CreateFileRemoteStore(Log(), Options);
+ Result.LatencySec = 0.5 / 1000.0; // 0.5 ms
+ Result.MaxRangeCountPerRequest = 1024u;
}
if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -317,19 +342,24 @@ namespace {
AccessToken = GetEnvVariable(AccessTokenEnvVariable);
}
}
- std::filesystem::path OidcExePath;
- if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+
+ std::filesystem::path OidcExePath = FindOidcTokenExePath(OidcTokenExePath.string());
+ if (OidcExePath.empty() && AllowExternalOidcTokenExe)
{
- std::filesystem::path OidcExePathMaybe(OidcExePathString);
- if (IsFile(OidcExePathMaybe))
- {
- OidcExePath = std::move(OidcExePathMaybe);
- }
- else
+ if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty())
{
- ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ std::filesystem::path OidcExePathMaybe(OidcExePathString);
+ if (IsFile(OidcExePathMaybe))
+ {
+ OidcExePath = std::move(OidcExePathMaybe);
+ }
+ else
+ {
+ ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ }
}
}
+
std::string_view KeyParam = Cloud["key"sv].AsString();
if (KeyParam.empty())
{
@@ -363,21 +393,32 @@ namespace {
bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false);
- JupiterRemoteStoreOptions Options = {
- RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
- Url,
- std::string(Namespace),
- std::string(Bucket),
- Key,
- BaseKey,
- std::string(OpenIdProvider),
- AccessToken,
- AuthManager,
- OidcExePath,
- ForceDisableBlocks,
- ForceDisableTempBlocks,
- AssumeHttp2};
- RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+ if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(Url, AssumeHttp2, /*Verbose*/ false); TestResult.Success)
+ {
+ Result.LatencySec = TestResult.LatencySeconds;
+ Result.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+
+ JupiterRemoteStoreOptions Options = {
+ RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+ Url,
+ std::string(Namespace),
+ std::string(Bucket),
+ Key,
+ BaseKey,
+ std::string(OpenIdProvider),
+ AccessToken,
+ AuthManager,
+ OidcExePath,
+ ForceDisableBlocks,
+ ForceDisableTempBlocks,
+ AssumeHttp2};
+ Result.Store =
+ CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+ }
+ else
+ {
+ return {nullptr, fmt::format("Unable to connect to jupiter host '{}'", Url)};
+ }
}
if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -393,12 +434,13 @@ namespace {
{
return {nullptr, "Missing oplog"};
}
+
ZenRemoteStoreOptions Options = {
RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
std::string(Url),
std::string(Project),
std::string(Oplog)};
- RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath);
+ Result.Store = CreateZenRemoteStore(Log(), Options, TempFilePath);
}
if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
@@ -435,19 +477,24 @@ namespace {
AccessToken = GetEnvVariable(AccessTokenEnvVariable);
}
}
- std::filesystem::path OidcExePath;
- if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+
+ std::filesystem::path OidcExePath = FindOidcTokenExePath(OidcTokenExePath.string());
+ if (OidcExePath.empty() && AllowExternalOidcTokenExe)
{
- std::filesystem::path OidcExePathMaybe(OidcExePathString);
- if (IsFile(OidcExePathMaybe))
+ if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty())
{
- OidcExePath = std::move(OidcExePathMaybe);
- }
- else
- {
- ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ std::filesystem::path OidcExePathMaybe(OidcExePathString);
+ if (IsFile(OidcExePathMaybe))
+ {
+ OidcExePath = std::move(OidcExePathMaybe);
+ }
+ else
+ {
+ ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ }
}
}
+
std::string_view BuildIdParam = Builds["buildsid"sv].AsString();
if (BuildIdParam.empty())
{
@@ -471,11 +518,76 @@ namespace {
MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
+ auto EnsureHttps = [](const std::string& Host, std::string_view PreferredProtocol) {
+ if (!Host.empty() && Host.find("://"sv) == std::string::npos)
+ {
+ // Assume https URL
+ return fmt::format("{}://{}"sv, PreferredProtocol, Host);
+ }
+ return Host;
+ };
+
+ Host = EnsureHttps(Host, "https");
+ OverrideHost = EnsureHttps(OverrideHost, "https");
+ ZenHost = EnsureHttps(ZenHost, "http");
+
+ std::function<HttpClientAccessToken()> TokenProvider;
+ if (!OpenIdProvider.empty())
+ {
+ TokenProvider = httpclientauth::CreateFromOpenIdProvider(AuthManager, OpenIdProvider);
+ }
+ else if (!AccessToken.empty())
+ {
+ TokenProvider = httpclientauth::CreateFromStaticToken(AccessToken);
+ }
+ else if (!OidcExePath.empty())
+ {
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(OidcExePath,
+ Host.empty() ? OverrideHost : Host,
+ /*Quiet*/ false,
+ /*Unattended*/ false,
+ /*Hidden*/ true);
+ TokenProviderMaybe)
+ {
+ TokenProvider = TokenProviderMaybe.value();
+ }
+ }
+
+ if (!TokenProvider)
+ {
+ TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(AuthManager);
+ }
+
+ BuildStorageResolveResult ResolveResult;
+ {
+ HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
+ .AccessTokenProvider = TokenProvider,
+ .AssumeHttp2 = AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 2};
+
+ std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(Log()));
+
+ try
+ {
+ ResolveResult = ResolveBuildStorage(*Output,
+ ClientSettings,
+ Host,
+ OverrideHost,
+ ZenHost,
+ ZenCacheResolveMode::Discovery,
+ /*Verbose*/ false);
+ }
+ catch (const std::exception& Ex)
+ {
+ return {nullptr, fmt::format("Failed resolving storage host and cache. Reason: '{}'", Ex.what())};
+ }
+ }
+ Result.LatencySec = ResolveResult.Cloud.LatencySec;
+ Result.MaxRangeCountPerRequest = ResolveResult.Cloud.Caps.MaxRangeCountPerRequest;
+
BuildsRemoteStoreOptions Options = {
RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
- Host,
- OverrideHost,
- ZenHost,
std::string(Namespace),
std::string(Bucket),
BuildId,
@@ -485,25 +597,43 @@ namespace {
OidcExePath,
ForceDisableBlocks,
ForceDisableTempBlocks,
- AssumeHttp2,
- PopulateCache,
MetaData,
MaximumInMemoryDownloadSize};
- RemoteStore = CreateJupiterBuildsRemoteStore(Log(),
- Options,
- TempFilePath,
- /*Quiet*/ false,
- /*Unattended*/ false,
- /*Hidden*/ true,
- GetTinyWorkerPool(EWorkloadType::Background));
+ Result.Store = CreateJupiterBuildsRemoteStore(Log(), ResolveResult, std::move(TokenProvider), Options, TempFilePath);
+
+ if (!ResolveResult.Cache.Address.empty())
+ {
+ Result.OptionalCache = std::make_unique<CreateRemoteStoreResult::Cache>();
+
+ HttpClientSettings CacheClientSettings{.LogCategory = "httpcacheclient",
+ .ConnectTimeout = std::chrono::milliseconds{3000},
+ .Timeout = std::chrono::milliseconds{30000},
+ .AssumeHttp2 = ResolveResult.Cache.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 0,
+ .MaximumInMemoryDownloadSize = MaximumInMemoryDownloadSize};
+
+ Result.OptionalCache->Http = std::make_unique<HttpClient>(ResolveResult.Cache.Address, CacheClientSettings);
+ Result.OptionalCache->Cache = CreateZenBuildStorageCache(*Result.OptionalCache->Http,
+ Result.OptionalCache->Stats,
+ Namespace,
+ Bucket,
+ TempFilePath,
+ GetTinyWorkerPool(EWorkloadType::Background));
+ Result.OptionalCache->BuildsId = BuildId;
+ Result.OptionalCache->LatencySec = ResolveResult.Cache.LatencySec;
+ Result.OptionalCache->MaxRangeCountPerRequest = ResolveResult.Cache.Caps.MaxRangeCountPerRequest;
+ Result.OptionalCache->Populate = PopulateCache;
+ Result.OptionalCache->Description =
+ fmt::format("[zenserver] {} namespace {} bucket {}", ResolveResult.Cache.Address, Namespace, Bucket);
+ }
}
-
- if (!RemoteStore)
+ if (!Result.Store)
{
return {nullptr, "Unknown remote store type"};
}
- return {std::move(RemoteStore), ""};
+ return Result;
}
std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result)
@@ -529,13 +659,16 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store,
- ProjectStore* Projects,
- HttpStatusService& StatusService,
- HttpStatsService& StatsService,
- AuthMgr& AuthMgr,
- OpenProcessCache& InOpenProcessCache,
- JobQueue& InJobQueue)
+HttpProjectService::HttpProjectService(CidStore& Store,
+ ProjectStore* Projects,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr,
+ OpenProcessCache& InOpenProcessCache,
+ JobQueue& InJobQueue,
+ bool InRestrictContentTypes,
+ const std::filesystem::path& InOidcTokenExePath,
+ bool InAllowExternalOidcTokenExe)
: m_Log(logging::Get("project"))
, m_CidStore(Store)
, m_ProjectStore(Projects)
@@ -544,6 +677,9 @@ HttpProjectService::HttpProjectService(CidStore& Store,
, m_AuthMgr(AuthMgr)
, m_OpenProcessCache(InOpenProcessCache)
, m_JobQueue(InJobQueue)
+, m_RestrictContentTypes(InRestrictContentTypes)
+, m_OidcTokenExePath(InOidcTokenExePath)
+, m_AllowExternalOidcTokenExe(InAllowExternalOidcTokenExe)
{
ZEN_MEMSCOPE(GetProjectHttpTag());
@@ -712,8 +848,8 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
}
}
-void
-HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+CbObject
+HttpProjectService::CollectStats()
{
ZEN_TRACE_CPU("ProjectService::Stats");
@@ -779,7 +915,13 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
Cbo.EndObject();
- return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ return Cbo.Save();
+}
+
+void
+HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+ HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
}
void
@@ -1866,6 +2008,14 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req)
{
return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
}
+
+ if (m_RestrictContentTypes && (HttpReq.RequestContentType() == HttpContentType::kText ||
+ HttpReq.RequestContentType() == HttpContentType::kUnknownContentType))
+ {
+ m_ProjectStats.BadRequestCount++;
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ }
+
std::filesystem::path OplogMarkerPath;
if (CbObject Params = HttpReq.ReadPayloadObject())
{
@@ -2156,6 +2306,13 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
}
+ if (m_RestrictContentTypes && (HttpReq.RequestContentType() == HttpContentType::kText ||
+ HttpReq.RequestContentType() == HttpContentType::kUnknownContentType))
+ {
+ m_ProjectStats.BadRequestCount++;
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ }
+
CbValidateError ValidateResult;
if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult);
ValidateResult == CbValidateError::None)
@@ -2356,15 +2513,19 @@ HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req)
tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); };
- auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
+ auto OnNeedBlock = [&AttachmentsLock, &Attachments](ThinChunkBlockDescription&& ThinBlockDescription,
+ std::vector<uint32_t>&& NeededChunkIndexes) {
RwLock::ExclusiveLockScope _(AttachmentsLock);
- if (BlockHash != IoHash::Zero)
+ if (ThinBlockDescription.BlockHash != IoHash::Zero)
{
- Attachments.insert(BlockHash);
+ Attachments.insert(ThinBlockDescription.BlockHash);
}
else
{
- Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
+ for (uint32_t ChunkIndex : NeededChunkIndexes)
+ {
+ Attachments.insert(ThinBlockDescription.ChunkRawHashes[ChunkIndex]);
+ }
}
};
auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
@@ -2567,10 +2728,17 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
CbObject Cb;
switch (PayloadContentType)
{
- case HttpContentType::kJSON:
- case HttpContentType::kUnknownContentType:
case HttpContentType::kText:
+ case HttpContentType::kUnknownContentType:
+ case HttpContentType::kJSON:
{
+ if (m_RestrictContentTypes &&
+ (PayloadContentType == HttpContentType::kText || PayloadContentType == HttpContentType::kUnknownContentType))
+ {
+ m_ProjectStats.BadRequestCount++;
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ }
+
std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize());
Cb = LoadCompactBinaryFromJson(JsonText).AsObject();
if (!Cb)
@@ -2663,36 +2831,40 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
bool CleanOplog = Params["clean"].AsBool(false);
bool BoostWorkerCount = Params["boostworkercount"].AsBool(false);
bool BoostWorkerMemory = Params["boostworkermemory"sv].AsBool(false);
-
- CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(),
- Params,
- m_AuthMgr,
- MaxBlockSize,
- MaxChunkEmbedSize,
- GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
- Oplog->TempPath());
-
- if (RemoteStoreResult.Store == nullptr)
+ EPartialBlockRequestMode PartialBlockRequestMode =
+ PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true"));
+
+ std::shared_ptr<CreateRemoteStoreResult> RemoteStoreResult =
+ std::make_shared<CreateRemoteStoreResult>(CreateRemoteStore(Log(),
+ Params,
+ m_AuthMgr,
+ MaxBlockSize,
+ MaxChunkEmbedSize,
+ GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
+ Oplog->TempPath(),
+ m_OidcTokenExePath,
+ m_AllowExternalOidcTokenExe));
+ if (RemoteStoreResult->Store == nullptr)
{
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult->Description);
}
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
- RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
JobId JobId = m_JobQueue.QueueJob(
fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()),
[this,
- ChunkStore = &m_CidStore,
- ActualRemoteStore = std::move(RemoteStore),
+ RemoteStoreResult = std::move(RemoteStoreResult),
Oplog,
Force,
IgnoreMissingAttachments,
CleanOplog,
+ PartialBlockRequestMode,
BoostWorkerCount](JobContext& Context) {
- Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}",
- Oplog->GetOuterProjectIdentifier(),
- Oplog->OplogId(),
- ActualRemoteStore->GetInfo().Description));
+ Context.ReportMessage(
+ fmt::format("Loading oplog '{}/{}'\n Host: {}\n Cache: {}",
+ Oplog->GetOuterProjectIdentifier(),
+ Oplog->OplogId(),
+ RemoteStoreResult->Store->GetInfo().Description,
+ RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Description : "<none>"));
Ref<TransferThreadWorkers> Workers = GetThreadWorkers(BoostWorkerCount, /*SingleThreaded*/ false);
@@ -2700,16 +2872,26 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
WorkerThreadPool& NetworkWorkerPool = Workers->GetNetworkPool();
Context.ReportMessage(fmt::format("{}", Workers->GetWorkersInfo()));
-
- RemoteProjectStore::Result Result = LoadOplog(m_CidStore,
- *ActualRemoteStore,
- *Oplog,
- NetworkWorkerPool,
- WorkerPool,
- Force,
- IgnoreMissingAttachments,
- CleanOplog,
- &Context);
+ RemoteProjectStore::Result Result = LoadOplog(LoadOplogContext{
+ .ChunkStore = m_CidStore,
+ .RemoteStore = *RemoteStoreResult->Store,
+ .OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr,
+ .CacheBuildId = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->BuildsId : Oid::Zero,
+ .OptionalCacheStats = RemoteStoreResult->OptionalCache ? &RemoteStoreResult->OptionalCache->Stats : nullptr,
+ .Oplog = *Oplog,
+ .NetworkWorkerPool = NetworkWorkerPool,
+ .WorkerPool = WorkerPool,
+ .ForceDownload = Force,
+ .IgnoreMissingAttachments = IgnoreMissingAttachments,
+ .CleanOplog = CleanOplog,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .PopulateCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Populate : false,
+ .StoreLatencySec = RemoteStoreResult->LatencySec,
+ .StoreMaxRangeCountPerRequest = RemoteStoreResult->MaxRangeCountPerRequest,
+ .CacheLatencySec = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->LatencySec : -1.0,
+ .CacheMaxRangeCountPerRequest =
+ RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->MaxRangeCountPerRequest : 0,
+ .OptionalJobContext = &Context});
auto Response = ConvertResult(Result);
ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
@@ -2740,7 +2922,9 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
MaxBlockSize,
MaxChunkEmbedSize,
GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
- Oplog->TempPath());
+ Oplog->TempPath(),
+ m_OidcTokenExePath,
+ m_AllowExternalOidcTokenExe);
if (RemoteStoreResult.Store == nullptr)
{
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h
index b742102a5..917337324 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.h
+++ b/src/zenserver/storage/projectstore/httpprojectstore.h
@@ -38,20 +38,24 @@ class TransferThreadWorkers;
class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(CidStore& Store,
- ProjectStore* InProjectStore,
- HttpStatusService& StatusService,
- HttpStatsService& StatsService,
- AuthMgr& AuthMgr,
- OpenProcessCache& InOpenProcessCache,
- JobQueue& InJobQueue);
+ HttpProjectService(CidStore& Store,
+ ProjectStore* InProjectStore,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr,
+ OpenProcessCache& InOpenProcessCache,
+ JobQueue& InJobQueue,
+ bool InRestrictContentTypes,
+ const std::filesystem::path& InOidcTokenExePath,
+ bool AllowExternalOidcTokenExe);
~HttpProjectService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct ProjectStats
@@ -109,6 +113,9 @@ private:
metrics::OperationTiming m_HttpRequests;
RwLock m_ThreadWorkersLock;
Ref<TransferThreadWorkers> m_ThreadWorkers;
+ bool m_RestrictContentTypes;
+ std::filesystem::path m_OidcTokenExePath;
+ bool m_AllowExternalOidcTokenExe;
Ref<TransferThreadWorkers> GetThreadWorkers(bool BoostWorkers, bool SingleThreaded);
};
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-27 02:19:08 +0000