diff options
Diffstat (limited to 'src/zenserver/storage/projectstore')
| -rw-r--r-- | src/zenserver/storage/projectstore/httpprojectstore.cpp | 642 | ||||
| -rw-r--r-- | src/zenserver/storage/projectstore/httpprojectstore.h | 34 |
2 files changed, 470 insertions, 206 deletions
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp index 2b5474d00..9844d02f0 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.cpp +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -13,7 +13,12 @@ #include <zencore/scopeguard.h> #include <zencore/stream.h> #include <zencore/trace.h> +#include <zenhttp/httpclientauth.h> #include <zenhttp/packageformat.h> +#include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/buildstorageresolve.h> +#include <zenremotestore/builds/buildstorageutil.h> +#include <zenremotestore/jupiter/jupiterhost.h> #include <zenremotestore/projectstore/buildsremoteprojectstore.h> #include <zenremotestore/projectstore/fileremoteprojectstore.h> #include <zenremotestore/projectstore/jupiterremoteprojectstore.h> @@ -23,6 +28,7 @@ #include <zenstore/oplogreferencedset.h> #include <zenstore/projectstore.h> #include <zenstore/zenstore.h> +#include <zenutil/authutils.h> #include <zenutil/openprocesscache.h> #include <zenutil/workerpools.h> @@ -101,7 +107,6 @@ CSVWriteOp(CidStore& CidStore, ////////////////////////////////////////////////////////////////////////// namespace { - void CbWriteOp(CidStore& CidStore, bool Details, bool OpDetails, @@ -244,8 +249,22 @@ namespace { { std::shared_ptr<RemoteProjectStore> Store; std::string Description; - double HostLatencySec = -1.0; - double CacheLatencySec = -1.0; + double LatencySec = -1.0; + uint64_t MaxRangeCountPerRequest = 1; + + struct Cache + { + std::unique_ptr<HttpClient> Http; + std::unique_ptr<BuildStorageCache> Cache; + Oid BuildsId = Oid::Zero; + std::string Description; + double LatencySec = -1.0; + uint64_t MaxRangeCountPerRequest = 1; + BuildStorageCache::Statistics Stats; + bool Populate = false; + }; + + std::unique_ptr<Cache> OptionalCache; }; CreateRemoteStoreResult CreateRemoteStore(LoggerRef InLog, @@ -254,17 +273,17 @@ namespace { size_t MaxBlockSize, size_t MaxChunkEmbedSize, size_t MaximumInMemoryDownloadSize, - const std::filesystem::path& TempFilePath) + const std::filesystem::path& TempFilePath, + const std::filesystem::path& OidcTokenExePath, + bool AllowExternalOidcTokenExe) { ZEN_MEMSCOPE(GetProjectHttpTag()); - auto Log = [InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; - std::shared_ptr<RemoteProjectStore> RemoteStore; - double HostLatencySec = -1.0; - double CacheLatencySec = -1.0; + CreateRemoteStoreResult Result; if (CbObjectView File = Params["file"sv].AsObjectView(); File) { @@ -289,7 +308,9 @@ namespace { std::string(OptionalBaseName), ForceDisableBlocks, ForceEnableTempBlocks}; - RemoteStore = CreateFileRemoteStore(Log(), Options); + Result.Store = CreateFileRemoteStore(Log(), Options); + Result.LatencySec = 0.5 / 1000.0; // 0.5 ms + Result.MaxRangeCountPerRequest = 1024u; } if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) @@ -321,19 +342,24 @@ namespace { AccessToken = GetEnvVariable(AccessTokenEnvVariable); } } - std::filesystem::path OidcExePath; - if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + + std::filesystem::path OidcExePath = FindOidcTokenExePath(OidcTokenExePath.string()); + if (OidcExePath.empty() && AllowExternalOidcTokenExe) { - std::filesystem::path OidcExePathMaybe(OidcExePathString); - if (IsFile(OidcExePathMaybe)) + if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty()) { - OidcExePath = std::move(OidcExePathMaybe); - } - else - { - ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } } } + std::string_view KeyParam = Cloud["key"sv].AsString(); if (KeyParam.empty()) { @@ -367,21 +393,32 @@ namespace { bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false); bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false); - JupiterRemoteStoreOptions Options = { - RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Url, - std::string(Namespace), - std::string(Bucket), - Key, - BaseKey, - std::string(OpenIdProvider), - AccessToken, - AuthManager, - OidcExePath, - ForceDisableBlocks, - ForceDisableTempBlocks, - AssumeHttp2}; - RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(Url, AssumeHttp2, /*Verbose*/ false); TestResult.Success) + { + Result.LatencySec = TestResult.LatencySeconds; + Result.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + + JupiterRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + Key, + BaseKey, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2}; + Result.Store = + CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + else + { + return {nullptr, fmt::format("Unable to connect to jupiter host '{}'", Url)}; + } } if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) @@ -397,12 +434,13 @@ namespace { { return {nullptr, "Missing oplog"}; } + ZenRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, std::string(Url), std::string(Project), std::string(Oplog)}; - RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath); + Result.Store = CreateZenRemoteStore(Log(), Options, TempFilePath); } if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds) @@ -439,19 +477,24 @@ namespace { AccessToken = GetEnvVariable(AccessTokenEnvVariable); } } - std::filesystem::path OidcExePath; - if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + + std::filesystem::path OidcExePath = FindOidcTokenExePath(OidcTokenExePath.string()); + if (OidcExePath.empty() && AllowExternalOidcTokenExe) { - std::filesystem::path OidcExePathMaybe(OidcExePathString); - if (IsFile(OidcExePathMaybe)) - { - OidcExePath = std::move(OidcExePathMaybe); - } - else + if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty()) { - ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } } } + std::string_view BuildIdParam = Builds["buildsid"sv].AsString(); if (BuildIdParam.empty()) { @@ -475,11 +518,74 @@ namespace { MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView(); IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize()); + auto EnsureHttps = [](const std::string& Host, std::string_view PreferredProtocol) { + if (!Host.empty() && Host.find("://"sv) == std::string::npos) + { + // Assume https URL + return fmt::format("{}://{}"sv, PreferredProtocol, Host); + } + return Host; + }; + + Host = EnsureHttps(Host, "https"); + OverrideHost = EnsureHttps(OverrideHost, "https"); + ZenHost = EnsureHttps(ZenHost, "http"); + + std::function<HttpClientAccessToken()> TokenProvider; + if (!OpenIdProvider.empty()) + { + TokenProvider = httpclientauth::CreateFromOpenIdProvider(AuthManager, OpenIdProvider); + } + else if (!AccessToken.empty()) + { + TokenProvider = httpclientauth::CreateFromStaticToken(AccessToken); + } + else if (!OidcExePath.empty()) + { + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(OidcExePath, + Host.empty() ? OverrideHost : Host, + /*Quiet*/ false, + /*Unattended*/ false, + /*Hidden*/ false); + TokenProviderMaybe) + { + TokenProvider = TokenProviderMaybe.value(); + } + } + + if (!TokenProvider) + { + TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(AuthManager); + } + + BuildStorageResolveResult ResolveResult; + { + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AccessTokenProvider = TokenProvider, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; + + try + { + ResolveResult = ResolveBuildStorage(Log(), + ClientSettings, + Host, + OverrideHost, + ZenHost, + ZenCacheResolveMode::Discovery, + /*Verbose*/ false); + } + catch (const std::exception& Ex) + { + return {nullptr, fmt::format("Failed resolving storage host and cache. Reason: '{}'", Ex.what())}; + } + } + Result.LatencySec = ResolveResult.Cloud.LatencySec; + Result.MaxRangeCountPerRequest = ResolveResult.Cloud.Caps.MaxRangeCountPerRequest; + BuildsRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Host, - OverrideHost, - ZenHost, std::string(Namespace), std::string(Bucket), BuildId, @@ -489,47 +595,43 @@ namespace { OidcExePath, ForceDisableBlocks, ForceDisableTempBlocks, - AssumeHttp2, - PopulateCache, MetaData, MaximumInMemoryDownloadSize}; - RemoteStore = CreateJupiterBuildsRemoteStore(Log(), - Options, - TempFilePath, - /*Quiet*/ false, - /*Unattended*/ false, - /*Hidden*/ true, - GetTinyWorkerPool(EWorkloadType::Background), - HostLatencySec, - CacheLatencySec); + Result.Store = CreateJupiterBuildsRemoteStore(Log(), ResolveResult, std::move(TokenProvider), Options, TempFilePath); + + if (!ResolveResult.Cache.Address.empty()) + { + Result.OptionalCache = std::make_unique<CreateRemoteStoreResult::Cache>(); + + HttpClientSettings CacheClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = ResolveResult.Cache.AssumeHttp2, + .AllowResume = true, + .RetryCount = 0, + .MaximumInMemoryDownloadSize = MaximumInMemoryDownloadSize}; + + Result.OptionalCache->Http = std::make_unique<HttpClient>(ResolveResult.Cache.Address, CacheClientSettings); + Result.OptionalCache->Cache = CreateZenBuildStorageCache(*Result.OptionalCache->Http, + Result.OptionalCache->Stats, + Namespace, + Bucket, + TempFilePath, + GetTinyWorkerPool(EWorkloadType::Background)); + Result.OptionalCache->BuildsId = BuildId; + Result.OptionalCache->LatencySec = ResolveResult.Cache.LatencySec; + Result.OptionalCache->MaxRangeCountPerRequest = ResolveResult.Cache.Caps.MaxRangeCountPerRequest; + Result.OptionalCache->Populate = PopulateCache; + Result.OptionalCache->Description = + fmt::format("[zenserver] {} namespace {} bucket {}", ResolveResult.Cache.Address, Namespace, Bucket); + } } - - if (!RemoteStore) + if (!Result.Store) { return {nullptr, "Unknown remote store type"}; } - return CreateRemoteStoreResult{.Store = std::move(RemoteStore), - .Description = "", - .HostLatencySec = HostLatencySec, - .CacheLatencySec = CacheLatencySec}; - } - - std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result) - { - if (Result.ErrorCode == 0) - { - return {HttpResponseCode::OK, Result.Text}; - } - return {static_cast<HttpResponseCode>(Result.ErrorCode), - Result.Reason.empty() ? Result.Text - : Result.Text.empty() ? Result.Reason - : fmt::format("{}: {}", Result.Reason, Result.Text)}; - } - - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; + return Result; } } // namespace @@ -538,13 +640,17 @@ namespace { ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(CidStore& Store, - ProjectStore* Projects, - HttpStatusService& StatusService, - HttpStatsService& StatsService, - AuthMgr& AuthMgr, - OpenProcessCache& InOpenProcessCache, - JobQueue& InJobQueue) +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue, + bool InRestrictContentTypes, + const std::filesystem::path& InOidcTokenExePath, + bool InAllowExternalOidcTokenExe, + const ILocalRefPolicy* InLocalRefPolicy) : m_Log(logging::Get("project")) , m_CidStore(Store) , m_ProjectStore(Projects) @@ -553,6 +659,10 @@ HttpProjectService::HttpProjectService(CidStore& Store, , m_AuthMgr(AuthMgr) , m_OpenProcessCache(InOpenProcessCache) , m_JobQueue(InJobQueue) +, m_RestrictContentTypes(InRestrictContentTypes) +, m_OidcTokenExePath(InOidcTokenExePath) +, m_AllowExternalOidcTokenExe(InAllowExternalOidcTokenExe) +, m_LocalRefPolicy(InLocalRefPolicy) { ZEN_MEMSCOPE(GetProjectHttpTag()); @@ -670,22 +780,22 @@ HttpProjectService::HttpProjectService(CidStore& Store, HttpVerb::kPost); m_Router.RegisterRoute( - "details\\$", + "details$", [this](HttpRouterRequest& Req) { HandleDetailsRequest(Req); }, HttpVerb::kGet); m_Router.RegisterRoute( - "details\\$/{project}", + "details$/{project}", [this](HttpRouterRequest& Req) { HandleProjectDetailsRequest(Req); }, HttpVerb::kGet); m_Router.RegisterRoute( - "details\\$/{project}/{log}", + "details$/{project}/{log}", [this](HttpRouterRequest& Req) { HandleOplogDetailsRequest(Req); }, HttpVerb::kGet); m_Router.RegisterRoute( - "details\\$/{project}/{log}/{chunk}", + "details$/{project}/{log}/{chunk}", [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, HttpVerb::kGet); @@ -705,6 +815,18 @@ HttpProjectService::BaseUri() const return "/prj/"; } +bool +HttpProjectService::AcceptsLocalFileReferences() const +{ + return true; +} + +const ILocalRefPolicy* +HttpProjectService::GetLocalRefPolicy() const +{ + return m_LocalRefPolicy; +} + void HttpProjectService::HandleRequest(HttpServerRequest& Request) { @@ -722,6 +844,15 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request) } void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { ZEN_TRACE_CPU("ProjectService::Stats"); @@ -733,6 +864,8 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) EmitSnapshot("requests", m_HttpRequests, Cbo); + Cbo << "project_count" << (uint64_t)m_ProjectStore->ProjectCount(); + Cbo.BeginObject("store"); { Cbo.BeginObject("size"); @@ -788,16 +921,25 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) } Cbo.EndObject(); - return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } -void -HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +CbObject +HttpProjectService::CollectStats() { - ZEN_TRACE_CPU("HttpProjectService::Status"); CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + // CollectStats does not use the HandleStatsRequest implementation to get stats since it uses some heavy operations such as + // m_ProjectStore->StorageSize(); + EmitSnapshot("requests", m_HttpRequests, Cbo); + Cbo << "project_count" << (uint64_t)m_ProjectStore->ProjectCount(); + + return Cbo.Save(); +} + +uint64_t +HttpProjectService::GetActivityCounter() +{ + return m_HttpRequests.Count(); } void @@ -1115,7 +1257,7 @@ HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req) const Oid Obj = Oid::FromHexString(ChunkId); - CbObject ResponsePayload = ProjectStore::GetChunkInfo(Log(), *Project, *FoundLog, Obj); + CbObject ResponsePayload = ProjectStore::GetChunkInfo(*Project, *FoundLog, Obj); if (ResponsePayload) { m_ProjectStats.ChunkHitCount++; @@ -1204,7 +1346,7 @@ HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req) HttpContentType AcceptType = HttpReq.AcceptContentType(); ProjectStore::GetChunkRangeResult Result = - ProjectStore::GetChunkRange(Log(), *Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); + ProjectStore::GetChunkRange(*Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); switch (Result.Error) { @@ -1533,7 +1675,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) CbPackage Package; - if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) + const bool ValidateHashes = false; + if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver, ValidateHashes)) { CbValidateError ValidateResult; if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); @@ -1875,6 +2018,14 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) { return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); } + + if (m_RestrictContentTypes && (HttpReq.RequestContentType() == HttpContentType::kText || + HttpReq.RequestContentType() == HttpContentType::kUnknownContentType)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + } + std::filesystem::path OplogMarkerPath; if (CbObject Params = HttpReq.ReadPayloadObject()) { @@ -2165,6 +2316,13 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); } + if (m_RestrictContentTypes && (HttpReq.RequestContentType() == HttpContentType::kText || + HttpReq.RequestContentType() == HttpContentType::kUnknownContentType)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + } + CbValidateError ValidateResult; if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); ValidateResult == CbValidateError::None) @@ -2523,44 +2681,39 @@ HttpProjectService::HandleOplogLoadRequest(HttpRouterRequest& Req) WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); - RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( - m_CidStore, - *Project, - *Oplog, - WorkerPool, - MaxBlockSize, - MaxChunkEmbedSize, - MaxChunksPerBlock, - ChunkFileSizeLimit, - /* BuildBlocks */ false, - /* IgnoreMissingAttachments */ false, - /* AllowChunking*/ false, - [](CompressedBuffer&&, ChunkBlockDescription&&) {}, - [](const IoHash&, TGetAttachmentBufferFunc&&) {}, - [](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {}, - /* EmbedLooseFiles*/ false); - - if (ContainerResult.ErrorCode == 0) - { - return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerResult.ContainerObject); - } - else - { - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - ContainerResult.ErrorCode, - ContainerResult.Reason); - - if (ContainerResult.Reason.empty()) + try + { + CbObject ContainerObject = BuildContainer( + Log(), + m_CidStore, + *Project, + *Oplog, + WorkerPool, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + /* BuildBlocks */ false, + /* IgnoreMissingAttachments */ false, + /* AllowChunking*/ false, + [](CompressedBuffer&&, ChunkBlockDescription&&) {}, + [](const IoHash&, TGetAttachmentBufferFunc&&) {}, + [](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {}, + /* EmbedLooseFiles*/ false); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + catch (const HttpClientError& HttpEx) + { + if (HttpEx.GetInternalErrorCode() != HttpClientErrorCode::kOK) { - return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode)); + return HttpReq.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, HttpEx.what()); } else { - return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode), HttpContentType::kText, ContainerResult.Reason); + return HttpReq.WriteResponse(HttpEx.GetHttpResponseCode(), HttpContentType::kText, HttpEx.what()); } } + // Let server request handler deal with other exceptions } void @@ -2580,10 +2733,17 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) CbObject Cb; switch (PayloadContentType) { - case HttpContentType::kJSON: - case HttpContentType::kUnknownContentType: case HttpContentType::kText: + case HttpContentType::kUnknownContentType: + case HttpContentType::kJSON: { + if (m_RestrictContentTypes && + (PayloadContentType == HttpContentType::kText || PayloadContentType == HttpContentType::kUnknownContentType)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + } + std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize()); Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); if (!Cb) @@ -2612,7 +2772,11 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) case HttpContentType::kCbPackage: try { - Package = ParsePackageMessage(Payload); + ParseFlags PkgFlags = (HttpReq.IsLocalMachineRequest() && AcceptsLocalFileReferences()) ? ParseFlags::kAllowLocalReferences + : ParseFlags::kDefault; + const ILocalRefPolicy* PkgPolicy = + EnumHasAllFlags(PkgFlags, ParseFlags::kAllowLocalReferences) ? GetLocalRefPolicy() : nullptr; + Package = ParsePackageMessage(Payload, {}, PkgFlags, PkgPolicy); Cb = Package.GetObject(); } catch (const std::invalid_argument& ex) @@ -2679,38 +2843,37 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) EPartialBlockRequestMode PartialBlockRequestMode = PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true")); - CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(), - Params, - m_AuthMgr, - MaxBlockSize, - MaxChunkEmbedSize, - GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory), - Oplog->TempPath()); - - if (RemoteStoreResult.Store == nullptr) + std::shared_ptr<CreateRemoteStoreResult> RemoteStoreResult = + std::make_shared<CreateRemoteStoreResult>(CreateRemoteStore(Log(), + Params, + m_AuthMgr, + MaxBlockSize, + MaxChunkEmbedSize, + GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory), + Oplog->TempPath(), + m_OidcTokenExePath, + m_AllowExternalOidcTokenExe)); + if (RemoteStoreResult->Store == nullptr) { - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult->Description); } - std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); - RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); JobId JobId = m_JobQueue.QueueJob( fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), [this, - ChunkStore = &m_CidStore, - ActualRemoteStore = std::move(RemoteStore), + RemoteStoreResult = std::move(RemoteStoreResult), Oplog, Force, IgnoreMissingAttachments, CleanOplog, PartialBlockRequestMode, - HostLatencySec = RemoteStoreResult.HostLatencySec, - CacheLatencySec = RemoteStoreResult.CacheLatencySec, BoostWorkerCount](JobContext& Context) { - Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}", - Oplog->GetOuterProjectIdentifier(), - Oplog->OplogId(), - ActualRemoteStore->GetInfo().Description)); + Context.ReportMessage( + fmt::format("Loading oplog '{}/{}'\n Host: {}\n Cache: {}", + Oplog->GetOuterProjectIdentifier(), + Oplog->OplogId(), + RemoteStoreResult->Store->GetInfo().Description, + RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Description : "<none>")); Ref<TransferThreadWorkers> Workers = GetThreadWorkers(BoostWorkerCount, /*SingleThreaded*/ false); @@ -2719,24 +2882,70 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) Context.ReportMessage(fmt::format("{}", Workers->GetWorkersInfo())); - RemoteProjectStore::Result Result = LoadOplog(m_CidStore, - *ActualRemoteStore, - *Oplog, - NetworkWorkerPool, - WorkerPool, - Force, - IgnoreMissingAttachments, - CleanOplog, - PartialBlockRequestMode, - HostLatencySec, - CacheLatencySec, - &Context); - auto Response = ConvertResult(Result); - ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); - if (!IsHttpSuccessCode(Response.first)) + try + { + LoadOplog(LoadOplogContext{ + .Log = Log(), + .ChunkStore = m_CidStore, + .RemoteStore = *RemoteStoreResult->Store, + .OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr, + .CacheBuildId = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->BuildsId : Oid::Zero, + .OptionalCacheStats = RemoteStoreResult->OptionalCache ? &RemoteStoreResult->OptionalCache->Stats : nullptr, + .Oplog = *Oplog, + .NetworkWorkerPool = NetworkWorkerPool, + .WorkerPool = WorkerPool, + .ForceDownload = Force, + .IgnoreMissingAttachments = IgnoreMissingAttachments, + .CleanOplog = CleanOplog, + .PartialBlockRequestMode = PartialBlockRequestMode, + .PopulateCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Populate : false, + .StoreLatencySec = RemoteStoreResult->LatencySec, + .StoreMaxRangeCountPerRequest = RemoteStoreResult->MaxRangeCountPerRequest, + .CacheLatencySec = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->LatencySec : -1.0, + .CacheMaxRangeCountPerRequest = + RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->MaxRangeCountPerRequest : 0, + .OptionalJobContext = &Context}); + } + catch (const HttpClientError& HttpEx) + { + if (HttpEx.GetInternalErrorCode() != HttpClientErrorCode::kOK) + { + throw JobError(fmt::format("Failed due to an http exception (Err: {}): {}", + static_cast<int>(HttpEx.GetInternalErrorCode()), + HttpEx.what()), + static_cast<int>(HttpEx.GetResponseClass())); + } + else + { + throw JobError(fmt::format("Failed due to an http exception (Status: {}): {}", + static_cast<int>(HttpEx.GetHttpResponseCode()), + HttpEx.what()), + static_cast<int>(HttpEx.GetHttpResponseCode())); + } + } + catch (const AssertException& AssertEx) + { + throw JobError(fmt::format("Failed due to an assert exception: {}", AssertEx.FullDescription()), + static_cast<int>(HttpResponseCode::InternalServerError)); + } + catch (const std::system_error& SysEx) + { + throw JobError(fmt::format("Failed due to a system error ({}): {}", SysEx.code().value(), SysEx.what()), + SysEx.code().value()); + } + catch (const std::exception& Ex) + { + throw JobError(fmt::format("Failed due to an exception: {}", Ex.what()), + static_cast<int>(HttpResponseCode::InternalServerError)); + } + + if (Context.IsCancelled()) { - throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, - (int)Response.first); + ZEN_INFO("LoadOplog: Operation cancelled"); + } + else + { + ZEN_INFO("LoadOplog: Complete"); } }); @@ -2761,7 +2970,9 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) MaxBlockSize, MaxChunkEmbedSize, GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory), - Oplog->TempPath()); + Oplog->TempPath(), + m_OidcTokenExePath, + m_AllowExternalOidcTokenExe); if (RemoteStoreResult.Store == nullptr) { @@ -2798,26 +3009,64 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) WorkerThreadPool& WorkerPool = Workers->GetIOWorkerPool(); WorkerThreadPool& NetworkWorkerPool = Workers->GetNetworkPool(); - RemoteProjectStore::Result Result = SaveOplog(m_CidStore, - *ActualRemoteStore, - *Project, - *Oplog, - NetworkWorkerPool, - WorkerPool, - MaxBlockSize, - MaxChunksPerBlock, - MaxChunkEmbedSize, - ChunkFileSizeLimit, - EmbedLooseFile, - Force, - IgnoreMissingAttachments, - &Context); - auto Response = ConvertResult(Result); - ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second); - if (!IsHttpSuccessCode(Response.first)) + try + { + SaveOplog(Log(), + m_CidStore, + *ActualRemoteStore, + *Project, + *Oplog, + NetworkWorkerPool, + WorkerPool, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + EmbedLooseFile, + Force, + IgnoreMissingAttachments, + &Context); + } + catch (const HttpClientError& HttpEx) + { + if (HttpEx.GetInternalErrorCode() != HttpClientErrorCode::kOK) + { + throw JobError(fmt::format("Failed due to an http exception (Err: {}): {}", + static_cast<int>(HttpEx.GetInternalErrorCode()), + HttpEx.what()), + static_cast<int>(HttpEx.GetResponseClass())); + } + else + { + throw JobError(fmt::format("Failed due to an http exception (Status: {}): {}", + static_cast<int>(HttpEx.GetHttpResponseCode()), + HttpEx.what()), + static_cast<int>(HttpEx.GetHttpResponseCode())); + } + } + catch (const AssertException& AssertEx) + { + throw JobError(fmt::format("Failed due to an assert exception: {}", AssertEx.FullDescription()), + static_cast<int>(HttpResponseCode::InternalServerError)); + } + catch (const std::system_error& SysEx) + { + throw JobError(fmt::format("Failed due to a system error ({}): {}", SysEx.code().value(), SysEx.what()), + SysEx.code().value()); + } + catch (const std::exception& Ex) + { + throw JobError(fmt::format("Failed due to an exception: {}", Ex.what()), + static_cast<int>(HttpResponseCode::InternalServerError)); + } + + if (Context.IsCancelled()) + { + ZEN_INFO("SaveOplog: Operation cancelled"); + } + else { - throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, - (int)Response.first); + ZEN_INFO("SaveOplog: Complete"); } }); @@ -2941,8 +3190,10 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) continue; } - std::error_code Ec; - const std::filesystem::path FilePath = std::filesystem::canonical(Project->RootDir / ServerPath, Ec); + std::error_code Ec; + // Long paths require MakeSafeAbsolutePath otherwise canonical will yield an error code + const std::filesystem::path SafeAbsFilePath = MakeSafeAbsolutePath(Project->RootDir / ServerPath); + const std::filesystem::path FilePath = std::filesystem::canonical(SafeAbsFilePath, Ec); if (Ec) { @@ -2963,7 +3214,8 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) } BasicFile DataFile; - DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); + // Must use SafeAbsFilePath because canonical will have removed leading chars for handling long paths + DataFile.Open(SafeAbsFilePath, BasicFile::Mode::kRead, Ec); if (Ec) { diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h index b742102a5..8aa345fa7 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.h +++ b/src/zenserver/storage/projectstore/httpprojectstore.h @@ -38,20 +38,28 @@ class TransferThreadWorkers; class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(CidStore& Store, - ProjectStore* InProjectStore, - HttpStatusService& StatusService, - HttpStatsService& StatsService, - AuthMgr& AuthMgr, - OpenProcessCache& InOpenProcessCache, - JobQueue& InJobQueue); + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue, + bool InRestrictContentTypes, + const std::filesystem::path& InOidcTokenExePath, + bool AllowExternalOidcTokenExe, + const ILocalRefPolicy* InLocalRefPolicy = nullptr); ~HttpProjectService(); - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + virtual bool AcceptsLocalFileReferences() const override; + virtual const ILocalRefPolicy* GetLocalRefPolicy() const override; - virtual void HandleStatsRequest(HttpServerRequest& Request) override; - virtual void HandleStatusRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual CbObject CollectStats() override; + virtual uint64_t GetActivityCounter() override; private: struct ProjectStats @@ -109,6 +117,10 @@ private: metrics::OperationTiming m_HttpRequests; RwLock m_ThreadWorkersLock; Ref<TransferThreadWorkers> m_ThreadWorkers; + bool m_RestrictContentTypes; + std::filesystem::path m_OidcTokenExePath; + bool m_AllowExternalOidcTokenExe; + const ILocalRefPolicy* m_LocalRefPolicy; Ref<TransferThreadWorkers> GetThreadWorkers(bool BoostWorkers, bool SingleThreaded); }; |