diff options
| author | Dan Engelbrecht <[email protected]> | 2025-01-23 12:49:46 +0100 |
|---|---|---|
| committer | Dan Engelbrecht <[email protected]> | 2025-01-23 12:49:46 +0100 |
| commit | 5d47e5946da5ccdba5bd2fb770b7bdfabb48fb4c (patch) | |
| tree | cb64f2d93a9f5e8b0e1529ab0dc3371602688a4f /src/zenserver/projectstore | |
| parent | changelog (diff) | |
| parent | handle special backslash followed by quote for paths (#279) (diff) | |
| download | zen-5d47e5946da5ccdba5bd2fb770b7bdfabb48fb4c.tar.xz zen-5d47e5946da5ccdba5bd2fb770b7bdfabb48fb4c.zip | |
Merge remote-tracking branch 'origin/main' into de/zen-service-command
Diffstat (limited to 'src/zenserver/projectstore')
| -rw-r--r-- | src/zenserver/projectstore/buildsremoteprojectstore.cpp | 113 | ||||
| -rw-r--r-- | src/zenserver/projectstore/httpprojectstore.h | 2 | ||||
| -rw-r--r-- | src/zenserver/projectstore/jupiterremoteprojectstore.cpp | 89 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 8 |
4 files changed, 99 insertions, 113 deletions
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index 0fe37449f..302b81729 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -6,8 +6,10 @@ #include <zencore/compress.h> #include <zencore/fmtutils.h> -#include <zenhttp/auth/authmgr.h> -#include <zenutil/jupiter.h> +#include <zenhttp/httpclientauth.h> + +#include <zenutil/jupiter/jupiterclient.h> +#include <zenutil/jupiter/jupitersession.h> namespace zen { @@ -18,7 +20,7 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv; class BuildsRemoteStore : public RemoteProjectStore { public: - BuildsRemoteStore(Ref<CloudCacheClient>&& CloudClient, + BuildsRemoteStore(Ref<JupiterClient>&& InJupiterClient, std::string_view Namespace, std::string_view Bucket, const Oid& BuildId, @@ -26,7 +28,7 @@ public: bool ForceDisableBlocks, bool ForceDisableTempBlocks, const std::filesystem::path& TempFilePath) - : m_CloudClient(std::move(CloudClient)) + : m_JupiterClient(std::move(InJupiterClient)) , m_Namespace(Namespace) , m_Bucket(Bucket) , m_BuildId(BuildId) @@ -50,7 +52,7 @@ public: .UseTempBlockFiles = m_UseTempBlocks, .AllowChunking = true, .ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId), - .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_CloudClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)}; + .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)}; } virtual Stats GetStats() const override @@ -68,18 +70,18 @@ public: { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); - CloudCacheSession Session(m_CloudClient.Get()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); IoBuffer Payload = m_MetaData; Payload.SetContentType(ZenContentType::kCbObject); - CloudCacheResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload); + JupiterResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload); AddStats(PutResult); CreateContainerResult Result{ConvertResult(PutResult)}; if (Result.ErrorCode) { Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -92,7 +94,7 @@ public: virtual SaveResult SaveContainer(const IoBuffer& Payload) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - CloudCacheSession Session(m_CloudClient.Get()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); PutBuildPartResult PutResult = Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload); AddStats(PutResult); @@ -101,7 +103,7 @@ public: if (Result.ErrorCode) { Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -115,9 +117,9 @@ public: virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&& Block) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - CloudCacheSession Session(m_CloudClient.Get()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); - CloudCacheResult PutResult = + JupiterResult PutResult = Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, ZenContentType::kCompressedBinary, Payload); AddStats(PutResult); @@ -125,7 +127,7 @@ public: if (Result.ErrorCode) { Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -176,14 +178,14 @@ public: IoBuffer MetaPayload = Writer.Save().GetBuffer().AsIoBuffer(); MetaPayload.SetContentType(ZenContentType::kCbObject); - CloudCacheResult PutMetaResult = + JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload); AddStats(PutMetaResult); RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult); if (MetaDataResult.ErrorCode) { ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -215,7 +217,7 @@ public: ZEN_UNUSED(RawHash); ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - CloudCacheSession Session(m_CloudClient.Get()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); FinalizeBuildPartResult FinalizeRefResult = Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash); AddStats(FinalizeRefResult); @@ -224,7 +226,7 @@ public: if (Result.ErrorCode) { Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -233,14 +235,14 @@ public: } else if (Result.Needs.empty()) { - CloudCacheResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId); + JupiterResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId); AddStats(FinalizeBuildResult); FinalizeBuildResult.ElapsedSeconds += FinalizeRefResult.ElapsedSeconds; Result = {ConvertResult(FinalizeBuildResult)}; if (Result.ErrorCode) { Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -254,14 +256,14 @@ public: { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); - CloudCacheSession Session(m_CloudClient.Get()); - CloudCacheResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId); AddStats(GetBuildResult); LoadContainerResult Result{ConvertResult(GetBuildResult)}; if (Result.ErrorCode) { Result.Reason = fmt::format("Failed fetching oplog container build from {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -273,7 +275,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("The build {}/{}/{}/{} payload is not formatted as a compact binary object"sv, - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId); @@ -284,7 +286,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv, - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId); @@ -295,7 +297,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv, - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -303,7 +305,7 @@ public: return Result; } - CloudCacheResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId); + JupiterResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId); AddStats(GetBuildPartResult); Result = {ConvertResult(GetBuildResult)}; Result.ElapsedSeconds += GetBuildResult.ElapsedSeconds; @@ -311,7 +313,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed fetching oplog build part from {}/{}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -325,7 +327,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("The build part for oplog container {}/{}/{}/{}/{} is not formatted as a compact binary object"sv, - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -339,15 +341,15 @@ public: virtual GetKnownBlocksResult GetKnownBlocks() override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - CloudCacheSession Session(m_CloudClient.Get()); - CloudCacheResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId); AddStats(FindResult); GetKnownBlocksResult Result{ConvertResult(FindResult)}; if (Result.ErrorCode) { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -360,7 +362,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv, - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -392,15 +394,15 @@ public: virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - CloudCacheSession Session(m_CloudClient.Get()); - CloudCacheResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath); AddStats(GetResult); LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)}; if (GetResult.ErrorCode) { Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}&{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, @@ -429,7 +431,7 @@ public: } private: - void AddStats(const CloudCacheResult& Result) + void AddStats(const JupiterResult& Result) { m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes)); m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes)); @@ -445,7 +447,7 @@ private: m_RequestCount.fetch_add(1); } - static Result ConvertResult(const CloudCacheResult& Response) + static Result ConvertResult(const JupiterResult& Response) { std::string Text; int32_t ErrorCode = 0; @@ -482,7 +484,7 @@ private: return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text}; } - Ref<CloudCacheClient> m_CloudClient; + Ref<JupiterClient> m_JupiterClient; const std::string m_Namespace; const std::string m_Bucket; const Oid m_BuildId; @@ -510,44 +512,35 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file // Assume https URL Url = fmt::format("https://{}"sv, Url); } - CloudCacheClientOptions ClientOptions{.Name = "Remote store"sv, - .ServiceUrl = Url, - .ConnectTimeout = std::chrono::milliseconds(2000), - .Timeout = std::chrono::milliseconds(1800000), - .AssumeHttp2 = Options.AssumeHttp2, - .AllowResume = true, - .RetryCount = 4}; + JupiterClientOptions ClientOptions{.Name = "Remote store"sv, + .ServiceUrl = Url, + .ConnectTimeout = std::chrono::milliseconds(2000), + .Timeout = std::chrono::milliseconds(1800000), + .AssumeHttp2 = Options.AssumeHttp2, + .AllowResume = true, + .RetryCount = 4}; // 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider // 2) Access token as parameter in request // 3) Environment variable (different win vs linux/mac) // 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider - std::unique_ptr<CloudCacheTokenProvider> TokenProvider; + std::function<HttpClientAccessToken()> TokenProvider; if (!Options.OpenIdProvider.empty()) { - TokenProvider = - CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() { - AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider); - return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; - }); + TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider); } else if (!Options.AccessToken.empty()) { - TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = "Bearer " + Options.AccessToken]() { - return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()}; - }); + TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken); } else { - TokenProvider = CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager]() { - AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken("Default"); - return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; - }); + TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); } - Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider))); + Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider))); - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(CloudClient), + std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(Client), Options.Namespace, Options.Bucket, Options.BuildId, diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 13810bd66..8e74c57a5 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -3,13 +3,13 @@ #pragma once #include <zencore/stats.h> -#include <zenhttp/auth/authmgr.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> #include <zenstore/cidstore.h> namespace zen { +class AuthMgr; class ProjectStore; ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index 5a42b2f50..e906127ff 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -5,8 +5,10 @@ #include <zencore/compress.h> #include <zencore/fmtutils.h> -#include <zenhttp/auth/authmgr.h> -#include <zenutil/jupiter.h> +#include <zenhttp/httpclientauth.h> + +#include <zenutil/jupiter/jupiterclient.h> +#include <zenutil/jupiter/jupitersession.h> namespace zen { @@ -15,7 +17,7 @@ using namespace std::literals; class JupiterRemoteStore : public RemoteProjectStore { public: - JupiterRemoteStore(Ref<CloudCacheClient>&& CloudClient, + JupiterRemoteStore(Ref<JupiterClient>&& InJupiterClient, std::string_view Namespace, std::string_view Bucket, const IoHash& Key, @@ -23,7 +25,7 @@ public: bool ForceDisableBlocks, bool ForceDisableTempBlocks, const std::filesystem::path& TempFilePath) - : m_CloudClient(std::move(CloudClient)) + : m_JupiterClient(std::move(InJupiterClient)) , m_Namespace(Namespace) , m_Bucket(Bucket) , m_Key(Key) @@ -47,7 +49,7 @@ public: .AllowChunking = true, .ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_Key), .Description = fmt::format("[cloud] {} as {}/{}/{}{}"sv, - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_Key, @@ -73,15 +75,15 @@ public: virtual SaveResult SaveContainer(const IoBuffer& Payload) override { - CloudCacheSession Session(m_CloudClient.Get()); - PutRefResult PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + PutRefResult PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject); AddStats(PutResult); SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash}; if (Result.ErrorCode) { Result.Reason = fmt::format("Failed saving oplog container to {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_Key, @@ -92,15 +94,15 @@ public: virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override { - CloudCacheSession Session(m_CloudClient.Get()); - CloudCacheResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload); AddStats(PutResult); SaveAttachmentResult Result{ConvertResult(PutResult)}; if (Result.ErrorCode) { Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, RawHash, Result.Reason); @@ -125,7 +127,7 @@ public: virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override { - CloudCacheSession Session(m_CloudClient.Get()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); FinalizeRefResult FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash); AddStats(FinalizeRefResult); @@ -133,7 +135,7 @@ public: if (Result.ErrorCode) { Result.Reason = fmt::format("Failed finalizing oplog container to {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_Key, @@ -162,8 +164,8 @@ public: {.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds}}; } - CloudCacheSession Session(m_CloudClient.Get()); - CloudCacheExistsResult ExistsResult = + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterExistsResult ExistsResult = Session.CompressedBlobExists(m_Namespace, std::set<IoHash>(BlockHashes.begin(), BlockHashes.end())); AddStats(ExistsResult); @@ -172,7 +174,7 @@ public: return GetKnownBlocksResult{{.ErrorCode = ExistsResult.ErrorCode, .ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds, .Reason = fmt::format("Failed checking attachment existance in {}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, ExistsResult.Reason)}}; } @@ -201,15 +203,15 @@ public: virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { - CloudCacheSession Session(m_CloudClient.Get()); - CloudCacheResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); AddStats(GetResult); LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)}; if (GetResult.ErrorCode) { Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, RawHash, Result.Reason); @@ -237,14 +239,14 @@ public: private: LoadContainerResult LoadContainer(const IoHash& Key) { - CloudCacheSession Session(m_CloudClient.Get()); - CloudCacheResult GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterResult GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject); AddStats(GetResult); if (GetResult.ErrorCode || !GetResult.Success) { LoadContainerResult Result{ConvertResult(GetResult)}; Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, Key, @@ -259,7 +261,7 @@ private: RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError), .ElapsedSeconds = GetResult.ElapsedSeconds, .Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv, - m_CloudClient->ServiceUrl(), + m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, Key)}, @@ -268,7 +270,7 @@ private: return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)}; } - void AddStats(const CloudCacheResult& Result) + void AddStats(const JupiterResult& Result) { m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes)); m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes)); @@ -284,7 +286,7 @@ private: m_RequestCount.fetch_add(1); } - static Result ConvertResult(const CloudCacheResult& Response) + static Result ConvertResult(const JupiterResult& Response) { std::string Text; int32_t ErrorCode = 0; @@ -321,7 +323,7 @@ private: return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text}; } - Ref<CloudCacheClient> m_CloudClient; + Ref<JupiterClient> m_JupiterClient; const std::string m_Namespace; const std::string m_Bucket; const IoHash m_Key; @@ -348,44 +350,35 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi // Assume https URL Url = fmt::format("https://{}"sv, Url); } - CloudCacheClientOptions ClientOptions{.Name = "Remote store"sv, - .ServiceUrl = Url, - .ConnectTimeout = std::chrono::milliseconds(2000), - .Timeout = std::chrono::milliseconds(1800000), - .AssumeHttp2 = Options.AssumeHttp2, - .AllowResume = true, - .RetryCount = 4}; + JupiterClientOptions ClientOptions{.Name = "Remote store"sv, + .ServiceUrl = Url, + .ConnectTimeout = std::chrono::milliseconds(2000), + .Timeout = std::chrono::milliseconds(1800000), + .AssumeHttp2 = Options.AssumeHttp2, + .AllowResume = true, + .RetryCount = 4}; // 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider // 2) Access token as parameter in request // 3) Environment variable (different win vs linux/mac) // 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider - std::unique_ptr<CloudCacheTokenProvider> TokenProvider; + std::function<HttpClientAccessToken()> TokenProvider; if (!Options.OpenIdProvider.empty()) { - TokenProvider = - CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() { - AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider); - return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; - }); + TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider); } else if (!Options.AccessToken.empty()) { - TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = "Bearer " + Options.AccessToken]() { - return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()}; - }); + TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken); } else { - TokenProvider = CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager]() { - AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken("Default"); - return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; - }); + TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); } - Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider))); + Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider))); - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(std::move(CloudClient), + std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(std::move(Client), Options.Namespace, Options.Bucket, Options.Key, diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 9e9fc0e77..46a236af9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -3506,7 +3506,7 @@ ProjectStore::Project::ScanForOplogs() const if (Project::Exists(m_OplogStoragePath)) { DirectoryContent DirContent; - GetDirectoryContent(m_OplogStoragePath, DirectoryContent::IncludeDirsFlag, DirContent); + GetDirectoryContent(m_OplogStoragePath, DirectoryContentFlags::IncludeDirs, DirContent); Oplogs.reserve(DirContent.Directories.size()); for (const std::filesystem::path& DirPath : DirContent.Directories) { @@ -3859,7 +3859,7 @@ ProjectStore::DiscoverProjects() } DirectoryContent DirContent; - GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent); + GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent); for (const std::filesystem::path& DirPath : DirContent.Directories) { @@ -3965,7 +3965,7 @@ ProjectStore::StorageSize() const if (std::filesystem::exists(m_ProjectBasePath)) { DirectoryContent ProjectsFolderContent; - GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, ProjectsFolderContent); + GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent); for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories) { @@ -3974,7 +3974,7 @@ ProjectStore::StorageSize() const { Result.DiskSize += Project::TotalSize(ProjectBasePath); DirectoryContent DirContent; - GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent); + GetDirectoryContent(ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent); for (const std::filesystem::path& OplogBasePath : DirContent.Directories) { Result.DiskSize += Oplog::TotalSize(OplogBasePath); |