aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/projectstore/buildsremoteprojectstore.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/projectstore/buildsremoteprojectstore.cpp')
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp113
1 files changed, 53 insertions, 60 deletions
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index 6d0d51a60..302b81729 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -6,8 +6,10 @@
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
-#include <upstream/jupiter.h>
-#include <zenhttp/auth/authmgr.h>
+#include <zenhttp/httpclientauth.h>
+
+#include <zenutil/jupiter/jupiterclient.h>
+#include <zenutil/jupiter/jupitersession.h>
namespace zen {
@@ -18,7 +20,7 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv;
class BuildsRemoteStore : public RemoteProjectStore
{
public:
- BuildsRemoteStore(Ref<CloudCacheClient>&& CloudClient,
+ BuildsRemoteStore(Ref<JupiterClient>&& InJupiterClient,
std::string_view Namespace,
std::string_view Bucket,
const Oid& BuildId,
@@ -26,7 +28,7 @@ public:
bool ForceDisableBlocks,
bool ForceDisableTempBlocks,
const std::filesystem::path& TempFilePath)
- : m_CloudClient(std::move(CloudClient))
+ : m_JupiterClient(std::move(InJupiterClient))
, m_Namespace(Namespace)
, m_Bucket(Bucket)
, m_BuildId(BuildId)
@@ -50,7 +52,7 @@ public:
.UseTempBlockFiles = m_UseTempBlocks,
.AllowChunking = true,
.ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId),
- .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_CloudClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)};
+ .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)};
}
virtual Stats GetStats() const override
@@ -68,18 +70,18 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
IoBuffer Payload = m_MetaData;
Payload.SetContentType(ZenContentType::kCbObject);
- CloudCacheResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload);
+ JupiterResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload);
AddStats(PutResult);
CreateContainerResult Result{ConvertResult(PutResult)};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -92,7 +94,7 @@ public:
virtual SaveResult SaveContainer(const IoBuffer& Payload) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
PutBuildPartResult PutResult =
Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload);
AddStats(PutResult);
@@ -101,7 +103,7 @@ public:
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -115,9 +117,9 @@ public:
virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&& Block) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
- CloudCacheResult PutResult =
+ JupiterResult PutResult =
Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, ZenContentType::kCompressedBinary, Payload);
AddStats(PutResult);
@@ -125,7 +127,7 @@ public:
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -176,14 +178,14 @@ public:
IoBuffer MetaPayload = Writer.Save().GetBuffer().AsIoBuffer();
MetaPayload.SetContentType(ZenContentType::kCbObject);
- CloudCacheResult PutMetaResult =
+ JupiterResult PutMetaResult =
Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload);
AddStats(PutMetaResult);
RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult);
if (MetaDataResult.ErrorCode)
{
ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -215,7 +217,7 @@ public:
ZEN_UNUSED(RawHash);
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
FinalizeBuildPartResult FinalizeRefResult =
Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash);
AddStats(FinalizeRefResult);
@@ -224,7 +226,7 @@ public:
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -233,14 +235,14 @@ public:
}
else if (Result.Needs.empty())
{
- CloudCacheResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId);
+ JupiterResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId);
AddStats(FinalizeBuildResult);
FinalizeBuildResult.ElapsedSeconds += FinalizeRefResult.ElapsedSeconds;
Result = {ConvertResult(FinalizeBuildResult)};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -254,14 +256,14 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
AddStats(GetBuildResult);
LoadContainerResult Result{ConvertResult(GetBuildResult)};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed fetching oplog container build from {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -273,7 +275,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build {}/{}/{}/{} payload is not formatted as a compact binary object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId);
@@ -284,7 +286,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId);
@@ -295,7 +297,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -303,7 +305,7 @@ public:
return Result;
}
- CloudCacheResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
+ JupiterResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
AddStats(GetBuildPartResult);
Result = {ConvertResult(GetBuildResult)};
Result.ElapsedSeconds += GetBuildResult.ElapsedSeconds;
@@ -311,7 +313,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("Failed fetching oplog build part from {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -325,7 +327,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build part for oplog container {}/{}/{}/{}/{} is not formatted as a compact binary object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -339,15 +341,15 @@ public:
virtual GetKnownBlocksResult GetKnownBlocks() override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
AddStats(FindResult);
GetKnownBlocksResult Result{ConvertResult(FindResult)};
if (Result.ErrorCode)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -360,7 +362,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -392,15 +394,15 @@ public:
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath);
AddStats(GetResult);
LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
if (GetResult.ErrorCode)
{
Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}&{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -429,7 +431,7 @@ public:
}
private:
- void AddStats(const CloudCacheResult& Result)
+ void AddStats(const JupiterResult& Result)
{
m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes));
m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes));
@@ -445,7 +447,7 @@ private:
m_RequestCount.fetch_add(1);
}
- static Result ConvertResult(const CloudCacheResult& Response)
+ static Result ConvertResult(const JupiterResult& Response)
{
std::string Text;
int32_t ErrorCode = 0;
@@ -482,7 +484,7 @@ private:
return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text};
}
- Ref<CloudCacheClient> m_CloudClient;
+ Ref<JupiterClient> m_JupiterClient;
const std::string m_Namespace;
const std::string m_Bucket;
const Oid m_BuildId;
@@ -510,44 +512,35 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
// Assume https URL
Url = fmt::format("https://{}"sv, Url);
}
- CloudCacheClientOptions ClientOptions{.Name = "Remote store"sv,
- .ServiceUrl = Url,
- .ConnectTimeout = std::chrono::milliseconds(2000),
- .Timeout = std::chrono::milliseconds(1800000),
- .AssumeHttp2 = Options.AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 4};
+ JupiterClientOptions ClientOptions{.Name = "Remote store"sv,
+ .ServiceUrl = Url,
+ .ConnectTimeout = std::chrono::milliseconds(2000),
+ .Timeout = std::chrono::milliseconds(1800000),
+ .AssumeHttp2 = Options.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 4};
// 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
// 2) Access token as parameter in request
// 3) Environment variable (different win vs linux/mac)
// 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
- std::unique_ptr<CloudCacheTokenProvider> TokenProvider;
+ std::function<HttpClientAccessToken()> TokenProvider;
if (!Options.OpenIdProvider.empty())
{
- TokenProvider =
- CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() {
- AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider);
- return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
- });
+ TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider);
}
else if (!Options.AccessToken.empty())
{
- TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = "Bearer " + Options.AccessToken]() {
- return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()};
- });
+ TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
}
else
{
- TokenProvider = CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager]() {
- AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken("Default");
- return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
- });
+ TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
}
- Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider)));
+ Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider)));
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(CloudClient),
+ std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(Client),
Options.Namespace,
Options.Bucket,
Options.BuildId,
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-20 05:43:44 +0000