aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/projectstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/projectstore')
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp113
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp12
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h2
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.cpp89
-rw-r--r--src/zenserver/projectstore/projectstore.cpp18
-rw-r--r--src/zenserver/projectstore/remoteprojectstore.cpp7
-rw-r--r--src/zenserver/projectstore/zenremoteprojectstore.cpp2
7 files changed, 118 insertions, 125 deletions
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index 6d0d51a60..302b81729 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -6,8 +6,10 @@
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
-#include <upstream/jupiter.h>
-#include <zenhttp/auth/authmgr.h>
+#include <zenhttp/httpclientauth.h>
+
+#include <zenutil/jupiter/jupiterclient.h>
+#include <zenutil/jupiter/jupitersession.h>
namespace zen {
@@ -18,7 +20,7 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv;
class BuildsRemoteStore : public RemoteProjectStore
{
public:
- BuildsRemoteStore(Ref<CloudCacheClient>&& CloudClient,
+ BuildsRemoteStore(Ref<JupiterClient>&& InJupiterClient,
std::string_view Namespace,
std::string_view Bucket,
const Oid& BuildId,
@@ -26,7 +28,7 @@ public:
bool ForceDisableBlocks,
bool ForceDisableTempBlocks,
const std::filesystem::path& TempFilePath)
- : m_CloudClient(std::move(CloudClient))
+ : m_JupiterClient(std::move(InJupiterClient))
, m_Namespace(Namespace)
, m_Bucket(Bucket)
, m_BuildId(BuildId)
@@ -50,7 +52,7 @@ public:
.UseTempBlockFiles = m_UseTempBlocks,
.AllowChunking = true,
.ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId),
- .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_CloudClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)};
+ .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)};
}
virtual Stats GetStats() const override
@@ -68,18 +70,18 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
IoBuffer Payload = m_MetaData;
Payload.SetContentType(ZenContentType::kCbObject);
- CloudCacheResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload);
+ JupiterResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload);
AddStats(PutResult);
CreateContainerResult Result{ConvertResult(PutResult)};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -92,7 +94,7 @@ public:
virtual SaveResult SaveContainer(const IoBuffer& Payload) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
PutBuildPartResult PutResult =
Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload);
AddStats(PutResult);
@@ -101,7 +103,7 @@ public:
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -115,9 +117,9 @@ public:
virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&& Block) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
- CloudCacheResult PutResult =
+ JupiterResult PutResult =
Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, ZenContentType::kCompressedBinary, Payload);
AddStats(PutResult);
@@ -125,7 +127,7 @@ public:
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -176,14 +178,14 @@ public:
IoBuffer MetaPayload = Writer.Save().GetBuffer().AsIoBuffer();
MetaPayload.SetContentType(ZenContentType::kCbObject);
- CloudCacheResult PutMetaResult =
+ JupiterResult PutMetaResult =
Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload);
AddStats(PutMetaResult);
RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult);
if (MetaDataResult.ErrorCode)
{
ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -215,7 +217,7 @@ public:
ZEN_UNUSED(RawHash);
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
FinalizeBuildPartResult FinalizeRefResult =
Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash);
AddStats(FinalizeRefResult);
@@ -224,7 +226,7 @@ public:
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -233,14 +235,14 @@ public:
}
else if (Result.Needs.empty())
{
- CloudCacheResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId);
+ JupiterResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId);
AddStats(FinalizeBuildResult);
FinalizeBuildResult.ElapsedSeconds += FinalizeRefResult.ElapsedSeconds;
Result = {ConvertResult(FinalizeBuildResult)};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -254,14 +256,14 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
AddStats(GetBuildResult);
LoadContainerResult Result{ConvertResult(GetBuildResult)};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed fetching oplog container build from {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -273,7 +275,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build {}/{}/{}/{} payload is not formatted as a compact binary object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId);
@@ -284,7 +286,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId);
@@ -295,7 +297,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -303,7 +305,7 @@ public:
return Result;
}
- CloudCacheResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
+ JupiterResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
AddStats(GetBuildPartResult);
Result = {ConvertResult(GetBuildResult)};
Result.ElapsedSeconds += GetBuildResult.ElapsedSeconds;
@@ -311,7 +313,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("Failed fetching oplog build part from {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -325,7 +327,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The build part for oplog container {}/{}/{}/{}/{} is not formatted as a compact binary object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -339,15 +341,15 @@ public:
virtual GetKnownBlocksResult GetKnownBlocks() override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
AddStats(FindResult);
GetKnownBlocksResult Result{ConvertResult(FindResult)};
if (Result.ErrorCode)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -360,7 +362,7 @@ public:
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -392,15 +394,15 @@ public:
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath);
AddStats(GetResult);
LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
if (GetResult.ErrorCode)
{
Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}&{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
@@ -429,7 +431,7 @@ public:
}
private:
- void AddStats(const CloudCacheResult& Result)
+ void AddStats(const JupiterResult& Result)
{
m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes));
m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes));
@@ -445,7 +447,7 @@ private:
m_RequestCount.fetch_add(1);
}
- static Result ConvertResult(const CloudCacheResult& Response)
+ static Result ConvertResult(const JupiterResult& Response)
{
std::string Text;
int32_t ErrorCode = 0;
@@ -482,7 +484,7 @@ private:
return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text};
}
- Ref<CloudCacheClient> m_CloudClient;
+ Ref<JupiterClient> m_JupiterClient;
const std::string m_Namespace;
const std::string m_Bucket;
const Oid m_BuildId;
@@ -510,44 +512,35 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
// Assume https URL
Url = fmt::format("https://{}"sv, Url);
}
- CloudCacheClientOptions ClientOptions{.Name = "Remote store"sv,
- .ServiceUrl = Url,
- .ConnectTimeout = std::chrono::milliseconds(2000),
- .Timeout = std::chrono::milliseconds(1800000),
- .AssumeHttp2 = Options.AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 4};
+ JupiterClientOptions ClientOptions{.Name = "Remote store"sv,
+ .ServiceUrl = Url,
+ .ConnectTimeout = std::chrono::milliseconds(2000),
+ .Timeout = std::chrono::milliseconds(1800000),
+ .AssumeHttp2 = Options.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 4};
// 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
// 2) Access token as parameter in request
// 3) Environment variable (different win vs linux/mac)
// 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
- std::unique_ptr<CloudCacheTokenProvider> TokenProvider;
+ std::function<HttpClientAccessToken()> TokenProvider;
if (!Options.OpenIdProvider.empty())
{
- TokenProvider =
- CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() {
- AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider);
- return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
- });
+ TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider);
}
else if (!Options.AccessToken.empty())
{
- TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = "Bearer " + Options.AccessToken]() {
- return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()};
- });
+ TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
}
else
{
- TokenProvider = CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager]() {
- AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken("Default");
- return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
- });
+ TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
}
- Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider)));
+ Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider)));
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(CloudClient),
+ std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(Client),
Options.Namespace,
Options.Bucket,
Options.BuildId,
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index eb6407e1f..0b8e5f13b 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -1061,12 +1061,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
bool IsValid = true;
std::vector<IoHash> MissingChunks;
- std::vector<IoHash> ReferencedChunks;
CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
- // We want to add all chunks here so we can properly clear them from the 'prep' call where we retained them earlier
- ReferencedChunks.push_back(Hash);
-
if (m_CidStore.ContainsChunk(Hash))
{
// Return null attachment as we already have it, no point in reading it and storing it again
@@ -1155,6 +1151,9 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
}
+ std::vector<IoHash> ReferencedChunks;
+ Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); });
+
// Write core to oplog
size_t AttachmentCount = Package.GetAttachments().size();
@@ -1168,7 +1167,10 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
m_ProjectStats.ChunkWriteCount += AttachmentCount;
// Once we stored the op, we no longer need to retain any chunks this op references
- FoundLog->RemovePendingChunkReferences(ReferencedChunks);
+ if (!ReferencedChunks.empty())
+ {
+ FoundLog->RemovePendingChunkReferences(ReferencedChunks);
+ }
m_ProjectStats.OpWriteCount++;
ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString());
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 13810bd66..8e74c57a5 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -3,13 +3,13 @@
#pragma once
#include <zencore/stats.h>
-#include <zenhttp/auth/authmgr.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
#include <zenstore/cidstore.h>
namespace zen {
+class AuthMgr;
class ProjectStore;
//////////////////////////////////////////////////////////////////////////
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index f4fe578ff..e906127ff 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -5,8 +5,10 @@
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
-#include <upstream/jupiter.h>
-#include <zenhttp/auth/authmgr.h>
+#include <zenhttp/httpclientauth.h>
+
+#include <zenutil/jupiter/jupiterclient.h>
+#include <zenutil/jupiter/jupitersession.h>
namespace zen {
@@ -15,7 +17,7 @@ using namespace std::literals;
class JupiterRemoteStore : public RemoteProjectStore
{
public:
- JupiterRemoteStore(Ref<CloudCacheClient>&& CloudClient,
+ JupiterRemoteStore(Ref<JupiterClient>&& InJupiterClient,
std::string_view Namespace,
std::string_view Bucket,
const IoHash& Key,
@@ -23,7 +25,7 @@ public:
bool ForceDisableBlocks,
bool ForceDisableTempBlocks,
const std::filesystem::path& TempFilePath)
- : m_CloudClient(std::move(CloudClient))
+ : m_JupiterClient(std::move(InJupiterClient))
, m_Namespace(Namespace)
, m_Bucket(Bucket)
, m_Key(Key)
@@ -47,7 +49,7 @@ public:
.AllowChunking = true,
.ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_Key),
.Description = fmt::format("[cloud] {} as {}/{}/{}{}"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_Key,
@@ -73,15 +75,15 @@ public:
virtual SaveResult SaveContainer(const IoBuffer& Payload) override
{
- CloudCacheSession Session(m_CloudClient.Get());
- PutRefResult PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ PutRefResult PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
AddStats(PutResult);
SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed saving oplog container to {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_Key,
@@ -92,15 +94,15 @@ public:
virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override
{
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
AddStats(PutResult);
SaveAttachmentResult Result{ConvertResult(PutResult)};
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
RawHash,
Result.Reason);
@@ -125,7 +127,7 @@ public:
virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override
{
- CloudCacheSession Session(m_CloudClient.Get());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
FinalizeRefResult FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
AddStats(FinalizeRefResult);
@@ -133,7 +135,7 @@ public:
if (Result.ErrorCode)
{
Result.Reason = fmt::format("Failed finalizing oplog container to {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_Key,
@@ -162,8 +164,8 @@ public:
{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds}};
}
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheExistsResult ExistsResult =
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterExistsResult ExistsResult =
Session.CompressedBlobExists(m_Namespace, std::set<IoHash>(BlockHashes.begin(), BlockHashes.end()));
AddStats(ExistsResult);
@@ -172,7 +174,7 @@ public:
return GetKnownBlocksResult{{.ErrorCode = ExistsResult.ErrorCode,
.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds,
.Reason = fmt::format("Failed checking attachment existance in {}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
ExistsResult.Reason)}};
}
@@ -201,15 +203,15 @@ public:
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
{
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
AddStats(GetResult);
LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
if (GetResult.ErrorCode)
{
Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
RawHash,
Result.Reason);
@@ -237,14 +239,14 @@ public:
private:
LoadContainerResult LoadContainer(const IoHash& Key)
{
- CloudCacheSession Session(m_CloudClient.Get());
- CloudCacheResult GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterResult GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
AddStats(GetResult);
if (GetResult.ErrorCode || !GetResult.Success)
{
LoadContainerResult Result{ConvertResult(GetResult)};
Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'",
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
Key,
@@ -259,7 +261,7 @@ private:
RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
.ElapsedSeconds = GetResult.ElapsedSeconds,
.Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv,
- m_CloudClient->ServiceUrl(),
+ m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
Key)},
@@ -268,7 +270,7 @@ private:
return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
}
- void AddStats(const CloudCacheResult& Result)
+ void AddStats(const JupiterResult& Result)
{
m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes));
m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes));
@@ -284,7 +286,7 @@ private:
m_RequestCount.fetch_add(1);
}
- static Result ConvertResult(const CloudCacheResult& Response)
+ static Result ConvertResult(const JupiterResult& Response)
{
std::string Text;
int32_t ErrorCode = 0;
@@ -321,7 +323,7 @@ private:
return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text};
}
- Ref<CloudCacheClient> m_CloudClient;
+ Ref<JupiterClient> m_JupiterClient;
const std::string m_Namespace;
const std::string m_Bucket;
const IoHash m_Key;
@@ -348,44 +350,35 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
// Assume https URL
Url = fmt::format("https://{}"sv, Url);
}
- CloudCacheClientOptions ClientOptions{.Name = "Remote store"sv,
- .ServiceUrl = Url,
- .ConnectTimeout = std::chrono::milliseconds(2000),
- .Timeout = std::chrono::milliseconds(1800000),
- .AssumeHttp2 = Options.AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 4};
+ JupiterClientOptions ClientOptions{.Name = "Remote store"sv,
+ .ServiceUrl = Url,
+ .ConnectTimeout = std::chrono::milliseconds(2000),
+ .Timeout = std::chrono::milliseconds(1800000),
+ .AssumeHttp2 = Options.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 4};
// 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
// 2) Access token as parameter in request
// 3) Environment variable (different win vs linux/mac)
// 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
- std::unique_ptr<CloudCacheTokenProvider> TokenProvider;
+ std::function<HttpClientAccessToken()> TokenProvider;
if (!Options.OpenIdProvider.empty())
{
- TokenProvider =
- CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() {
- AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider);
- return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
- });
+ TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider);
}
else if (!Options.AccessToken.empty())
{
- TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = "Bearer " + Options.AccessToken]() {
- return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()};
- });
+ TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
}
else
{
- TokenProvider = CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager]() {
- AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken("Default");
- return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
- });
+ TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
}
- Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider)));
+ Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider)));
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(std::move(CloudClient),
+ std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(std::move(Client),
Options.Namespace,
Options.Bucket,
Options.Key,
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index d39d78cb9..46a236af9 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -16,12 +16,12 @@
#include <zencore/stream.h>
#include <zencore/timer.h>
#include <zencore/trace.h>
+#include <zenhttp/packageformat.h>
#include <zenstore/caslog.h>
#include <zenstore/cidstore.h>
#include <zenstore/scrubcontext.h>
#include <zenutil/cache/rpcrecording.h>
#include <zenutil/openprocesscache.h>
-#include <zenutil/packageformat.h>
#include <zenutil/referencemetadata.h>
#include <zenutil/workerpools.h>
@@ -3506,7 +3506,7 @@ ProjectStore::Project::ScanForOplogs() const
if (Project::Exists(m_OplogStoragePath))
{
DirectoryContent DirContent;
- GetDirectoryContent(m_OplogStoragePath, DirectoryContent::IncludeDirsFlag, DirContent);
+ GetDirectoryContent(m_OplogStoragePath, DirectoryContentFlags::IncludeDirs, DirContent);
Oplogs.reserve(DirContent.Directories.size());
for (const std::filesystem::path& DirPath : DirContent.Directories)
{
@@ -3859,7 +3859,7 @@ ProjectStore::DiscoverProjects()
}
DirectoryContent DirContent;
- GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+ GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent);
for (const std::filesystem::path& DirPath : DirContent.Directories)
{
@@ -3965,7 +3965,7 @@ ProjectStore::StorageSize() const
if (std::filesystem::exists(m_ProjectBasePath))
{
DirectoryContent ProjectsFolderContent;
- GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, ProjectsFolderContent);
+ GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent);
for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories)
{
@@ -3974,7 +3974,7 @@ ProjectStore::StorageSize() const
{
Result.DiskSize += Project::TotalSize(ProjectBasePath);
DirectoryContent DirContent;
- GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+ GetDirectoryContent(ProjectBasePath, DirectoryContentFlags::IncludeDirs, DirContent);
for (const std::filesystem::path& OplogBasePath : DirContent.Directories)
{
Result.DiskSize += Oplog::TotalSize(OplogBasePath);
@@ -5433,7 +5433,11 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Project->TouchProject();
- ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true);
+ std::string_view Method = Cb["method"sv].AsString();
+
+ bool VerifyPathOnDisk = Method != "getchunks"sv;
+
+ ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, VerifyPathOnDisk);
if (!Oplog)
{
HttpReq.WriteResponse(HttpResponseCode::NotFound,
@@ -5443,8 +5447,6 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Project->TouchOplog(OplogId);
- std::string_view Method = Cb["method"sv].AsString();
-
if (Method == "import"sv)
{
if (!AreDiskWritesAllowed())
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 216b1c4dd..0589fdc5f 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -266,7 +266,10 @@ namespace remotestore_impl {
AppendBatch();
}
- ReportProgress(OptionalContext, "Writing oplog"sv, ""sv, OpCount, 0);
+ if (OpCount > 0)
+ {
+ ReportProgress(OptionalContext, "Writing oplog"sv, ""sv, OpCount, 0);
+ }
return RemoteProjectStore::Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0};
}
@@ -1685,7 +1688,7 @@ BuildContainer(CidStore& ChunkStore,
IoBuffer RawData = IoBufferBuilder::MakeFromFile(FilePath);
if (RawData)
{
- if (RawData.GetSize() > ChunkFileSizeLimit)
+ if (AllowChunking && RawData.GetSize() > ChunkFileSizeLimit)
{
IoBufferFileReference FileRef;
(void)RawData.GetFileReference(FileRef);
diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp
index 566d0d4b2..42519b108 100644
--- a/src/zenserver/projectstore/zenremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp
@@ -8,7 +8,7 @@
#include <zencore/fmtutils.h>
#include <zencore/stream.h>
#include <zenhttp/httpclient.h>
-#include <zenutil/packageformat.h>
+#include <zenhttp/packageformat.h>
namespace zen {
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-11 11:43:54 +0000