diff options
Diffstat (limited to 'src/zenserver/projectstore/jupiterremoteprojectstore.cpp')
| -rw-r--r-- | src/zenserver/projectstore/jupiterremoteprojectstore.cpp | 90 |
1 files changed, 65 insertions, 25 deletions
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index d926499c4..f4fe578ff 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -46,7 +46,6 @@ public: .UseTempBlockFiles = m_UseTempBlocks, .AllowChunking = true, .ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_Key), - .BaseContainerName = m_OptionalBaseKey == IoHash::Zero ? "" : fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_Key), .Description = fmt::format("[cloud] {} as {}/{}/{}{}"sv, m_CloudClient->ServiceUrl(), m_Namespace, @@ -66,6 +65,12 @@ public: .m_PeakBytesPerSec = m_PeakBytesPerSec.load()}; } + virtual CreateContainerResult CreateContainer() override + { + // Nothing to do here + return {}; + } + virtual SaveResult SaveContainer(const IoBuffer& Payload) override { CloudCacheSession Session(m_CloudClient.Get()); @@ -85,7 +90,7 @@ public: return Result; } - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override + virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override { CloudCacheSession Session(m_CloudClient.Get()); CloudCacheResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload); @@ -109,7 +114,7 @@ public: for (const SharedBuffer& Chunk : Chunks) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer()); - SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash()); + SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash(), {}); if (ChunkResult.ErrorCode) { return SaveAttachmentsResult{ChunkResult}; @@ -139,31 +144,58 @@ public: virtual LoadContainerResult LoadContainer() override { return LoadContainer(m_Key); } - virtual LoadContainerResult LoadBaseContainer() override + virtual GetKnownBlocksResult GetKnownBlocks() override { if (m_OptionalBaseKey == IoHash::Zero) { - return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}}; + return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}}; + } + LoadContainerResult LoadResult = LoadContainer(m_OptionalBaseKey); + if (LoadResult.ErrorCode) + { + return GetKnownBlocksResult{LoadResult}; + } + std::vector<IoHash> BlockHashes = GetBlockHashesFromOplog(LoadResult.ContainerObject); + if (BlockHashes.empty()) + { + return GetKnownBlocksResult{ + {.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds}}; } - return LoadContainer(m_OptionalBaseKey); - } - virtual HasAttachmentsResult HasAttachments(const std::span<IoHash> RawHashes) override - { CloudCacheSession Session(m_CloudClient.Get()); CloudCacheExistsResult ExistsResult = - Session.CompressedBlobExists(m_Namespace, std::set<IoHash>(RawHashes.begin(), RawHashes.end())); + Session.CompressedBlobExists(m_Namespace, std::set<IoHash>(BlockHashes.begin(), BlockHashes.end())); AddStats(ExistsResult); - HasAttachmentsResult Result{ConvertResult(ExistsResult), - std::unordered_set<IoHash, IoHash::Hasher>(ExistsResult.Needs.begin(), ExistsResult.Needs.end())}; if (ExistsResult.ErrorCode) { - Result.Reason = fmt::format("Failed checking attachment existance in {}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), - m_Namespace, - Result.Reason); + return GetKnownBlocksResult{{.ErrorCode = ExistsResult.ErrorCode, + .ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds, + .Reason = fmt::format("Failed checking attachment existance in {}/{}. Reason: '{}'", + m_CloudClient->ServiceUrl(), + m_Namespace, + ExistsResult.Reason)}}; + } + + Stopwatch Timer; + std::vector<IoHash> ExistingBlockHashes; + for (const IoHash& RawHash : BlockHashes) + { + if (!ExistsResult.Needs.contains(RawHash)) + { + ExistingBlockHashes.push_back(RawHash); + } } + if (ExistingBlockHashes.empty()) + { + return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), + .ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds}}; + } + std::vector<RemoteProjectStore::Block> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); + + GetKnownBlocksResult Result{ + {.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000.0}}; + Result.Blocks = std::move(KnownBlocks); return Result; } @@ -323,12 +355,21 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi .AssumeHttp2 = Options.AssumeHttp2, .AllowResume = true, .RetryCount = 4}; - // 1) Access token as parameter in request - // 2) Environment variable (different win vs linux/mac) - // 3) openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider + // 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider + // 2) Access token as parameter in request + // 3) Environment variable (different win vs linux/mac) + // 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider std::unique_ptr<CloudCacheTokenProvider> TokenProvider; - if (!Options.AccessToken.empty()) + if (!Options.OpenIdProvider.empty()) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() { + AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider); + return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; + }); + } + else if (!Options.AccessToken.empty()) { TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = "Bearer " + Options.AccessToken]() { return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()}; @@ -336,11 +377,10 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi } else { - TokenProvider = - CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() { - AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider); - return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; - }); + TokenProvider = CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager]() { + AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken("Default"); + return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; + }); } Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider))); |