From b9475bf300aff23029250f366d98215a22e4bc25 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Mon, 27 Sep 2021 11:31:14 -0700 Subject: Add Get\PutBlob & Get\PutObject --- zenserver/upstream/jupiter.cpp | 132 +++++++++++++++++++++++++++++++++++++++++ zenserver/upstream/jupiter.h | 4 ++ 2 files changed, 136 insertions(+) diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 6eaa6423b..85049075f 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -161,6 +161,37 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } +CloudCacheResult +CloudCacheSession::GetBlob(const IoHash& Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); + + cpr::Response Response = Session.Get(); + ZEN_DEBUG("GET {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; +} + CloudCacheResult CloudCacheSession::GetCompressedBlob(const IoHash& Key) { @@ -192,6 +223,37 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } +CloudCacheResult +CloudCacheSession::GetObject(const IoHash& Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); + + cpr::Response Response = Session.Get(); + ZEN_DEBUG("GET {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; +} + CloudCacheResult CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Key, IoBuffer DerivedData) { @@ -278,6 +340,41 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer .Success = (Response.status_code == 200 || Response.status_code == 201)}; } +CloudCacheResult +CloudCacheSession::PutBlob(const IoHash& Key, IoBuffer Blob) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/octet-stream"}}); + Session.SetBody(cpr::Body{(const char*)Blob.Data(), Blob.Size()}); + + cpr::Response Response = Session.Put(); + ZEN_DEBUG("PUT {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.Bytes = Response.uploaded_bytes, + .ElapsedSeconds = Response.elapsed, + .Success = (Response.status_code == 200 || Response.status_code == 201)}; +} + CloudCacheResult CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) { @@ -313,6 +410,41 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) .Success = (Response.status_code == 200 || Response.status_code == 201)}; } +CloudCacheResult +CloudCacheSession::PutObject(const IoHash& Key, IoBuffer Object) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-cb"}}); + Session.SetBody(cpr::Body{(const char*)Object.Data(), Object.Size()}); + + cpr::Response Response = Session.Put(); + ZEN_DEBUG("PUT {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.Bytes = Response.uploaded_bytes, + .ElapsedSeconds = Response.elapsed, + .Success = (Response.status_code == 200 || Response.status_code == 201)}; +} + std::vector CloudCacheSession::Filter(std::string_view BucketId, const std::vector& ChunkHashes) { diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index 868a7b099..8e1a4ea7f 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -70,12 +70,16 @@ public: CloudCacheResult GetDerivedData(std::string_view BucketId, std::string_view Key); CloudCacheResult GetDerivedData(std::string_view BucketId, const IoHash& Key); CloudCacheResult GetRef(std::string_view BucketId, const IoHash& Key, ZenContentType RefType); + CloudCacheResult GetBlob(const IoHash& Key); CloudCacheResult GetCompressedBlob(const IoHash& Key); + CloudCacheResult GetObject(const IoHash& Key); CloudCacheResult PutDerivedData(std::string_view BucketId, std::string_view Key, IoBuffer DerivedData); CloudCacheResult PutDerivedData(std::string_view BucketId, const IoHash& Key, IoBuffer DerivedData); CloudCacheResult PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType); + CloudCacheResult PutBlob(const IoHash& Key, IoBuffer Blob); CloudCacheResult PutCompressedBlob(const IoHash& Key, IoBuffer Blob); + CloudCacheResult PutObject(const IoHash& Key, IoBuffer Object); std::vector Filter(std::string_view BucketId, const std::vector& ChunkHashes); -- cgit v1.2.3 From 5e755eacfc44d774249a6e5ec1d94f144394a66f Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Mon, 27 Sep 2021 11:32:52 -0700 Subject: Add Exists --- zenserver/upstream/jupiter.cpp | 164 +++++++++++++++++++++++++++++++++++++++++ zenserver/upstream/jupiter.h | 7 ++ 2 files changed, 171 insertions(+) diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 85049075f..949d45cf2 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -445,6 +445,170 @@ CloudCacheSession::PutObject(const IoHash& Key, IoBuffer Object) .Success = (Response.status_code == 200 || Response.status_code == 201)}; } +CloudCacheResult +CloudCacheSession::DerivedDataExists(std::string_view BucketId, std::string_view Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key; + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); + + cpr::Response Response = Session.Get(); + ZEN_DEBUG("HEAD {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; +} + +CloudCacheResult +CloudCacheSession::DerivedDataExists(std::string_view BucketId, const IoHash& Key) +{ + return DerivedDataExists(BucketId, Key.ToHexString()); +} + +CloudCacheResult +CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << m_CacheClient->BlobStoreNamespace() << "/" << BucketId << "/" + << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); + + cpr::Response Response = Session.Put(); + ZEN_DEBUG("HEAD {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; +} + +CloudCacheResult +CloudCacheSession::BlobExists(const IoHash& Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + + cpr::Response Response = Session.Put(); + ZEN_DEBUG("HEAD {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; +} + +CloudCacheResult +CloudCacheSession::CompressedBlobExists(const IoHash& Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + + cpr::Response Response = Session.Put(); + ZEN_DEBUG("HEAD {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; +} + +CloudCacheResult +CloudCacheSession::ObjectExists(const IoHash& Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + + cpr::Response Response = Session.Put(); + ZEN_DEBUG("HEAD {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; +} + std::vector CloudCacheSession::Filter(std::string_view BucketId, const std::vector& ChunkHashes) { diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index 8e1a4ea7f..d8844279e 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -81,6 +81,13 @@ public: CloudCacheResult PutCompressedBlob(const IoHash& Key, IoBuffer Blob); CloudCacheResult PutObject(const IoHash& Key, IoBuffer Object); + CloudCacheResult DerivedDataExists(std::string_view BucketId, std::string_view Key); + CloudCacheResult DerivedDataExists(std::string_view BucketId, const IoHash& Key); + CloudCacheResult RefExists(std::string_view BucketId, const IoHash& Key); + CloudCacheResult BlobExists(const IoHash& Key); + CloudCacheResult CompressedBlobExists(const IoHash& Key); + CloudCacheResult ObjectExists(const IoHash& Key); + std::vector Filter(std::string_view BucketId, const std::vector& ChunkHashes); private: -- cgit v1.2.3 From 7649b33a6ed22bbec7abd6daca8d13dc14eb5994 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Mon, 27 Sep 2021 15:18:16 -0700 Subject: Add .Exists to CloudCacheResult --- zenserver/upstream/jupiter.cpp | 20 +++++++++++++++----- zenserver/upstream/jupiter.h | 1 + 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 949d45cf2..e15dd53bc 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -474,7 +474,9 @@ CloudCacheSession::DerivedDataExists(std::string_view BucketId, std::string_view return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; + return {.ElapsedSeconds = Response.elapsed, + .Success = (Response.status_code == 200 || Response.status_code == 404), + .Exists = Response.status_code == 200}; } CloudCacheResult @@ -513,7 +515,9 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; + return {.ElapsedSeconds = Response.elapsed, + .Success = (Response.status_code == 200 || Response.status_code == 404), + .Exists = Response.status_code == 200}; } CloudCacheResult @@ -544,7 +548,9 @@ CloudCacheSession::BlobExists(const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; + return {.ElapsedSeconds = Response.elapsed, + .Success = (Response.status_code == 200 || Response.status_code == 404), + .Exists = Response.status_code == 200}; } CloudCacheResult @@ -575,7 +581,9 @@ CloudCacheSession::CompressedBlobExists(const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; + return {.ElapsedSeconds = Response.elapsed, + .Success = (Response.status_code == 200 || Response.status_code == 404), + .Exists = Response.status_code == 200}; } CloudCacheResult @@ -606,7 +614,9 @@ CloudCacheSession::ObjectExists(const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, .Success = (Response.status_code == 200 || Response.status_code == 404)}; + return {.ElapsedSeconds = Response.elapsed, + .Success = (Response.status_code == 200 || Response.status_code == 404), + .Exists = Response.status_code == 200}; } std::vector diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index d8844279e..f05e059bd 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -51,6 +51,7 @@ struct CloudCacheResult int32_t ErrorCode = {}; std::string Reason; bool Success = false; + bool Exists = false; }; /** -- cgit v1.2.3 From 0168cd2cd37e6e74d18faf3c526a6054d5d6c3ec Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Tue, 28 Sep 2021 10:54:35 -0700 Subject: Remove DerivedDataExists & Exists prop --- zenserver/upstream/jupiter.cpp | 56 +++--------------------------------------- 1 file changed, 4 insertions(+), 52 deletions(-) diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index e15dd53bc..b93635e76 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -445,46 +445,6 @@ CloudCacheSession::PutObject(const IoHash& Key, IoBuffer Object) .Success = (Response.status_code == 200 || Response.status_code == 201)}; } -CloudCacheResult -CloudCacheSession::DerivedDataExists(std::string_view BucketId, std::string_view Key) -{ - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - if (!AccessToken.IsValid()) - { - return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; - } - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key; - - cpr::Session& Session = m_SessionState->Session; - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); - - cpr::Response Response = Session.Get(); - ZEN_DEBUG("HEAD {}", Response); - - if (Response.error) - { - return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; - } - else if (!VerifyAccessToken(Response.status_code)) - { - return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; - } - - return {.ElapsedSeconds = Response.elapsed, - .Success = (Response.status_code == 200 || Response.status_code == 404), - .Exists = Response.status_code == 200}; -} - -CloudCacheResult -CloudCacheSession::DerivedDataExists(std::string_view BucketId, const IoHash& Key) -{ - return DerivedDataExists(BucketId, Key.ToHexString()); -} - CloudCacheResult CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) { @@ -515,9 +475,7 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, - .Success = (Response.status_code == 200 || Response.status_code == 404), - .Exists = Response.status_code == 200}; + return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } CloudCacheResult @@ -548,9 +506,7 @@ CloudCacheSession::BlobExists(const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, - .Success = (Response.status_code == 200 || Response.status_code == 404), - .Exists = Response.status_code == 200}; + return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } CloudCacheResult @@ -581,9 +537,7 @@ CloudCacheSession::CompressedBlobExists(const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, - .Success = (Response.status_code == 200 || Response.status_code == 404), - .Exists = Response.status_code == 200}; + return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } CloudCacheResult @@ -614,9 +568,7 @@ CloudCacheSession::ObjectExists(const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - return {.ElapsedSeconds = Response.elapsed, - .Success = (Response.status_code == 200 || Response.status_code == 404), - .Exists = Response.status_code == 200}; + return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } std::vector -- cgit v1.2.3