// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC # include # include # include #endif static std::atomic HttpClientRequestIdCounter{0}; namespace zen { using namespace std::literals; ////////////////////////////////////////////////////////////////////////// // // CPR helpers static cpr::Body AsCprBody(const CbObject& Obj) { return cpr::Body((const char*)Obj.GetBuffer().GetData(), Obj.GetBuffer().GetSize()); } static cpr::Body AsCprBody(const IoBuffer& Obj) { return cpr::Body((const char*)Obj.GetData(), Obj.GetSize()); } ////////////////////////////////////////////////////////////////////////// static HttpClient::Response ResponseWithPayload(cpr::Response& HttpResponse, const HttpResponseCode WorkResponseCode, IoBuffer&& Payload) { // This ends up doing a memcpy, would be good to get rid of it by streaming results // into buffer directly IoBuffer ResponseBuffer = Payload ? std::move(Payload) : IoBuffer(IoBuffer::Clone, HttpResponse.text.data(), HttpResponse.text.size()); if (auto It = HttpResponse.header.find("Content-Type"); It != HttpResponse.header.end()) { const HttpContentType ContentType = ParseContentType(It->second); ResponseBuffer.SetContentType(ContentType); } if (!IsHttpSuccessCode(WorkResponseCode) && WorkResponseCode != HttpResponseCode::NotFound) { ZEN_WARN("HttpClient request failed: {}", HttpResponse); } return HttpClient::Response{.StatusCode = WorkResponseCode, .ResponsePayload = std::move(ResponseBuffer), .Header = HttpClient::KeyValueMap(HttpResponse.header.begin(), HttpResponse.header.end()), .UploadedBytes = gsl::narrow(HttpResponse.uploaded_bytes), .DownloadedBytes = gsl::narrow(HttpResponse.downloaded_bytes), .ElapsedSeconds = HttpResponse.elapsed}; } static HttpClient::Response CommonResponse(cpr::Response&& HttpResponse, IoBuffer&& Payload = {}) { const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code); if (HttpResponse.error) { ZEN_WARN("HttpClient client error: {}", HttpResponse); // Client side failure code return HttpClient::Response{ .StatusCode = WorkResponseCode, .ResponsePayload = IoBufferBuilder::MakeCloneFromMemory(HttpResponse.text.data(), HttpResponse.text.size()), .Header = HttpClient::KeyValueMap(HttpResponse.header.begin(), HttpResponse.header.end()), .UploadedBytes = gsl::narrow(HttpResponse.uploaded_bytes), .DownloadedBytes = gsl::narrow(HttpResponse.downloaded_bytes), .ElapsedSeconds = HttpResponse.elapsed, .Error = HttpClient::ErrorContext{.ErrorCode = gsl::narrow(HttpResponse.error.code), .ErrorMessage = HttpResponse.error.message}}; } if (WorkResponseCode == HttpResponseCode::NoContent || (HttpResponse.text.empty() && !Payload)) { return HttpClient::Response{.StatusCode = WorkResponseCode, .Header = HttpClient::KeyValueMap(HttpResponse.header.begin(), HttpResponse.header.end()), .UploadedBytes = gsl::narrow(HttpResponse.uploaded_bytes), .DownloadedBytes = gsl::narrow(HttpResponse.downloaded_bytes), .ElapsedSeconds = HttpResponse.elapsed}; } else { return ResponseWithPayload(HttpResponse, WorkResponseCode, std::move(Payload)); } } static bool ShouldRetry(const cpr::Response& Response) { switch (Response.error.code) { case cpr::ErrorCode::OK: break; case cpr::ErrorCode::OPERATION_TIMEDOUT: case cpr::ErrorCode::NETWORK_RECEIVE_ERROR: case cpr::ErrorCode::NETWORK_SEND_FAILURE: return true; default: return false; } switch ((HttpResponseCode)Response.status_code) { case HttpResponseCode::RequestTimeout: case HttpResponseCode::TooManyRequests: case HttpResponseCode::InternalServerError: case HttpResponseCode::ServiceUnavailable: case HttpResponseCode::GatewayTimeout: return true; default: return false; } }; static cpr::Response DoWithRetry(std::function&& Func, uint8_t RetryCount) { uint8_t Attempt = 0; cpr::Response Result = Func(); while (Attempt < RetryCount && ShouldRetry(Result)) { Sleep(100 * (Attempt + 1)); Attempt++; ZEN_INFO("{} Attempt {}/{}", CommonResponse(std::move(Result)).ErrorMessage("Retry"), Attempt, RetryCount + 1); Result = Func(); } return Result; } static std::pair HeaderContentType(ZenContentType ContentType) { return std::make_pair("Content-Type", std::string(MapContentTypeToString(ContentType))); } ////////////////////////////////////////////////////////////////////////// struct HttpClient::Impl : public RefCounted { Impl(LoggerRef Log); ~Impl(); // Session allocation struct Session { Session(Impl* InOuter, cpr::Session* InSession) : Outer(InOuter), CprSession(InSession) {} ~Session() { Outer->ReleaseSession(CprSession); } inline cpr::Session* operator->() const { return CprSession; } inline cpr::Response Get() { cpr::Response Result = CprSession->Get(); ZEN_TRACE("GET {}", Result); return Result; } inline cpr::Response Download(cpr::WriteCallback&& Write, std::optional&& Header = {}) { if (Header) { CprSession->SetHeaderCallback(std::move(Header.value())); } cpr::Response Result = CprSession->Download(Write); ZEN_TRACE("GET {}", Result); CprSession->SetHeaderCallback({}); CprSession->SetWriteCallback({}); return Result; } inline cpr::Response Head() { cpr::Response Result = CprSession->Head(); ZEN_TRACE("HEAD {}", Result); return Result; } inline cpr::Response Put(std::optional&& Read = {}) { if (Read) { CprSession->SetReadCallback(std::move(Read.value())); } cpr::Response Result = CprSession->Put(); ZEN_TRACE("PUT {}", Result); CprSession->SetReadCallback({}); return Result; } inline cpr::Response Post(std::optional&& Read = {}) { if (Read) { CprSession->SetReadCallback(std::move(Read.value())); } cpr::Response Result = CprSession->Post(); ZEN_TRACE("POST {}", Result); CprSession->SetReadCallback({}); return Result; } inline cpr::Response Delete() { cpr::Response Result = CprSession->Delete(); ZEN_TRACE("DELETE {}", Result); return Result; } LoggerRef Logger() { return Outer->Logger(); } private: Impl* Outer; cpr::Session* CprSession; Session(Session&&) = delete; Session& operator=(Session&&) = delete; }; Session AllocSession(const std::string_view BaseUrl, const std::string_view Url, const HttpClientSettings& ConnectionSettings, const KeyValueMap& AdditionalHeader, const KeyValueMap& Parameters, const std::string_view SessionId, std::optional AccessToken); LoggerRef Logger() { return m_Log; } private: LoggerRef m_Log; RwLock m_SessionLock; std::vector m_Sessions; void ReleaseSession(cpr::Session*); }; HttpClient::Impl::Impl(LoggerRef Log) : m_Log(Log) { } HttpClient::Impl::~Impl() { m_SessionLock.WithExclusiveLock([&] { for (auto CprSession : m_Sessions) { delete CprSession; } m_Sessions.clear(); }); } HttpClient::Impl::Session HttpClient::Impl::AllocSession(const std::string_view BaseUrl, const std::string_view ResourcePath, const HttpClientSettings& ConnectionSettings, const KeyValueMap& AdditionalHeader, const KeyValueMap& Parameters, const std::string_view SessionId, std::optional AccessToken) { cpr::Session* CprSession = nullptr; m_SessionLock.WithExclusiveLock([&] { if (!m_Sessions.empty()) { CprSession = m_Sessions.back(); m_Sessions.pop_back(); } }); if (CprSession == nullptr) { CprSession = new cpr::Session(); CprSession->SetConnectTimeout(ConnectionSettings.ConnectTimeout); CprSession->SetTimeout(ConnectionSettings.Timeout); if (ConnectionSettings.AssumeHttp2) { CprSession->SetHttpVersion(cpr::HttpVersion{cpr::HttpVersionCode::VERSION_2_0_PRIOR_KNOWLEDGE}); } } if (!AdditionalHeader->empty()) { CprSession->SetHeader(cpr::Header(AdditionalHeader->begin(), AdditionalHeader->end())); } if (!SessionId.empty()) { CprSession->UpdateHeader({{"UE-Session", std::string(SessionId)}}); } if (AccessToken) { CprSession->UpdateHeader({{"Authorization", AccessToken->Value}}); } if (!Parameters->empty()) { cpr::Parameters Tmp; for (auto It = Parameters->begin(); It != Parameters->end(); It++) { Tmp.Add({It->first, It->second}); } CprSession->SetParameters(Tmp); } else { CprSession->SetParameters({}); } ExtendableStringBuilder<128> UrlBuffer; UrlBuffer << BaseUrl << ResourcePath; CprSession->SetUrl(UrlBuffer.c_str()); return Session(this, CprSession); } void HttpClient::Impl::ReleaseSession(cpr::Session* CprSession) { CprSession->SetUrl({}); CprSession->SetHeader({}); CprSession->SetBody({}); m_SessionLock.WithExclusiveLock([&] { m_Sessions.push_back(CprSession); }); } namespace detail { static std::atomic_uint32_t TempFileBaseIndex; } // namespace detail class TempPayloadFile { public: TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {} ~TempPayloadFile() { try { if (m_FileHandle) { #if ZEN_PLATFORM_WINDOWS // Mark file for deletion when final handle is closed FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); BOOL Success = CloseHandle(m_FileHandle); #else std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); unlink(FilePath.c_str()); int Fd = int(uintptr_t(m_FileHandle)); bool Success = (close(Fd) == 0); #endif if (!Success) { ZEN_WARN("Error reported on file handle close, reason '{}'", GetLastErrorAsString()); } m_FileHandle = nullptr; } } catch (const std::exception& Ex) { ZEN_ERROR("Failed deleting temp file {}. Reason '{}'", m_FileHandle, Ex.what()); } } std::error_code Open(const std::filesystem::path& TempFolderPath) { ZEN_ASSERT(m_FileHandle == nullptr); std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) | detail::TempFileBaseIndex.fetch_add(1); std::filesystem::path FileName = TempFolderPath / fmt::to_string(TmpIndex); #if ZEN_PLATFORM_WINDOWS LPCWSTR lpFileName = FileName.c_str(); const DWORD dwDesiredAccess = (GENERIC_READ | GENERIC_WRITE | DELETE); const DWORD dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; LPSECURITY_ATTRIBUTES lpSecurityAttributes = nullptr; const DWORD dwCreationDisposition = CREATE_ALWAYS; const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; const HANDLE hTemplateFile = nullptr; const HANDLE FileHandle = CreateFile(lpFileName, dwDesiredAccess, dwShareMode, lpSecurityAttributes, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile); if (FileHandle == INVALID_HANDLE_VALUE) { return MakeErrorCodeFromLastError(); } #else // ZEN_PLATFORM_WINDOWS int OpenFlags = O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC; int Fd = open(FileName.c_str(), OpenFlags, 0666); if (Fd < 0) { return MakeErrorCodeFromLastError(); } fchmod(Fd, 0666); void* FileHandle = (void*)(uintptr_t(Fd)); #endif // ZEN_PLATFORM_WINDOWS m_FileHandle = FileHandle; return {}; } std::error_code Write(std::string_view DataString) { const uint8_t* DataPtr = (const uint8_t*)DataString.data(); size_t DataSize = DataString.size(); if (DataSize >= CacheBufferSize) { std::error_code Ec = Flush(); if (Ec) { return Ec; } return AppendData(DataPtr, DataSize); } size_t CopySize = Min(DataSize, CacheBufferSize - m_CacheBufferOffset); memcpy(&m_CacheBuffer[m_CacheBufferOffset], DataPtr, CopySize); m_CacheBufferOffset += CopySize; DataSize -= CopySize; if (m_CacheBufferOffset == CacheBufferSize) { AppendData(m_CacheBuffer, CacheBufferSize); if (DataSize > 0) { ZEN_ASSERT(DataSize < CacheBufferSize); memcpy(m_CacheBuffer, DataPtr + CopySize, DataSize); } m_CacheBufferOffset = DataSize; } else { ZEN_ASSERT(DataSize == 0); } return {}; } IoBuffer DetachToIoBuffer() { if (std::error_code Ec = Flush(); Ec) { ThrowSystemError(Ec.value(), Ec.message()); } ZEN_ASSERT(m_FileHandle != nullptr); void* FileHandle = m_FileHandle; IoBuffer Buffer(IoBuffer::File, FileHandle, 0, m_WriteOffset, /*IsWholeFile*/ true); Buffer.SetDeleteOnClose(true); m_FileHandle = 0; m_WriteOffset = 0; return Buffer; } uint64_t GetSize() const { return m_WriteOffset; } void ResetWritePos(uint64_t WriteOffset) { Flush(); m_WriteOffset = WriteOffset; } private: std::error_code Flush() { if (m_CacheBufferOffset == 0) { return {}; } std::error_code Res = AppendData(m_CacheBuffer, m_CacheBufferOffset); m_CacheBufferOffset = 0; return Res; } std::error_code AppendData(const void* Data, uint64_t Size) { ZEN_ASSERT(m_FileHandle != nullptr); const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; while (Size) { const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); uint64_t NumberOfBytesWritten = 0; #if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(m_WriteOffset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(m_WriteOffset >> 32); DWORD dwNumberOfBytesWritten = 0; BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); if (Success) { NumberOfBytesWritten = static_cast(dwNumberOfBytesWritten); } #else static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); int Fd = int(uintptr_t(m_FileHandle)); int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, m_WriteOffset); bool Success = (BytesWritten > 0); if (Success) { NumberOfBytesWritten = static_cast(BytesWritten); } #endif if (!Success) { return MakeErrorCodeFromLastError(); } Size -= NumberOfBytesWritten; m_WriteOffset += NumberOfBytesWritten; Data = reinterpret_cast(Data) + NumberOfBytesWritten; } return {}; } void* m_FileHandle; std::uint64_t m_WriteOffset; static constexpr uint64_t CacheBufferSize = 512u * 1024u; uint8_t m_CacheBuffer[CacheBufferSize]; std::uint64_t m_CacheBufferOffset = 0; }; ////////////////////////////////////////////////////////////////////////// HttpClient::HttpClient(std::string_view BaseUri, const HttpClientSettings& Connectionsettings) : m_Log(zen::logging::Get(Connectionsettings.LogCategory)) , m_BaseUri(BaseUri) , m_ConnectionSettings(Connectionsettings) , m_Impl(new Impl(m_Log)) { m_SessionId = GetSessionIdString(); } HttpClient::~HttpClient() { } bool HttpClient::Authenticate() { std::optional Token = GetAccessToken(); if (!Token) { return false; } return Token->IsValid(); } const std::optional HttpClient::GetAccessToken() { if (!m_ConnectionSettings.AccessTokenProvider.has_value()) { return {}; } { RwLock::SharedLockScope _(m_AccessTokenLock); if (m_CachedAccessToken.IsValid()) { return m_CachedAccessToken; } } RwLock::ExclusiveLockScope _(m_AccessTokenLock); if (m_CachedAccessToken.IsValid()) { return m_CachedAccessToken; } m_CachedAccessToken = m_ConnectionSettings.AccessTokenProvider.value()(); return m_CachedAccessToken; } HttpClient::Response HttpClient::TransactPackage(std::string_view Url, CbPackage Package, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::TransactPackage"); Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); // First, list of offered chunks for filtering on the server end std::vector AttachmentsToSend; std::span Attachments = Package.GetAttachments(); const uint32_t RequestId = ++HttpClientRequestIdCounter; auto RequestIdString = fmt::to_string(RequestId); if (Attachments.empty() == false) { CbObjectWriter Writer; Writer.BeginArray("offer"); for (const CbAttachment& Attachment : Attachments) { Writer.AddHash(Attachment.GetHash()); } Writer.EndArray(); BinaryWriter MemWriter; Writer.Save(MemWriter); Sess->UpdateHeader({HeaderContentType(HttpContentType::kCbPackageOffer), {"UE-Request", RequestIdString}}); Sess->SetBody(cpr::Body{(const char*)MemWriter.Data(), MemWriter.Size()}); cpr::Response FilterResponse = Sess.Post(); if (FilterResponse.status_code == 200) { IoBuffer ResponseBuffer(IoBuffer::Wrap, FilterResponse.text.data(), FilterResponse.text.size()); CbObject ResponseObject = LoadCompactBinaryObject(ResponseBuffer); for (CbFieldView& Entry : ResponseObject["need"]) { ZEN_ASSERT(Entry.IsHash()); AttachmentsToSend.push_back(Entry.AsHash()); } } } // Prepare package for send CbPackage SendPackage; SendPackage.SetObject(Package.GetObject(), Package.GetObjectHash()); for (const IoHash& AttachmentCid : AttachmentsToSend) { const CbAttachment* Attachment = Package.FindAttachment(AttachmentCid); if (Attachment) { SendPackage.AddAttachment(*Attachment); } else { // This should be an error -- server asked to have something we can't find } } // Transmit package payload CompositeBuffer Message = FormatPackageMessageBuffer(SendPackage); SharedBuffer FlatMessage = Message.Flatten(); Sess->UpdateHeader({HeaderContentType(HttpContentType::kCbPackage), {"UE-Request", RequestIdString}}); Sess->SetBody(cpr::Body{(const char*)FlatMessage.GetData(), FlatMessage.GetSize()}); cpr::Response FilterResponse = Sess.Post(); if (!IsHttpSuccessCode(FilterResponse.status_code)) { return {.StatusCode = HttpResponseCode(FilterResponse.status_code)}; } IoBuffer ResponseBuffer(IoBuffer::Clone, FilterResponse.text.data(), FilterResponse.text.size()); if (auto It = FilterResponse.header.find("Content-Type"); It != FilterResponse.header.end()) { HttpContentType ContentType = ParseContentType(It->second); ResponseBuffer.SetContentType(ContentType); } return {.StatusCode = HttpResponseCode(FilterResponse.status_code), .ResponsePayload = ResponseBuffer}; } ////////////////////////////////////////////////////////////////////////// // // Standard HTTP verbs // HttpClient::Response HttpClient::Put(std::string_view Url, const IoBuffer& Payload, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Put"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->SetBody(AsCprBody(Payload)); Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())}); return Sess.Put(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Put(std::string_view Url, const KeyValueMap& Parameters) { ZEN_TRACE_CPU("HttpClient::Put"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, {{"Content-Length", "0"}}, Parameters, m_SessionId, GetAccessToken()); return Sess.Put(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Get(std::string_view Url, const KeyValueMap& AdditionalHeader, const KeyValueMap& Parameters) { ZEN_TRACE_CPU("HttpClient::Get"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken()); return Sess.Get(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Head(std::string_view Url, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Head"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); return Sess.Head(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Delete(std::string_view Url, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Delete"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); return Sess.Delete(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Post(std::string_view Url, const KeyValueMap& AdditionalHeader, const KeyValueMap& Parameters) { ZEN_TRACE_CPU("HttpClient::PostNoPayload"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken()); return Sess.Post(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Post(std::string_view Url, const IoBuffer& Payload, const KeyValueMap& AdditionalHeader) { return Post(Url, Payload, Payload.GetContentType(), AdditionalHeader); } HttpClient::Response HttpClient::Post(std::string_view Url, const IoBuffer& Payload, ZenContentType ContentType, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::PostWithPayload"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->SetBody(AsCprBody(Payload)); Sess->UpdateHeader({HeaderContentType(ContentType)}); return Sess.Post(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Post(std::string_view Url, CbObject Payload, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::PostObjectPayload"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->SetBody(AsCprBody(Payload)); Sess->UpdateHeader({HeaderContentType(ZenContentType::kCbObject)}); return Sess.Post(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Post(std::string_view Url, CbPackage Pkg, const KeyValueMap& AdditionalHeader) { return Post(Url, zen::FormatPackageMessageBuffer(Pkg), ZenContentType::kCbPackage, AdditionalHeader); } HttpClient::Response HttpClient::Post(std::string_view Url, const CompositeBuffer& Payload, ZenContentType ContentType, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Post"); return CommonResponse(DoWithRetry( [&]() { uint64_t SizeLeft = Payload.GetSize(); CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { size = Min(size, SizeLeft); MutableMemoryView Data(buffer, size); Payload.CopyTo(Data, BufferIt); SizeLeft -= size; return true; }; Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->UpdateHeader({HeaderContentType(ContentType)}); return Sess.Post(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Upload(std::string_view Url, const IoBuffer& Payload, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Upload"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())}); uint64_t Offset = 0; if (Payload.IsWholeFile()) { auto ReadCallback = [&Payload, &Offset](char* buffer, size_t& size, intptr_t) { size = Min(size, Payload.GetSize() - Offset); IoBuffer PayloadRange = IoBuffer(Payload, Offset, size); MutableMemoryView Data(buffer, size); Data.CopyFrom(PayloadRange.GetView()); Offset += size; return true; }; return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); } Sess->SetBody(AsCprBody(Payload)); return Sess.Put(); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Upload(std::string_view Url, const CompositeBuffer& Payload, ZenContentType ContentType, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Upload"); return CommonResponse(DoWithRetry( [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->UpdateHeader({HeaderContentType(ContentType)}); uint64_t SizeLeft = Payload.GetSize(); CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { size = Min(size, SizeLeft); MutableMemoryView Data(buffer, size); Payload.CopyTo(Data, BufferIt); SizeLeft -= size; return true; }; return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); }, m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFolderPath, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Download"); std::string PayloadString; std::unique_ptr PayloadFile; cpr::Response Response = DoWithRetry( [&]() { auto GetHeader = [&](std::string header) -> std::pair { size_t DelimiterPos = header.find(':'); if (DelimiterPos != std::string::npos) { std::string Key = header.substr(0, DelimiterPos); constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n"); Key = AsciiSet::TrimSuffixWith(Key, WhitespaceCharacters); Key = AsciiSet::TrimPrefixWith(Key, WhitespaceCharacters); std::string Value = header.substr(DelimiterPos + 1); Value = AsciiSet::TrimSuffixWith(Value, WhitespaceCharacters); Value = AsciiSet::TrimPrefixWith(Value, WhitespaceCharacters); return std::make_pair(Key, Value); } return std::make_pair(header, ""); }; auto DownloadCallback = [&](std::string data, intptr_t) { if (PayloadFile) { ZEN_ASSERT(PayloadString.empty()); std::error_code Ec = PayloadFile->Write(data); if (Ec) { ZEN_WARN("Failed to write to temp file in '{}' for HttpClient::Download. Reason: {}", TempFolderPath.string(), Ec.message()); return false; } } else { PayloadString.append(data); } return true; }; cpr::Response Response; { std::vector> ReceivedHeaders; auto HeaderCallback = [&](std::string header, intptr_t) { std::pair Header = GetHeader(header); if (Header.first == "Content-Length"sv) { std::optional ContentSize = ParseInt(Header.second); if (ContentSize.has_value()) { if (ContentSize.value() > 1024 * 1024) { PayloadFile = std::make_unique(); std::error_code Ec = PayloadFile->Open(TempFolderPath); if (Ec) { ZEN_WARN("Failed to create temp file in '{}' for HttpClient::Download. Reason: {}", TempFolderPath.string(), Ec.message()); PayloadFile.reset(); } } else { PayloadString.reserve(ContentSize.value()); } } } if (!Header.first.empty()) { ReceivedHeaders.emplace_back(std::move(Header)); } return 1; }; Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); for (const std::pair& H : ReceivedHeaders) { Response.header.insert_or_assign(H.first, H.second); } } if (m_ConnectionSettings.AllowResume) { auto SupportsRanges = [](const cpr::Response& Response) -> bool { if (Response.header.find("Content-Range") != Response.header.end()) { return true; } if (auto It = Response.header.find("Accept-Ranges"); It != Response.header.end()) { return It->second == "bytes"sv; } return false; }; auto ShouldResume = [&SupportsRanges](const cpr::Response& Response) -> bool { if (ShouldRetry(Response)) { return SupportsRanges(Response); } return false; }; if (ShouldResume(Response)) { auto It = Response.header.find("Content-Length"); if (It != Response.header.end()) { std::optional ContentLength = ParseInt(It->second); if (ContentLength) { std::vector> ReceivedHeaders; auto HeaderCallback = [&](std::string header, intptr_t) { std::pair Header = GetHeader(header); if (!Header.first.empty()) { ReceivedHeaders.emplace_back(std::move(Header)); } if (Header.first == "Content-Range"sv) { if (Header.second.starts_with("bytes "sv)) { size_t RangeStartEnd = Header.second.find('-', 6); if (RangeStartEnd != std::string::npos) { const auto Start = ParseInt(Header.second.substr(6, RangeStartEnd - 6)); if (Start) { uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); if (Start.value() == DownloadedSize) { return 1; } else if (Start.value() > DownloadedSize) { return 0; } if (PayloadFile) { PayloadFile->ResetWritePos(Start.value()); } else { PayloadString = PayloadString.substr(0, Start.value()); } return 1; } } } return 0; } return 1; }; KeyValueMap HeadersWithRange(AdditionalHeader); do { uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); std::string Range = fmt::format("bytes={}-{}", DownloadedSize, ContentLength.value()); if (auto RangeIt = HeadersWithRange.Entries.find("Range"); RangeIt != HeadersWithRange.Entries.end()) { if (RangeIt->second == Range) { // If we didn't make any progress, abort break; } } HeadersWithRange.Entries.insert_or_assign("Range", Range); Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, HeadersWithRange, {}, m_SessionId, GetAccessToken()); Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); for (const std::pair& H : ReceivedHeaders) { Response.header.insert_or_assign(H.first, H.second); } ReceivedHeaders.clear(); } while (ShouldResume(Response)); } } } } if (!PayloadString.empty()) { Response.text = std::move(PayloadString); } return Response; }, m_ConnectionSettings.RetryCount); return CommonResponse(std::move(Response), PayloadFile ? PayloadFile->DetachToIoBuffer() : IoBuffer{}); } ////////////////////////////////////////////////////////////////////////// CbObject HttpClient::Response::AsObject() const { // TODO: sanity check the payload format etc if (ResponsePayload) { return LoadCompactBinaryObject(ResponsePayload); } return {}; } CbPackage HttpClient::Response::AsPackage() const { // TODO: sanity checks and error handling if (ResponsePayload) { return ParsePackageMessage(ResponsePayload); } return {}; } std::string_view HttpClient::Response::AsText() const { if (ResponsePayload) { return std::string_view(reinterpret_cast(ResponsePayload.GetData()), ResponsePayload.GetSize()); } return {}; } std::string HttpClient::Response::ToText() const { if (!ResponsePayload) return {}; switch (ResponsePayload.GetContentType()) { case ZenContentType::kCbObject: { zen::ExtendableStringBuilder<1024> ObjStr; zen::CbObject Object{SharedBuffer(ResponsePayload)}; zen::CompactBinaryToJson(Object, ObjStr); return ObjStr.ToString(); } break; case ZenContentType::kCSS: case ZenContentType::kHTML: case ZenContentType::kJavaScript: case ZenContentType::kJSON: case ZenContentType::kText: case ZenContentType::kYAML: return std::string{AsText()}; default: return ""; } } bool HttpClient::Response::IsSuccess() const noexcept { return !Error && IsHttpSuccessCode(StatusCode); } std::string HttpClient::Response::ErrorMessage(std::string_view Prefix) const { if (Error.has_value()) { return fmt::format("{}: {}", Prefix, Error->ErrorMessage); } else if (StatusCode != HttpResponseCode::ImATeapot && (int)StatusCode) { return fmt::format("{}{}HTTP error {} {} ({})", Prefix, Prefix.empty() ? ""sv : ": "sv, (int)StatusCode, zen::ToString(StatusCode), ToText()); } else { return fmt::format("{}{}unknown error", Prefix, Prefix.empty() ? ""sv : ": "sv); } } void HttpClient::Response::ThrowError(std::string_view ErrorPrefix) { if (!IsSuccess()) { throw std::runtime_error(ErrorMessage(ErrorPrefix)); } } ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS TEST_CASE("httpclient") { using namespace std::literals; SUBCASE("client") {} } void httpclient_forcelink() { } #endif } // namespace zen