zenremoteprojectstore with httpclient (#35)

- Bugfix: Fix log of Success/Failure for oplog import
- Improvement: Use HttpClient when doing oplog export/import with a zenserver as a remote target. Includes retry logic
- Improvement: Increase the retry count to 4 (5 attempts in total) when talking to Jupiter for oplog export/import

1 files changed, 32 insertions, 18 deletions

diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 9811e5814..277b93a0f 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -51,16 +51,6 @@ AsCprBody(const IoBuffer& Obj)
 	return cpr::Body((const char*)Obj.GetData(), Obj.GetSize());
 }
 
-static cpr::Body
-AsCprBody(const CompositeBuffer& Buffers)
-{
-	SharedBuffer Buffer = Buffers.Flatten();
-
-	// This is super inefficient, should be fixed
-	std::string String{(const char*)Buffer.GetData(), Buffer.GetSize()};
-	return cpr::Body{std::move(String)};
-}
-
 //////////////////////////////////////////////////////////////////////////
 
 static HttpClient::Response
@@ -221,10 +211,15 @@ struct HttpClient::Impl : public RefCounted
 			CprSession->SetReadCallback({});
 			return Result;
 		}
-		inline cpr::Response Post()
+		inline cpr::Response Post(std::optional<cpr::ReadCallback>&& Read = {})
 		{
+			if (Read)
+			{
+				CprSession->SetReadCallback(std::move(Read.value()));
+			}
 			cpr::Response Result = CprSession->Post();
 			ZEN_TRACE("POST {}", Result);
+			CprSession->SetReadCallback({});
 			return Result;
 		}
 		inline cpr::Response Delete()
@@ -724,6 +719,12 @@ HttpClient::Post(std::string_view Url, const KeyValueMap& AdditionalHeader, cons
 HttpClient::Response
 HttpClient::Post(std::string_view Url, const IoBuffer& Payload, const KeyValueMap& AdditionalHeader)
 {
+	return Post(Url, Payload, Payload.GetContentType(), AdditionalHeader);
+}
+
+HttpClient::Response
+HttpClient::Post(std::string_view Url, const IoBuffer& Payload, ZenContentType ContentType, const KeyValueMap& AdditionalHeader)
+{
 	ZEN_TRACE_CPU("HttpClient::PostWithPayload");
 
 	return CommonResponse(DoWithRetry(
@@ -732,7 +733,7 @@ HttpClient::Post(std::string_view Url, const IoBuffer& Payload, const KeyValueMa
 				m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken());
 
 			Sess->SetBody(AsCprBody(Payload));
-			Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())});
+			Sess->UpdateHeader({HeaderContentType(ContentType)});
 			return Sess.Post();
 		},
 		m_ConnectionSettings.RetryCount));
@@ -758,17 +759,30 @@ HttpClient::Post(std::string_view Url, CbObject Payload, const KeyValueMap& Addi
 HttpClient::Response
 HttpClient::Post(std::string_view Url, CbPackage Pkg, const KeyValueMap& AdditionalHeader)
 {
-	ZEN_TRACE_CPU("HttpClient::PostPackage");
+	return Post(Url, zen::FormatPackageMessageBuffer(Pkg), ZenContentType::kCbPackage, AdditionalHeader);
+}
+
+HttpClient::Response
+HttpClient::Post(std::string_view Url, const CompositeBuffer& Payload, ZenContentType ContentType, const KeyValueMap& AdditionalHeader)
+{
+	ZEN_TRACE_CPU("HttpClient::Post");
 
 	return CommonResponse(DoWithRetry(
 		[&]() {
-			CompositeBuffer Message = zen::FormatPackageMessageBuffer(Pkg);
-
+			uint64_t				  SizeLeft	   = Payload.GetSize();
+			CompositeBuffer::Iterator BufferIt	   = Payload.GetIterator(0);
+			auto					  ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) {
+				 size = Min<size_t>(size, SizeLeft);
+				 MutableMemoryView Data(buffer, size);
+				 Payload.CopyTo(Data, BufferIt);
+				 SizeLeft -= size;
+				 return true;
+			};
 			Impl::Session Sess =
 				m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken());
-			Sess->SetBody(AsCprBody(Message));
-			Sess->UpdateHeader({HeaderContentType(ZenContentType::kCbPackage)});
-			return Sess.Post();
+			Sess->UpdateHeader({HeaderContentType(ContentType)});
+
+			return Sess.Post(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(Payload.GetSize()), ReadCallback));
 		},
 		m_ConnectionSettings.RetryCount));
 }