use streaming read for PutCompressedBlob if source is single file (#338)

* use streaming read for PutCompressedBlob if source is single file
* changelog

2 files changed, 19 insertions, 1 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4445bf9cd..35164d10d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@
 - Improvement: Keep reason and status code when parsing responses from jupiter remote requests
 - Improvement: Add additional context for errors when importing/exporting oplogs
 - Improvement: Added `ZenServerInstance::SpawnServerAndWait` and improved logic around process termination when using `ZenServerInstance::AttachToRunningServer`
+- Improvement: When uploading compressed blob to jupiter, use streaming reading of source file if it is a "whole file" - a large attachment.
 
 ## 0.2.13
 - Feature: Project store now has a `snapshot` RPC on oplogs which may be used to inline any files referenced by name into Zen store. This makes the oplog transportable
diff --git a/src/zenserver/upstream/jupiter.cpp b/src/zenserver/upstream/jupiter.cpp
index 32537508c..e7bb2cbcf 100644
--- a/src/zenserver/upstream/jupiter.cpp
+++ b/src/zenserver/upstream/jupiter.cpp
@@ -445,7 +445,24 @@ CloudCacheSession::PutCompressedBlob(std::string_view Namespace, const IoHash& K
 
 	Session.SetOption(cpr::Url{Uri.c_str()});
 	Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-comp"}});
-	Session.SetBody(cpr::Body{(const char*)Blob.Data(), Blob.Size()});
+
+	uint64_t Offset = 0;
+	if (Blob.IsWholeFile())
+	{
+		auto ReadCallback = [&Blob, &Offset](char* buffer, size_t& size, intptr_t) {
+			size						   = Min<size_t>(size, Blob.GetSize() - Offset);
+			IoBuffer		  PayloadRange = IoBuffer(Blob, Offset, size);
+			MutableMemoryView Data(buffer, size);
+			Data.CopyFrom(PayloadRange.GetView());
+			Offset += size;
+			return true;
+		};
+		Session.SetReadCallback(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(Blob.GetSize()), ReadCallback));
+	}
+	else
+	{
+		Session.SetBody(cpr::Body{(const char*)Blob.Data(), Blob.Size()});
+	}
 
 	cpr::Response Response = Session.Put();
 	ZEN_DEBUG("PUT {}", Response);