partial block fetch (#298)

 - Improvement: Do partial requests of blocks if not all of the block is needed
  - Improvement: Better progress/statistics on download
  - Bugfix: Ensure that temporary folder for Jupiter downloads exists during verify phase

2 files changed, 16 insertions, 6 deletions

diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 309885b05..bf89ce785 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -217,13 +217,15 @@ public:
 		return WorkList;
 	}
 
-	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
 	{
 		ZEN_TRACE_CPU("Jupiter::GetBuildBlob");
 
-		Stopwatch	  ExecutionTimer;
-		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
-		JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		CreateDirectories(m_TempFolderPath);
+		JupiterResult GetBuildBlobResult =
+			m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, RangeOffset, RangeBytes);
 		AddStatistic(GetBuildBlobResult);
 		if (!GetBuildBlobResult.Success)
 		{
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index 06ac6ae36..68f214c06 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -698,11 +698,19 @@ JupiterSession::GetBuildBlob(std::string_view	   Namespace,
 							 std::string_view	   BucketId,
 							 const Oid&			   BuildId,
 							 const IoHash&		   Hash,
-							 std::filesystem::path TempFolderPath)
+							 std::filesystem::path TempFolderPath,
+							 uint64_t			   Offset,
+							 uint64_t			   Size)
 {
+	HttpClient::KeyValueMap Headers;
+	if (Offset != 0 || Size != (uint64_t)-1)
+	{
+		Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", Offset, Offset + Size - 1)});
+	}
 	HttpClient::Response Response =
 		m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
-							  TempFolderPath);
+							  TempFolderPath,
+							  Headers);
 
 	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
 }