partial block fetch (#298)

 - Improvement: Do partial requests of blocks if not all of the block is needed
  - Improvement: Better progress/statistics on download
  - Bugfix: Ensure that temporary folder for Jupiter downloads exists during verify phase

5 files changed, 36 insertions, 12 deletions

diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index a4bb759e7..e57109006 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -325,7 +325,7 @@ public:
 		return {};
 	}
 
-	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
 	{
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
@@ -337,10 +337,19 @@ public:
 		if (std::filesystem::is_regular_file(BlockPath))
 		{
 			BasicFile File(BlockPath, BasicFile::Mode::kRead);
-			IoBuffer  Payload = File.ReadAll();
-			ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload))));
-			m_Stats.TotalBytesRead += Payload.GetSize();
+			IoBuffer  Payload;
+			if (RangeOffset != 0 || RangeBytes != (uint64_t)-1)
+			{
+				Payload = IoBuffer(RangeBytes);
+				File.Read(Payload.GetMutableView().GetData(), RangeBytes, RangeOffset);
+			}
+			else
+			{
+				Payload = File.ReadAll();
+				ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload))));
+			}
 			Payload.SetContentType(ZenContentType::kCompressedBinary);
+			m_Stats.TotalBytesRead += Payload.GetSize();
 			SimulateLatency(0, Payload.GetSize());
 			return Payload;
 		}
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
index 9c236310f..9d2bab170 100644
--- a/src/zenutil/include/zenutil/buildstorage.h
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -40,7 +40,10 @@ public:
 																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
 																 std::function<void(uint64_t, bool)>&&					   OnSentBytes)					 = 0;
 
-	virtual IoBuffer						   GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) = 0;
+	virtual IoBuffer						   GetBuildBlob(const Oid&	  BuildId,
+															const IoHash& RawHash,
+															uint64_t	  RangeOffset = 0,
+															uint64_t	  RangeBytes  = (uint64_t)-1) = 0;
 	virtual std::vector<std::function<void()>> GetLargeBuildBlob(
 		const Oid&																			   BuildId,
 		const IoHash&																		   RawHash,
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
index 852271868..2c5fc73b8 100644
--- a/src/zenutil/include/zenutil/jupiter/jupitersession.h
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -123,7 +123,9 @@ public:
 									std::string_view	  BucketId,
 									const Oid&			  BuildId,
 									const IoHash&		  Hash,
-									std::filesystem::path TempFolderPath);
+									std::filesystem::path TempFolderPath,
+									uint64_t			  Offset = 0,
+									uint64_t			  Size	 = (uint64_t)-1);
 
 	JupiterResult			PutMultipartBuildBlob(std::string_view												  Namespace,
 												  std::string_view												  BucketId,
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 309885b05..bf89ce785 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -217,13 +217,15 @@ public:
 		return WorkList;
 	}
 
-	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
 	{
 		ZEN_TRACE_CPU("Jupiter::GetBuildBlob");
 
-		Stopwatch	  ExecutionTimer;
-		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
-		JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		CreateDirectories(m_TempFolderPath);
+		JupiterResult GetBuildBlobResult =
+			m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, RangeOffset, RangeBytes);
 		AddStatistic(GetBuildBlobResult);
 		if (!GetBuildBlobResult.Success)
 		{
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index 06ac6ae36..68f214c06 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -698,11 +698,19 @@ JupiterSession::GetBuildBlob(std::string_view	   Namespace,
 							 std::string_view	   BucketId,
 							 const Oid&			   BuildId,
 							 const IoHash&		   Hash,
-							 std::filesystem::path TempFolderPath)
+							 std::filesystem::path TempFolderPath,
+							 uint64_t			   Offset,
+							 uint64_t			   Size)
 {
+	HttpClient::KeyValueMap Headers;
+	if (Offset != 0 || Size != (uint64_t)-1)
+	{
+		Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", Offset, Offset + Size - 1)});
+	}
 	HttpClient::Response Response =
 		m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
-							  TempFolderPath);
+							  TempFolderPath,
+							  Headers);
 
 	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
 }