use multi range requests (#800)

- Improvement: `zen builds download` now uses multi-range requests for blocks to reduce download size
- Improvement: `zen oplog-import` now uses partial block with multi-range requests for blocks to reduce download size
- Improvement: Improved feedback in log/console during `zen oplog-import`
- Improvement: `--allow-partial-block-requests` now defaults to `true` for `zen builds download` and `zen oplog-import` (was `mixed`)
- Improvement: Improved range merging analysis when downloading partial blocks

1 files changed, 39 insertions, 0 deletions

diff --git a/src/zenremotestore/builds/filebuildstorage.cpp b/src/zenremotestore/builds/filebuildstorage.cpp
index 55e69de61..2f4904449 100644
--- a/src/zenremotestore/builds/filebuildstorage.cpp
+++ b/src/zenremotestore/builds/filebuildstorage.cpp
@@ -432,6 +432,45 @@ public:
 		return IoBuffer{};
 	}
 
+	virtual BuildBlobRanges GetBuildBlobRanges(const Oid&									  BuildId,
+											   const IoHash&								  RawHash,
+											   std::span<const std::pair<uint64_t, uint64_t>> Ranges) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlobRanges");
+		ZEN_UNUSED(BuildId);
+		ZEN_ASSERT(!Ranges.empty());
+
+		uint64_t ReceivedBytes = 0;
+		uint64_t SentBytes	   = Ranges.size() * 2 * 8;
+
+		SimulateLatency(SentBytes, 0);
+		auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
+		Stopwatch ExecutionTimer;
+		auto	  __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
+
+		BuildBlobRanges Result;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (IsFile(BlockPath))
+		{
+			BasicFile File(BlockPath, BasicFile::Mode::kRead);
+
+			uint64_t RangeOffset = Ranges.front().first;
+			uint64_t RangeBytes	 = Ranges.back().first + Ranges.back().second - RangeOffset;
+			Result.PayloadBuffer = IoBufferBuilder::MakeFromFileHandle(File.Detach(), RangeOffset, RangeBytes);
+
+			Result.Ranges.reserve(Ranges.size());
+
+			for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+			{
+				Result.Ranges.push_back(std::make_pair(Range.first - RangeOffset, Range.second));
+			}
+			ReceivedBytes = Result.PayloadBuffer.GetSize();
+		}
+		return Result;
+	}
+
 	virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid&													   BuildId,
 																 const IoHash&												   RawHash,
 																 uint64_t													   ChunkSize,