refactor builds cmd (#566)

Move builds download code from builds_cmd.cpp to remotestorelib

2 files changed, 118 insertions, 0 deletions

diff --git a/src/zenutil/bufferedopenfile.cpp b/src/zenutil/bufferedopenfile.cpp
new file mode 100644
index 000000000..360011302
--- /dev/null
+++ b/src/zenutil/bufferedopenfile.cpp
@@ -0,0 +1,75 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/bufferedopenfile.h>
+
+#include <zencore/scopeguard.h>
+#include <zencore/trace.h>
+
+namespace zen {
+
+BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path,
+								   std::atomic<uint64_t>&	   OpenReadCount,
+								   std::atomic<uint64_t>&	   CurrentOpenFileCount,
+								   std::atomic<uint64_t>&	   ReadCount,
+								   std::atomic<uint64_t>&	   ReadByteCount)
+: m_Source(Path, BasicFile::Mode::kRead)
+, m_SourceSize(m_Source.FileSize())
+, m_OpenReadCount(OpenReadCount)
+, m_CurrentOpenFileCount(CurrentOpenFileCount)
+, m_ReadCount(ReadCount)
+, m_ReadByteCount(ReadByteCount)
+
+{
+	m_OpenReadCount++;
+	m_CurrentOpenFileCount++;
+}
+
+BufferedOpenFile::~BufferedOpenFile()
+{
+	m_CurrentOpenFileCount--;
+}
+
+CompositeBuffer
+BufferedOpenFile::GetRange(uint64_t Offset, uint64_t Size)
+{
+	ZEN_TRACE_CPU("BufferedOpenFile::GetRange");
+
+	ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache);
+	auto _ = MakeGuard([&]() { ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); });
+
+	ZEN_ASSERT((Offset + Size) <= m_SourceSize);
+	const uint64_t BlockIndexStart = Offset / BlockSize;
+	const uint64_t BlockIndexEnd   = (Offset + Size - 1) / BlockSize;
+
+	std::vector<SharedBuffer> BufferRanges;
+	BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1);
+
+	uint64_t ReadOffset = Offset;
+	for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++)
+	{
+		const uint64_t BlockStartOffset = BlockIndex * BlockSize;
+		if (m_CacheBlockIndex != BlockIndex)
+		{
+			uint64_t CacheSize = Min(BlockSize, m_SourceSize - BlockStartOffset);
+			ZEN_ASSERT(CacheSize > 0);
+			m_Cache = IoBuffer(CacheSize);
+			m_Source.Read(m_Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset);
+			m_ReadCount++;
+			m_ReadByteCount += CacheSize;
+			m_CacheBlockIndex = BlockIndex;
+		}
+
+		const uint64_t BytesRead = ReadOffset - Offset;
+		ZEN_ASSERT(BlockStartOffset <= ReadOffset);
+		const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset;
+		ZEN_ASSERT(OffsetIntoBlock < m_Cache.GetSize());
+		const uint64_t BlockBytes = Min(m_Cache.GetSize() - OffsetIntoBlock, Size - BytesRead);
+		BufferRanges.emplace_back(SharedBuffer(IoBuffer(m_Cache, OffsetIntoBlock, BlockBytes)));
+		ReadOffset += BlockBytes;
+	}
+	CompositeBuffer Result(std::move(BufferRanges));
+	ZEN_ASSERT(Result.GetSize() == Size);
+	return Result;
+}
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/bufferedopenfile.h b/src/zenutil/include/zenutil/bufferedopenfile.h
new file mode 100644
index 000000000..3af7c9991
--- /dev/null
+++ b/src/zenutil/include/zenutil/bufferedopenfile.h
@@ -0,0 +1,43 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/basicfile.h>
+
+namespace zen {
+
+class CompositeBuffer;
+
+class BufferedOpenFile
+{
+public:
+	BufferedOpenFile(const std::filesystem::path Path,
+					 std::atomic<uint64_t>&		 OpenReadCount,
+					 std::atomic<uint64_t>&		 CurrentOpenFileCount,
+					 std::atomic<uint64_t>&		 ReadCount,
+					 std::atomic<uint64_t>&		 ReadByteCount);
+	~BufferedOpenFile();
+	BufferedOpenFile()						  = delete;
+	BufferedOpenFile(const BufferedOpenFile&) = delete;
+	BufferedOpenFile(BufferedOpenFile&&)	  = delete;
+	BufferedOpenFile& operator=(BufferedOpenFile&&) = delete;
+	BufferedOpenFile& operator=(const BufferedOpenFile&) = delete;
+
+	const uint64_t	BlockSize = 256u * 1024u;
+	CompositeBuffer GetRange(uint64_t Offset, uint64_t Size);
+
+public:
+	void* Handle() { return m_Source.Handle(); }
+
+private:
+	BasicFile			   m_Source;
+	const uint64_t		   m_SourceSize;
+	std::atomic<uint64_t>& m_OpenReadCount;
+	std::atomic<uint64_t>& m_CurrentOpenFileCount;
+	std::atomic<uint64_t>& m_ReadCount;
+	std::atomic<uint64_t>& m_ReadByteCount;
+	uint64_t			   m_CacheBlockIndex = (uint64_t)-1;
+	IoBuffer			   m_Cache;
+};
+
+}  // namespace zen