Merge branch 'main' into fd-fix-binary-json

1 files changed, 31 insertions, 17 deletions

diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index 130fec355..7aa252e44 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -235,7 +235,7 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (!std::filesystem::is_regular_file(BlockPath))
+		if (!IsFile(BlockPath))
 		{
 			CreateDirectories(BlockPath.parent_path());
 			TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView());
@@ -260,7 +260,7 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (!std::filesystem::is_regular_file(BlockPath))
+		if (!IsFile(BlockPath))
 		{
 			CreateDirectories(BlockPath.parent_path());
 
@@ -346,7 +346,7 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (std::filesystem::is_regular_file(BlockPath))
+		if (IsFile(BlockPath))
 		{
 			BasicFile File(BlockPath, BasicFile::Mode::kRead);
 			IoBuffer  Payload;
@@ -383,7 +383,7 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (std::filesystem::is_regular_file(BlockPath))
+		if (IsFile(BlockPath))
 		{
 			struct WorkloadData
 			{
@@ -442,63 +442,77 @@ public:
 		SimulateLatency(0, 0);
 	}
 
-	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
+	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::FindBlocks");
 		ZEN_UNUSED(BuildId);
-		SimulateLatency(0, 0);
+		SimulateLatency(sizeof(BuildId), 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		m_Stats.TotalRequestCount++;
 
+		uint64_t FoundCount = 0;
+
 		DirectoryContent Content;
 		GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content);
-		std::vector<ChunkBlockDescription> Result;
+		CbObjectWriter Writer;
+		Writer.BeginArray("blocks");
 		for (const std::filesystem::path& MetaDataFile : Content.Files)
 		{
 			IoHash ChunkHash;
 			if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash))
 			{
 				std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash);
-				if (std::filesystem::is_regular_file(BlockPath))
+				if (IsFile(BlockPath))
 				{
 					IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
 
 					m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
 
 					CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
-					Result.emplace_back(ParseChunkBlockDescription(BlockObject));
+					Writer.AddObject(BlockObject);
+					FoundCount++;
+					if (FoundCount == MaxBlockCount)
+					{
+						break;
+					}
 				}
 			}
 		}
-		SimulateLatency(0, sizeof(IoHash) * Result.size());
+		Writer.EndArray();	// blocks
+		CbObject Result = Writer.Save();
+		SimulateLatency(0, Result.GetSize());
 		return Result;
 	}
 
-	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata");
 		ZEN_UNUSED(BuildId);
-		SimulateLatency(0, 0);
+		SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		m_Stats.TotalRequestCount++;
 
-		std::vector<ChunkBlockDescription> Result;
+		CbObjectWriter Writer;
+		Writer.BeginArray("blocks");
+
 		for (const IoHash& BlockHash : BlockHashes)
 		{
 			std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash);
-			if (std::filesystem::is_regular_file(MetaDataFile))
+			if (IsFile(MetaDataFile))
 			{
 				IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
 
 				m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
 
 				CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
-				Result.emplace_back(ParseChunkBlockDescription(BlockObject));
+				Writer.AddObject(BlockObject);
 			}
 		}
-		SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size());
+		Writer.EndArray();	// blocks
+		CbObject Result = Writer.Save();
+		SimulateLatency(0, Result.GetSize());
 		return Result;
 	}
 
@@ -616,7 +630,7 @@ protected:
 		BuildPartObject.IterateAttachments([&](CbFieldView FieldView) {
 			const IoHash				AttachmentHash = FieldView.AsBinaryAttachment();
 			const std::filesystem::path BlockPath	   = GetBlobPayloadPath(AttachmentHash);
-			if (!std::filesystem::is_regular_file(BlockPath))
+			if (!IsFile(BlockPath))
 			{
 				NeededAttachments.push_back(AttachmentHash);
 			}