From 28bc5ebf05984385cc0567c89b1d8e7a541ebef8 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Wed, 26 Mar 2025 17:06:23 +0100
Subject: zen build cache service (#318)

- **EXPERIMENTAL** `zen builds`
  - Feature: `--zen-cache-host` option for `upload` and `download` operations to use a zenserver host `/builds` endpoint for storing build blob and blob metadata
  - Feature: New `/builds` endpoint for caching build blobs and blob metadata
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` `GET` and `PUT` method for storing and fetching blobs
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata` `POST` method for storing metadata about blobs
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata` `POST` method for fetching metadata about blobs
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/exists` `POST` method for checking existance of blobs
---
 src/zenutil/jupiter/jupiterbuildstorage.cpp | 35 ++++++++---------------------
 1 file changed, 9 insertions(+), 26 deletions(-)

(limited to 'src/zenutil/jupiter/jupiterbuildstorage.cpp')

diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index d70fd8c00..b6d9e3990 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -49,7 +49,7 @@ public:
 		{
 			throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode));
 		}
-		return PayloadToJson("Failed listing builds"sv, ListResult.Response);
+		return PayloadToCbObject("Failed listing builds"sv, ListResult.Response);
 	}
 
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
@@ -66,7 +66,7 @@ public:
 		{
 			throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode));
 		}
-		return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
+		return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
 	}
 
 	virtual CbObject GetBuild(const Oid& BuildId) override
@@ -81,7 +81,7 @@ public:
 		{
 			throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode));
 		}
-		return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
+		return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
 	}
 
 	virtual void FinalizeBuild(const Oid& BuildId) override
@@ -134,7 +134,7 @@ public:
 												 GetBuildPartResult.Reason,
 												 GetBuildPartResult.ErrorCode));
 		}
-		return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
+		return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
 	}
 
 	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
@@ -289,7 +289,7 @@ public:
 		}
 	}
 
-	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
+	virtual CbObject FindBlocks(const Oid& BuildId) override
 	{
 		ZEN_TRACE_CPU("Jupiter::FindBlocks");
 
@@ -301,10 +301,10 @@ public:
 		{
 			throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode));
 		}
-		return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response));
+		return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response);
 	}
 
-	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
 		ZEN_TRACE_CPU("Jupiter::GetBlockMetadata");
 
@@ -328,24 +328,7 @@ public:
 			throw std::runtime_error(
 				fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode));
 		}
-		std::vector<ChunkBlockDescription> UnorderedList =
-			ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response));
-		tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup;
-		for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++)
-		{
-			const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex];
-			BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex);
-		}
-		std::vector<ChunkBlockDescription> SortedBlockDescriptions;
-		SortedBlockDescriptions.reserve(BlockDescriptionLookup.size());
-		for (const IoHash& BlockHash : BlockHashes)
-		{
-			if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end())
-			{
-				SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second]));
-			}
-		}
-		return SortedBlockDescriptions;
+		return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response);
 	}
 
 	virtual void PutBuildPartStats(const Oid&								  BuildId,
@@ -373,7 +356,7 @@ public:
 	}
 
 private:
-	static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload)
+	static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload)
 	{
 		if (Payload.GetContentType() == ZenContentType::kJSON)
 		{
-- 
cgit v1.2.3