Builds API remote project store (#258)

Feature: zen command oplog-export and oplog-import now supports --builds remote target using the Jupiter builds API

1 files changed, 71 insertions, 3 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index f25042a62..63a80fbd8 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -24,6 +24,7 @@
 #include <zenutil/referencemetadata.h>
 #include <zenutil/workerpools.h>
 
+#include "buildsremoteprojectstore.h"
 #include "fileremoteprojectstore.h"
 #include "jupiterremoteprojectstore.h"
 #include "remoteprojectstore.h"
@@ -266,6 +267,72 @@ namespace {
 			RemoteStore					  = CreateZenRemoteStore(Options, TempFilePath);
 		}
 
+		if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
+		{
+			std::string_view BuildsServiceUrl = Builds["url"sv].AsString();
+			if (BuildsServiceUrl.empty())
+			{
+				return {nullptr, "Missing service url"};
+			}
+
+			std::string		 Url	   = cpr::util::urlDecode(std::string(BuildsServiceUrl));
+			std::string_view Namespace = Builds["namespace"sv].AsString();
+			if (Namespace.empty())
+			{
+				return {nullptr, "Missing namespace"};
+			}
+			std::string_view Bucket = Builds["bucket"sv].AsString();
+			if (Bucket.empty())
+			{
+				return {nullptr, "Missing bucket"};
+			}
+			std::string_view OpenIdProvider = Builds["openid-provider"sv].AsString();
+			std::string		 AccessToken	= std::string(Builds["access-token"sv].AsString());
+			if (AccessToken.empty())
+			{
+				std::string_view AccessTokenEnvVariable = Builds["access-token-env"].AsString();
+				if (!AccessTokenEnvVariable.empty())
+				{
+					AccessToken = GetEnvVariable(AccessTokenEnvVariable);
+				}
+			}
+			std::string_view BuildIdParam = Builds["buildsid"sv].AsString();
+			if (BuildIdParam.empty())
+			{
+				return {nullptr, "Missing build id"};
+			}
+			if (BuildIdParam.length() != Oid::StringLength)
+			{
+				return {nullptr, "Invalid build id"};
+			}
+			Oid BuildId = Oid::FromHexString(BuildIdParam);
+			if (BuildId == Oid::Zero)
+			{
+				return {nullptr, "Invalid build id string"};
+			}
+
+			bool ForceDisableBlocks		= Builds["disableblocks"sv].AsBool(false);
+			bool ForceDisableTempBlocks = Builds["disabletempblocks"sv].AsBool(false);
+			bool AssumeHttp2			= Builds["assumehttp2"sv].AsBool(false);
+
+			MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
+			IoBuffer   MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
+
+			BuildsRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+												Url,
+												std::string(Namespace),
+												std::string(Bucket),
+												BuildId,
+												std::string(OpenIdProvider),
+												AccessToken,
+												AuthManager,
+												ForceDisableBlocks,
+												ForceDisableTempBlocks,
+												AssumeHttp2,
+												MetaData};
+			RemoteStore						 = CreateBuildsRemoteStore(Options, TempFilePath);
+		}
+
 		if (!RemoteStore)
 		{
 			return {nullptr, "Unknown remote store type"};
@@ -5132,7 +5199,7 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 		/* BuildBlocks */ false,
 		/* IgnoreMissingAttachments */ false,
 		/* AllowChunking*/ false,
-		[](CompressedBuffer&&, const IoHash&) {},
+		[](CompressedBuffer&&, RemoteProjectStore::Block&&) {},
 		[](const IoHash&, TGetAttachmentBufferFunc&&) {},
 		[](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {},
 		/* EmbedLooseFiles*/ false);
@@ -8354,8 +8421,9 @@ TEST_CASE("project.store.block")
 											return CompositeBuffer(SharedBuffer(Buffer));
 										}));
 	}
-	CompressedBuffer Block = GenerateBlock(std::move(Chunks));
-	CHECK(IterateBlock(Block.Decompress(), [](CompressedBuffer&&, const IoHash&) {}));
+	RemoteProjectStore::Block Block;
+	CompressedBuffer		  BlockBuffer = GenerateBlock(std::move(Chunks), Block);
+	CHECK(IterateBlock(BlockBuffer.Decompress(), [](CompressedBuffer&&, const IoHash&) {}));
 }
 
 TEST_CASE("project.store.iterateoplog")