Merge branch 'main' of https://github.ol.epicgames.net/ue-foundation/zensb/cleanup-main

1 files changed, 505 insertions, 0 deletions

diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
new file mode 100644
index 000000000..f706a7efc
--- /dev/null
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -0,0 +1,505 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/jupiter/jupitersession.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/fmtutils.h>
+#include <zencore/trace.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+//#include <cpr/cpr.h>
+//#include <fmt/format.h>
+#include <json11.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+using namespace std::literals;
+
+namespace zen {
+
+namespace detail {
+	JupiterResult ConvertResponse(const HttpClient::Response& Response, const std::string_view ErrorPrefix = ""sv)
+	{
+		if (Response.Error)
+		{
+			return {.SentBytes		= gsl::narrow<uint64_t>(Response.UploadedBytes),
+					.ReceivedBytes	= gsl::narrow<uint64_t>(Response.DownloadedBytes),
+					.ElapsedSeconds = Response.ElapsedSeconds,
+					.ErrorCode		= Response.Error.value().ErrorCode,
+					.Reason			= Response.ErrorMessage(ErrorPrefix),
+					.Success		= false};
+		}
+		if (!Response.IsSuccess())
+		{
+			return {.SentBytes		= gsl::narrow<uint64_t>(Response.UploadedBytes),
+					.ReceivedBytes	= gsl::narrow<uint64_t>(Response.DownloadedBytes),
+					.ElapsedSeconds = Response.ElapsedSeconds,
+					.ErrorCode		= static_cast<int32_t>(Response.StatusCode),
+					.Reason			= Response.ErrorMessage(ErrorPrefix),
+					.Success		= false};
+		}
+		return {.Response		= Response.ResponsePayload,
+				.SentBytes		= gsl::narrow<uint64_t>(Response.UploadedBytes),
+				.ReceivedBytes	= gsl::narrow<uint64_t>(Response.DownloadedBytes),
+				.ElapsedSeconds = Response.ElapsedSeconds,
+				.ErrorCode		= 0,
+				.Success		= true};
+	}
+}  // namespace detail
+
+JupiterSession::JupiterSession(LoggerRef InLog, HttpClient& InHttpClient) : m_Log(InLog), m_HttpClient(InHttpClient)
+{
+}
+
+JupiterSession::~JupiterSession()
+{
+}
+
+JupiterResult
+JupiterSession::Authenticate()
+{
+	bool OK = m_HttpClient.Authenticate();
+	return {.Success = OK};
+}
+
+JupiterResult
+JupiterSession::GetRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType RefType)
+{
+	ZEN_TRACE_CPU("JupiterClient::GetRef");
+
+	HttpClient::Response Response =
+		m_HttpClient.Get(fmt::format("/api/v1/refs/{}/{}/{}", Namespace, BucketId, Key.ToHexString()), {HttpClient::Accept(RefType)});
+
+	return detail::ConvertResponse(Response, "JupiterSession::GetRef"sv);
+}
+
+JupiterResult
+JupiterSession::GetBlob(std::string_view Namespace, const IoHash& Key)
+{
+	ZEN_TRACE_CPU("JupiterClient::GetBlob");
+	HttpClient::Response Response =
+		m_HttpClient.Get(fmt::format("/api/v1/blobs/{}/{}", Namespace, Key.ToHexString()), {HttpClient::Accept(ZenContentType::kBinary)});
+
+	return detail::ConvertResponse(Response);
+}
+
+JupiterResult
+JupiterSession::GetCompressedBlob(std::string_view Namespace, const IoHash& Key, std::filesystem::path TempFolderPath)
+{
+	ZEN_TRACE_CPU("JupiterClient::GetCompressedBlob");
+
+	HttpClient::Response Response = m_HttpClient.Download(fmt::format("/api/v1/compressed-blobs/{}/{}", Namespace, Key.ToHexString()),
+														  TempFolderPath,
+														  {HttpClient::Accept(ZenContentType::kCompressedBinary)});
+
+	return detail::ConvertResponse(Response);
+}
+
+JupiterResult
+JupiterSession::GetInlineBlob(std::string_view		Namespace,
+							  std::string_view		BucketId,
+							  const IoHash&			Key,
+							  IoHash&				OutPayloadHash,
+							  std::filesystem::path TempFolderPath)
+{
+	ZEN_TRACE_CPU("JupiterClient::GetInlineBlob");
+
+	HttpClient::Response Response = m_HttpClient.Download(fmt::format("/api/v1/refs/{}/{}/{}", Namespace, BucketId, Key.ToHexString()),
+														  TempFolderPath,
+														  {{"Accept", "application/x-jupiter-inline"}});
+
+	JupiterResult Result = detail::ConvertResponse(Response);
+
+	if (auto It = Response.Header->find("X-Jupiter-InlinePayloadHash"); It != Response.Header->end())
+	{
+		const std::string& PayloadHashHeader = It->second;
+		if (PayloadHashHeader.length() == IoHash::StringLength)
+		{
+			OutPayloadHash = IoHash::FromHexString(PayloadHashHeader);
+		}
+	}
+
+	return Result;
+}
+
+JupiterResult
+JupiterSession::GetObject(std::string_view Namespace, const IoHash& Key)
+{
+	ZEN_TRACE_CPU("JupiterClient::GetObject");
+
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v1/objects/{}/{}", Namespace, Key.ToHexString()),
+													 {HttpClient::Accept(ZenContentType::kCbObject)});
+
+	return detail::ConvertResponse(Response);
+}
+
+PutRefResult
+JupiterSession::PutRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType)
+{
+	ZEN_TRACE_CPU("JupiterClient::PutRef");
+
+	Ref.SetContentType(RefType);
+
+	IoHash Hash = IoHash::HashBuffer(Ref.Data(), Ref.Size());
+
+	HttpClient::Response Response = m_HttpClient.Put(fmt::format("/api/v1/refs/{}/{}/{}", Namespace, BucketId, Key.ToHexString()),
+													 Ref,
+													 {{"X-Jupiter-IoHash", Hash.ToHexString()}});
+
+	PutRefResult Result = {detail::ConvertResponse(Response)};
+	if (Result.Success)
+	{
+		std::string	 JsonError;
+		json11::Json Json = json11::Json::parse(Response.ToText(), JsonError);
+		if (JsonError.empty())
+		{
+			json11::Json::array Needs = Json["needs"].array_items();
+			for (const auto& Need : Needs)
+			{
+				Result.Needs.emplace_back(IoHash::FromHexString(Need.string_value()));
+			}
+		}
+		Result.RawHash = Hash;
+	}
+	return Result;
+}
+
+FinalizeRefResult
+JupiterSession::FinalizeRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& RefHash)
+{
+	ZEN_TRACE_CPU("JupiterClient::FinalizeRef");
+
+	HttpClient::Response Response =
+		m_HttpClient.Post(fmt::format("/api/v1/refs/{}/{}/{}/finalize/{}", Namespace, BucketId, Key.ToHexString(), RefHash.ToHexString()),
+						  {{"X-Jupiter-IoHash", RefHash.ToHexString()}, {"Content-Type", "application/x-ue-cb"}});
+
+	FinalizeRefResult Result = {detail::ConvertResponse(Response)};
+
+	if (Result.Success)
+	{
+		std::string	 JsonError;
+		json11::Json Json = json11::Json::parse(std::string(Response.ToText()), JsonError);
+		if (JsonError.empty())
+		{
+			json11::Json::array Needs = Json["needs"].array_items();
+			for (const auto& Need : Needs)
+			{
+				Result.Needs.emplace_back(IoHash::FromHexString(Need.string_value()));
+			}
+		}
+	}
+	return Result;
+}
+
+JupiterResult
+JupiterSession::PutBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob)
+{
+	ZEN_TRACE_CPU("JupiterClient::PutBlob");
+
+	HttpClient::Response Response = m_HttpClient.Put(fmt::format("/api/v1/blobs/{}/{}", Namespace, Key.ToHexString()), Blob);
+
+	return detail::ConvertResponse(Response);
+}
+
+JupiterResult
+JupiterSession::PutCompressedBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob)
+{
+	ZEN_TRACE_CPU("JupiterClient::PutCompressedBlob");
+
+	Blob.SetContentType(ZenContentType::kCompressedBinary);
+	HttpClient::Response Response = m_HttpClient.Upload(fmt::format("/api/v1/compressed-blobs/{}/{}", Namespace, Key.ToHexString()), Blob);
+
+	return detail::ConvertResponse(Response);
+}
+
+JupiterResult
+JupiterSession::PutCompressedBlob(std::string_view Namespace, const IoHash& Key, const CompositeBuffer& Payload)
+{
+	ZEN_TRACE_CPU("JupiterClient::PutCompressedBlob");
+
+	HttpClient::Response Response = m_HttpClient.Upload(fmt::format("/api/v1/compressed-blobs/{}/{}", Namespace, Key.ToHexString()),
+														Payload,
+														ZenContentType::kCompressedBinary);
+
+	return detail::ConvertResponse(Response);
+}
+
+JupiterResult
+JupiterSession::PutObject(std::string_view Namespace, const IoHash& Key, IoBuffer Object)
+{
+	ZEN_TRACE_CPU("JupiterClient::PutObject");
+
+	Object.SetContentType(ZenContentType::kCbObject);
+	HttpClient::Response Response = m_HttpClient.Upload(fmt::format("/api/v1/objects/{}/{}", Namespace, Key.ToHexString()), Object);
+
+	return detail::ConvertResponse(Response);
+}
+
+JupiterResult
+JupiterSession::RefExists(std::string_view Namespace, std::string_view BucketId, const IoHash& Key)
+{
+	ZEN_TRACE_CPU("JupiterClient::RefExists");
+
+	HttpClient::Response Response = m_HttpClient.Head(fmt::format("/api/v1/refs/{}/{}/{}", Namespace, BucketId, Key.ToHexString()));
+
+	return detail::ConvertResponse(Response);
+}
+
+GetObjectReferencesResult
+JupiterSession::GetObjectReferences(std::string_view Namespace, const IoHash& Key)
+{
+	ZEN_TRACE_CPU("JupiterClient::GetObjectReferences");
+
+	HttpClient::Response Response = m_HttpClient.Head(fmt::format("/api/v1/refs/{}/{}/references", Namespace, Key.ToHexString()),
+													  {HttpClient::Accept(ZenContentType::kCbObject)});
+
+	GetObjectReferencesResult Result = {detail::ConvertResponse(Response)};
+
+	if (Result.Success)
+	{
+		const CbObject ReferencesResponse = Response.AsObject();
+		for (auto& Item : ReferencesResponse["references"sv])
+		{
+			Result.References.insert(Item.AsHash());
+		}
+	}
+	return Result;
+}
+
+JupiterResult
+JupiterSession::BlobExists(std::string_view Namespace, const IoHash& Key)
+{
+	return CacheTypeExists(Namespace, "blobs"sv, Key);
+}
+
+JupiterResult
+JupiterSession::CompressedBlobExists(std::string_view Namespace, const IoHash& Key)
+{
+	return CacheTypeExists(Namespace, "compressed-blobs"sv, Key);
+}
+
+JupiterResult
+JupiterSession::ObjectExists(std::string_view Namespace, const IoHash& Key)
+{
+	return CacheTypeExists(Namespace, "objects"sv, Key);
+}
+
+JupiterExistsResult
+JupiterSession::BlobExists(std::string_view Namespace, const std::set<IoHash>& Keys)
+{
+	return CacheTypeExists(Namespace, "blobs"sv, Keys);
+}
+
+JupiterExistsResult
+JupiterSession::CompressedBlobExists(std::string_view Namespace, const std::set<IoHash>& Keys)
+{
+	return CacheTypeExists(Namespace, "compressed-blobs"sv, Keys);
+}
+
+JupiterExistsResult
+JupiterSession::ObjectExists(std::string_view Namespace, const std::set<IoHash>& Keys)
+{
+	return CacheTypeExists(Namespace, "objects"sv, Keys);
+}
+
+std::vector<IoHash>
+JupiterSession::Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes)
+{
+	//	ExtendableStringBuilder<256> Uri;
+	//	Uri << m_CacheClient->ServiceUrl();
+	//	Uri << "/api/v1/s/" << Namespace;
+
+	ZEN_UNUSED(Namespace, BucketId, ChunkHashes);
+
+	return {};
+}
+
+JupiterResult
+JupiterSession::CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const IoHash& Key)
+{
+	ZEN_TRACE_CPU("JupiterClient::CacheTypeExists");
+
+	HttpClient::Response Response = m_HttpClient.Head(fmt::format("/api/v1/{}/{}/{}", TypeId, Namespace, Key.ToHexString()));
+
+	return detail::ConvertResponse(Response);
+}
+
+JupiterExistsResult
+JupiterSession::CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const std::set<IoHash>& Keys)
+{
+	ZEN_TRACE_CPU("JupiterClient::CacheTypeExists");
+
+	ExtendableStringBuilder<256> Body;
+	Body << "[";
+	for (const auto& Key : Keys)
+	{
+		Body << (Body.Size() != 1 ? ",\"" : "\"") << Key.ToHexString() << "\"";
+	}
+	Body << "]";
+	IoBuffer Payload = IoBuffer(IoBuffer::Wrap, Body.Data(), Body.Size());
+	Payload.SetContentType(ZenContentType::kJSON);
+
+	HttpClient::Response Response =
+		m_HttpClient.Post(fmt::format("/api/v1/{}/{}/exist", TypeId, Namespace), Payload, {HttpClient::Accept(ZenContentType::kCbObject)});
+
+	JupiterExistsResult Result = {detail::ConvertResponse(Response)};
+
+	if (Result.Success)
+	{
+		const CbObject ExistsResponse = Response.AsObject();
+		for (auto& Item : ExistsResponse["needs"sv])
+		{
+			Result.Needs.insert(Item.AsHash());
+		}
+	}
+	return Result;
+}
+
+JupiterResult
+JupiterSession::PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+	HttpClient::Response Response = m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}", Namespace, BucketId, BuildId), Payload);
+	return detail::ConvertResponse(Response, "JupiterSession::PutBuild"sv);
+}
+
+JupiterResult
+JupiterSession::GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId)
+{
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}", Namespace, BucketId, BuildId),
+													 HttpClient::Accept(ZenContentType::kCbObject));
+	return detail::ConvertResponse(Response, "JupiterSession::GetBuild"sv);
+}
+
+JupiterResult
+JupiterSession::FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId)
+{
+	HttpClient::Response Response = m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/finalize", Namespace, BucketId, BuildId));
+	return detail::ConvertResponse(Response, "JupiterSession::FinalizeBuild"sv);
+}
+
+PutBuildPartResult
+JupiterSession::PutBuildPart(std::string_view Namespace,
+							 std::string_view BucketId,
+							 const Oid&		  BuildId,
+							 const Oid&		  PartId,
+							 std::string_view PartName,
+							 const IoBuffer&  Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+
+	IoHash Hash = IoHash::HashBuffer(Payload.Data(), Payload.Size());
+
+	HttpClient::Response Response =
+		m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/{}", Namespace, BucketId, BuildId, PartId, PartName),
+						 Payload,
+						 {{"X-Jupiter-IoHash", Hash.ToHexString()}});
+
+	PutBuildPartResult Result = {detail::ConvertResponse(Response, "JupiterSession::PutBuildPart"sv)};
+	if (Result.Success)
+	{
+		std::string	 JsonError;
+		json11::Json Json = json11::Json::parse(Response.ToText(), JsonError);
+		if (JsonError.empty())
+		{
+			json11::Json::array Needs = Json["needs"].array_items();
+			for (const auto& Need : Needs)
+			{
+				Result.Needs.emplace_back(IoHash::FromHexString(Need.string_value()));
+			}
+		}
+		Result.RawHash = Hash;
+	}
+	return Result;
+}
+
+JupiterResult
+JupiterSession::GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId)
+{
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}", Namespace, BucketId, BuildId, PartId),
+													 HttpClient::Accept(ZenContentType::kCbObject));
+	return detail::ConvertResponse(Response, "JupiterSession::GetBuildPart"sv);
+}
+
+JupiterResult
+JupiterSession::PutBuildBlob(std::string_view		Namespace,
+							 std::string_view		BucketId,
+							 const Oid&				BuildId,
+							 const Oid&				PartId,
+							 const IoHash&			Hash,
+							 ZenContentType			ContentType,
+							 const CompositeBuffer& Payload)
+{
+	HttpClient::Response Response = m_HttpClient.Upload(
+		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
+		Payload,
+		ContentType);
+	return detail::ConvertResponse(Response, "JupiterSession::PutBuildBlob"sv);
+}
+
+JupiterResult
+JupiterSession::GetBuildBlob(std::string_view	   Namespace,
+							 std::string_view	   BucketId,
+							 const Oid&			   BuildId,
+							 const Oid&			   PartId,
+							 const IoHash&		   Hash,
+							 std::filesystem::path TempFolderPath)
+{
+	HttpClient::Response Response = m_HttpClient.Download(
+		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
+		TempFolderPath);
+	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
+}
+
+JupiterResult
+JupiterSession::PutBlockMetadata(std::string_view Namespace,
+								 std::string_view BucketId,
+								 const Oid&		  BuildId,
+								 const Oid&		  PartId,
+								 const IoHash&	  Hash,
+								 const IoBuffer&  Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+	HttpClient::Response Response = m_HttpClient.Put(
+		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
+		Payload);
+	return detail::ConvertResponse(Response, "JupiterSession::PutBlockMetadata"sv);
+}
+
+FinalizeBuildPartResult
+JupiterSession::FinalizeBuildPart(std::string_view Namespace,
+								  std::string_view BucketId,
+								  const Oid&	   BuildId,
+								  const Oid&	   PartId,
+								  const IoHash&	   RawHash)
+{
+	HttpClient::Response Response = m_HttpClient.Post(
+		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/finalize/{}", Namespace, BucketId, BuildId, PartId, RawHash.ToHexString()),
+		HttpClient::Accept(ZenContentType::kCbObject));
+
+	FinalizeBuildPartResult Result = {detail::ConvertResponse(Response, "JupiterSession::FinalizeBuildPart"sv)};
+	if (Result.Success)
+	{
+		std::string	 JsonError;
+		json11::Json Json = json11::Json::parse(Response.ToText(), JsonError);
+		if (JsonError.empty())
+		{
+			json11::Json::array Needs = Json["needs"].array_items();
+			for (const auto& Need : Needs)
+			{
+				Result.Needs.emplace_back(IoHash::FromHexString(Need.string_value()));
+			}
+		}
+	}
+	return Result;
+}
+
+JupiterResult
+JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId)
+{
+	HttpClient::Response Response =
+		m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/listBlocks", Namespace, BucketId, BuildId, PartId),
+						 HttpClient::Accept(ZenContentType::kCbObject));
+	return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv);
+}
+
+}  // namespace zen