Merge remote-tracking branch 'origin/main' into de/zen-service-command

3 files changed, 209 insertions, 256 deletions

diff --git a/src/zenutil/include/zenutil/jupiter.h b/src/zenutil/include/zenutil/jupiter.h
deleted file mode 100644
index 50e4ad68a..000000000
--- a/src/zenutil/include/zenutil/jupiter.h
+++ /dev/null
@@ -1,256 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zenbase/refcount.h>
-#include <zencore/iohash.h>
-#include <zencore/logging.h>
-#include <zencore/thread.h>
-#include <zenhttp/httpclient.h>
-#include <zenhttp/httpserver.h>
-
-#include <atomic>
-#include <chrono>
-#include <list>
-#include <memory>
-#include <set>
-#include <vector>
-
-struct ZenCacheValue;
-
-namespace cpr {
-class Session;
-}
-
-namespace zen {
-
-class CbObjectView;
-class CloudCacheClient;
-class IoBuffer;
-struct IoHash;
-
-/**
- * Cached access token, for use with `Authorization:` header
- */
-struct CloudCacheAccessToken
-{
-	using Clock		= std::chrono::system_clock;
-	using TimePoint = Clock::time_point;
-
-	static constexpr int64_t ExpireMarginInSeconds = 30;
-
-	std::string Value;
-	TimePoint	ExpireTime;
-
-	bool IsValid() const
-	{
-		return Value.empty() == false &&
-			   ExpireMarginInSeconds < std::chrono::duration_cast<std::chrono::seconds>(ExpireTime - Clock::now()).count();
-	}
-};
-
-struct CloudCacheResult
-{
-	IoBuffer	Response;
-	uint64_t	SentBytes{};
-	uint64_t	ReceivedBytes{};
-	double		ElapsedSeconds{};
-	int32_t		ErrorCode{};
-	std::string Reason;
-	bool		Success = false;
-};
-
-struct PutRefResult : CloudCacheResult
-{
-	std::vector<IoHash> Needs;
-	IoHash				RawHash;
-};
-
-struct FinalizeRefResult : CloudCacheResult
-{
-	std::vector<IoHash> Needs;
-};
-
-struct CloudCacheExistsResult : CloudCacheResult
-{
-	std::set<IoHash> Needs;
-};
-
-struct GetObjectReferencesResult : CloudCacheResult
-{
-	std::set<IoHash> References;
-};
-
-struct PutBuildPartResult : CloudCacheResult
-{
-	std::vector<IoHash> Needs;
-	IoHash				RawHash;
-};
-
-struct FinalizeBuildPartResult : CloudCacheResult
-{
-	std::vector<IoHash> Needs;
-};
-
-/**
- * Context for performing Jupiter operations
- *
- * Maintains an HTTP connection so that subsequent operations don't need to go
- * through the whole connection setup process
- *
- */
-class CloudCacheSession
-{
-public:
-	CloudCacheSession(CloudCacheClient* CacheClient);
-	~CloudCacheSession();
-
-	CloudCacheResult Authenticate();
-
-	CloudCacheResult GetRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType RefType);
-	CloudCacheResult GetBlob(std::string_view Namespace, const IoHash& Key);
-	CloudCacheResult GetCompressedBlob(std::string_view Namespace, const IoHash& Key, std::filesystem::path TempFolderPath = {});
-	CloudCacheResult GetObject(std::string_view Namespace, const IoHash& Key);
-	CloudCacheResult GetInlineBlob(std::string_view		 Namespace,
-								   std::string_view		 BucketId,
-								   const IoHash&		 Key,
-								   IoHash&				 OutPayloadHash,
-								   std::filesystem::path TempFolderPath = {});
-
-	PutRefResult	 PutRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType);
-	CloudCacheResult PutBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob);
-	CloudCacheResult PutCompressedBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob);
-	CloudCacheResult PutCompressedBlob(std::string_view Namespace, const IoHash& Key, const CompositeBuffer& Blob);
-	CloudCacheResult PutObject(std::string_view Namespace, const IoHash& Key, IoBuffer Object);
-
-	FinalizeRefResult FinalizeRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& RefHah);
-
-	CloudCacheResult RefExists(std::string_view Namespace, std::string_view BucketId, const IoHash& Key);
-
-	GetObjectReferencesResult GetObjectReferences(std::string_view Namespace, const IoHash& Key);
-
-	CloudCacheResult BlobExists(std::string_view Namespace, const IoHash& Key);
-	CloudCacheResult CompressedBlobExists(std::string_view Namespace, const IoHash& Key);
-	CloudCacheResult ObjectExists(std::string_view Namespace, const IoHash& Key);
-
-	CloudCacheExistsResult BlobExists(std::string_view Namespace, const std::set<IoHash>& Keys);
-	CloudCacheExistsResult CompressedBlobExists(std::string_view Namespace, const std::set<IoHash>& Keys);
-	CloudCacheExistsResult ObjectExists(std::string_view Namespace, const std::set<IoHash>& Keys);
-
-	std::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes);
-
-	CloudCacheResult		PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload);
-	CloudCacheResult		GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
-	CloudCacheResult		FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
-	PutBuildPartResult		PutBuildPart(std::string_view Namespace,
-										 std::string_view BucketId,
-										 const Oid&		  BuildId,
-										 const Oid&		  PartId,
-										 std::string_view PartName,
-										 const IoBuffer&  Payload);
-	CloudCacheResult		GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
-	CloudCacheResult		PutBuildBlob(std::string_view		Namespace,
-										 std::string_view		BucketId,
-										 const Oid&				BuildId,
-										 const Oid&				PartId,
-										 const IoHash&			Hash,
-										 ZenContentType			ContentType,
-										 const CompositeBuffer& Payload);
-	CloudCacheResult		GetBuildBlob(std::string_view	   Namespace,
-										 std::string_view	   BucketId,
-										 const Oid&			   BuildId,
-										 const Oid&			   PartId,
-										 const IoHash&		   Hash,
-										 std::filesystem::path TempFolderPath);
-	CloudCacheResult		PutBlockMetadata(std::string_view Namespace,
-											 std::string_view BucketId,
-											 const Oid&		  BuildId,
-											 const Oid&		  PartId,
-											 const IoHash&	  Hash,
-											 const IoBuffer&  Payload);
-	FinalizeBuildPartResult FinalizeBuildPart(std::string_view Namespace,
-											  std::string_view BucketId,
-											  const Oid&	   BuildId,
-											  const Oid&	   PartId,
-											  const IoHash&	   RawHash);
-	CloudCacheResult		FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
-
-	CloudCacheClient& Client() { return *m_CacheClient; };
-
-private:
-	inline LoggerRef Log() { return m_Log; }
-
-	CloudCacheResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const IoHash& Key);
-
-	CloudCacheExistsResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const std::set<IoHash>& Keys);
-
-	LoggerRef				 m_Log;
-	RefPtr<CloudCacheClient> m_CacheClient;
-};
-
-/**
- * Access token provider interface
- */
-class CloudCacheTokenProvider
-{
-public:
-	virtual ~CloudCacheTokenProvider() = default;
-
-	virtual CloudCacheAccessToken AcquireAccessToken() = 0;
-
-	static std::unique_ptr<CloudCacheTokenProvider> CreateFromStaticToken(CloudCacheAccessToken Token);
-
-	struct OAuthClientCredentialsParams
-	{
-		std::string_view Url;
-		std::string_view ClientId;
-		std::string_view ClientSecret;
-	};
-
-	static std::unique_ptr<CloudCacheTokenProvider> CreateFromOAuthClientCredentials(const OAuthClientCredentialsParams& Params);
-
-	static std::unique_ptr<CloudCacheTokenProvider> CreateFromCallback(std::function<CloudCacheAccessToken()>&& Callback);
-};
-
-struct CloudCacheClientOptions
-{
-	std::string_view		  Name;
-	std::string_view		  ServiceUrl;
-	std::string_view		  DdcNamespace;
-	std::string_view		  BlobStoreNamespace;
-	std::string_view		  ComputeCluster;
-	std::chrono::milliseconds ConnectTimeout{5000};
-	std::chrono::milliseconds Timeout{};
-	bool					  AssumeHttp2 = false;
-	bool					  AllowResume = false;
-	uint8_t					  RetryCount  = 0;
-};
-
-/**
- * Jupiter upstream cache client
- */
-class CloudCacheClient : public RefCounted
-{
-public:
-	CloudCacheClient(const CloudCacheClientOptions& Options, std::unique_ptr<CloudCacheTokenProvider> TokenProvider);
-	~CloudCacheClient();
-
-	std::string_view DefaultDdcNamespace() const { return m_DefaultDdcNamespace; }
-	std::string_view DefaultBlobStoreNamespace() const { return m_DefaultBlobStoreNamespace; }
-	std::string_view ComputeCluster() const { return m_ComputeCluster; }
-	std::string_view ServiceUrl() const { return m_HttpClient.GetBaseUri(); }
-
-	LoggerRef Logger() { return m_Log; }
-
-private:
-	LoggerRef									   m_Log;
-	const std::string							   m_DefaultDdcNamespace;
-	const std::string							   m_DefaultBlobStoreNamespace;
-	const std::string							   m_ComputeCluster;
-	const std::unique_ptr<CloudCacheTokenProvider> m_TokenProvider;
-	HttpClient									   m_HttpClient;
-
-	friend class CloudCacheSession;
-};
-
-}  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupiterclient.h b/src/zenutil/include/zenutil/jupiter/jupiterclient.h
new file mode 100644
index 000000000..defe50edc
--- /dev/null
+++ b/src/zenutil/include/zenutil/jupiter/jupiterclient.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/refcount.h>
+#include <zencore/logging.h>
+#include <zenhttp/httpclient.h>
+
+#include <chrono>
+
+namespace zen {
+
+class IoBuffer;
+
+struct JupiterClientOptions
+{
+	std::string_view		  Name;
+	std::string_view		  ServiceUrl;
+	std::string_view		  DdcNamespace;
+	std::string_view		  BlobStoreNamespace;
+	std::string_view		  ComputeCluster;
+	std::chrono::milliseconds ConnectTimeout{5000};
+	std::chrono::milliseconds Timeout{};
+	bool					  AssumeHttp2 = false;
+	bool					  AllowResume = false;
+	uint8_t					  RetryCount  = 0;
+};
+
+/**
+ * Jupiter upstream cache client
+ */
+class JupiterClient : public RefCounted
+{
+public:
+	JupiterClient(const JupiterClientOptions& Options, std::function<HttpClientAccessToken()>&& TokenProvider);
+	~JupiterClient();
+
+	std::string_view DefaultDdcNamespace() const { return m_DefaultDdcNamespace; }
+	std::string_view DefaultBlobStoreNamespace() const { return m_DefaultBlobStoreNamespace; }
+	std::string_view ComputeCluster() const { return m_ComputeCluster; }
+	std::string_view ServiceUrl() const { return m_HttpClient.GetBaseUri(); }
+
+	LoggerRef	Logger() { return m_Log; }
+	HttpClient& Client() { return m_HttpClient; }
+
+private:
+	LoggerRef							   m_Log;
+	const std::string					   m_DefaultDdcNamespace;
+	const std::string					   m_DefaultBlobStoreNamespace;
+	const std::string					   m_ComputeCluster;
+	std::function<HttpClientAccessToken()> m_TokenProvider;
+	HttpClient							   m_HttpClient;
+
+	friend class JupiterSession;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
new file mode 100644
index 000000000..6a80332f4
--- /dev/null
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -0,0 +1,152 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/logging.h>
+#include <zenhttp/httpclient.h>
+
+#include <set>
+
+namespace zen {
+
+class IoBuffer;
+
+struct JupiterResult
+{
+	IoBuffer	Response;
+	uint64_t	SentBytes{};
+	uint64_t	ReceivedBytes{};
+	double		ElapsedSeconds{};
+	int32_t		ErrorCode{};
+	std::string Reason;
+	bool		Success = false;
+};
+
+struct PutRefResult : JupiterResult
+{
+	std::vector<IoHash> Needs;
+	IoHash				RawHash;
+};
+
+struct FinalizeRefResult : JupiterResult
+{
+	std::vector<IoHash> Needs;
+};
+
+struct JupiterExistsResult : JupiterResult
+{
+	std::set<IoHash> Needs;
+};
+
+struct GetObjectReferencesResult : JupiterResult
+{
+	std::set<IoHash> References;
+};
+
+struct PutBuildPartResult : JupiterResult
+{
+	std::vector<IoHash> Needs;
+	IoHash				RawHash;
+};
+
+struct FinalizeBuildPartResult : JupiterResult
+{
+	std::vector<IoHash> Needs;
+};
+
+/**
+ * Context for performing Jupiter operations
+ *
+ * Maintains an HTTP connection so that subsequent operations don't need to go
+ * through the whole connection setup process
+ *
+ */
+class JupiterSession
+{
+public:
+	JupiterSession(LoggerRef InLog, HttpClient& InHttpClient);
+	~JupiterSession();
+
+	JupiterResult Authenticate();
+
+	JupiterResult GetRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType RefType);
+	JupiterResult GetBlob(std::string_view Namespace, const IoHash& Key);
+	JupiterResult GetCompressedBlob(std::string_view Namespace, const IoHash& Key, std::filesystem::path TempFolderPath = {});
+	JupiterResult GetObject(std::string_view Namespace, const IoHash& Key);
+	JupiterResult GetInlineBlob(std::string_view	  Namespace,
+								std::string_view	  BucketId,
+								const IoHash&		  Key,
+								IoHash&				  OutPayloadHash,
+								std::filesystem::path TempFolderPath = {});
+
+	PutRefResult  PutRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType);
+	JupiterResult PutBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob);
+	JupiterResult PutCompressedBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob);
+	JupiterResult PutCompressedBlob(std::string_view Namespace, const IoHash& Key, const CompositeBuffer& Blob);
+	JupiterResult PutObject(std::string_view Namespace, const IoHash& Key, IoBuffer Object);
+
+	FinalizeRefResult FinalizeRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& RefHah);
+
+	JupiterResult RefExists(std::string_view Namespace, std::string_view BucketId, const IoHash& Key);
+
+	GetObjectReferencesResult GetObjectReferences(std::string_view Namespace, const IoHash& Key);
+
+	JupiterResult BlobExists(std::string_view Namespace, const IoHash& Key);
+	JupiterResult CompressedBlobExists(std::string_view Namespace, const IoHash& Key);
+	JupiterResult ObjectExists(std::string_view Namespace, const IoHash& Key);
+
+	JupiterExistsResult BlobExists(std::string_view Namespace, const std::set<IoHash>& Keys);
+	JupiterExistsResult CompressedBlobExists(std::string_view Namespace, const std::set<IoHash>& Keys);
+	JupiterExistsResult ObjectExists(std::string_view Namespace, const std::set<IoHash>& Keys);
+
+	std::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes);
+
+	JupiterResult			PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload);
+	JupiterResult			GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
+	JupiterResult			FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
+	PutBuildPartResult		PutBuildPart(std::string_view Namespace,
+										 std::string_view BucketId,
+										 const Oid&		  BuildId,
+										 const Oid&		  PartId,
+										 std::string_view PartName,
+										 const IoBuffer&  Payload);
+	JupiterResult			GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
+	JupiterResult			PutBuildBlob(std::string_view		Namespace,
+										 std::string_view		BucketId,
+										 const Oid&				BuildId,
+										 const Oid&				PartId,
+										 const IoHash&			Hash,
+										 ZenContentType			ContentType,
+										 const CompositeBuffer& Payload);
+	JupiterResult			GetBuildBlob(std::string_view	   Namespace,
+										 std::string_view	   BucketId,
+										 const Oid&			   BuildId,
+										 const Oid&			   PartId,
+										 const IoHash&		   Hash,
+										 std::filesystem::path TempFolderPath);
+	JupiterResult			PutBlockMetadata(std::string_view Namespace,
+											 std::string_view BucketId,
+											 const Oid&		  BuildId,
+											 const Oid&		  PartId,
+											 const IoHash&	  Hash,
+											 const IoBuffer&  Payload);
+	FinalizeBuildPartResult FinalizeBuildPart(std::string_view Namespace,
+											  std::string_view BucketId,
+											  const Oid&	   BuildId,
+											  const Oid&	   PartId,
+											  const IoHash&	   RawHash);
+	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
+
+private:
+	inline LoggerRef Log() { return m_Log; }
+
+	JupiterResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const IoHash& Key);
+
+	JupiterExistsResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const std::set<IoHash>& Keys);
+
+	LoggerRef	m_Log;
+	HttpClient& m_HttpClient;
+};
+
+}  // namespace zen