move remoteproject to remotestorelib (#542)

* move remoteproject code to remotestorelib

5 files changed, 280 insertions, 0 deletions

diff --git a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
new file mode 100644
index 000000000..fc59e41f4
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
@@ -0,0 +1,32 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenremotestore/projectstore/remoteprojectstore.h>
+
+namespace zen {
+
+class AuthMgr;
+
+struct BuildsRemoteStoreOptions : RemoteStoreOptions
+{
+	std::string			  Url;
+	std::string			  Namespace;
+	std::string			  Bucket;
+	Oid					  BuildId;
+	std::string			  OpenIdProvider;
+	std::string			  AccessToken;
+	AuthMgr&			  AuthManager;
+	std::filesystem::path OidcExePath;
+	bool				  ForceDisableBlocks	 = false;
+	bool				  ForceDisableTempBlocks = false;
+	bool				  AssumeHttp2			 = false;
+	IoBuffer			  MetaData;
+};
+
+std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
+																   const std::filesystem::path&	   TempFilePath,
+																   bool							   Quiet,
+																   bool							   Unattended);
+
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h
new file mode 100644
index 000000000..ff2ecb405
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h
@@ -0,0 +1,20 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenremotestore/projectstore/remoteprojectstore.h>
+
+namespace zen {
+
+struct FileRemoteStoreOptions : RemoteStoreOptions
+{
+	std::filesystem::path FolderPath;
+	std::string			  Name;
+	std::string			  OptionalBaseName;
+	bool				  ForceDisableBlocks	= false;
+	bool				  ForceEnableTempBlocks = false;
+};
+
+std::shared_ptr<RemoteProjectStore> CreateFileRemoteStore(const FileRemoteStoreOptions& Options);
+
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h
new file mode 100644
index 000000000..d800005a9
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h
@@ -0,0 +1,32 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenremotestore/projectstore/remoteprojectstore.h>
+
+namespace zen {
+
+class AuthMgr;
+
+struct JupiterRemoteStoreOptions : RemoteStoreOptions
+{
+	std::string			  Url;
+	std::string			  Namespace;
+	std::string			  Bucket;
+	IoHash				  Key;
+	IoHash				  OptionalBaseKey;
+	std::string			  OpenIdProvider;
+	std::string			  AccessToken;
+	AuthMgr&			  AuthManager;
+	std::filesystem::path OidcExePath;
+	bool				  ForceDisableBlocks	 = false;
+	bool				  ForceDisableTempBlocks = false;
+	bool				  AssumeHttp2			 = false;
+};
+
+std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
+															 const std::filesystem::path&	  TempFilePath,
+															 bool							  Quiet,
+															 bool							  Unattended);
+
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
new file mode 100644
index 000000000..11cc58e4d
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
@@ -0,0 +1,178 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/jobqueue.h>
+#include <zenstore/projectstore.h>
+
+#include <zenutil/chunkblock.h>
+
+#include <unordered_set>
+
+namespace zen {
+
+class CidStore;
+class WorkerThreadPool;
+struct ChunkedInfo;
+
+class RemoteProjectStore
+{
+public:
+	struct Result
+	{
+		int32_t		ErrorCode{};
+		double		ElapsedSeconds{};
+		std::string Reason;
+		std::string Text;
+	};
+
+	struct CreateContainerResult : public Result
+	{
+	};
+
+	struct SaveResult : public Result
+	{
+		std::unordered_set<IoHash, IoHash::Hasher> Needs;
+		IoHash									   RawHash;
+	};
+
+	struct FinalizeResult : public Result
+	{
+		std::unordered_set<IoHash, IoHash::Hasher> Needs;
+	};
+
+	struct SaveAttachmentResult : public Result
+	{
+	};
+
+	struct SaveAttachmentsResult : public Result
+	{
+	};
+
+	struct LoadAttachmentResult : public Result
+	{
+		IoBuffer Bytes;
+	};
+
+	struct LoadContainerResult : public Result
+	{
+		CbObject ContainerObject;
+	};
+
+	struct LoadAttachmentsResult : public Result
+	{
+		std::vector<std::pair<IoHash, CompressedBuffer>> Chunks;
+	};
+
+	struct GetKnownBlocksResult : public Result
+	{
+		std::vector<ThinChunkBlockDescription> Blocks;
+	};
+
+	struct RemoteStoreInfo
+	{
+		bool		CreateBlocks;
+		bool		UseTempBlockFiles;
+		bool		AllowChunking;
+		std::string ContainerName;
+		std::string Description;
+	};
+
+	struct Stats
+	{
+		std::uint64_t m_SentBytes;
+		std::uint64_t m_ReceivedBytes;
+		std::uint64_t m_RequestTimeNS;
+		std::uint64_t m_RequestCount;
+		std::uint64_t m_PeakSentBytes;
+		std::uint64_t m_PeakReceivedBytes;
+		std::uint64_t m_PeakBytesPerSec;
+	};
+
+	RemoteProjectStore();
+	virtual ~RemoteProjectStore();
+
+	virtual RemoteStoreInfo GetInfo() const	 = 0;
+	virtual Stats			GetStats() const = 0;
+
+	virtual CreateContainerResult CreateContainer()																					   = 0;
+	virtual SaveResult			  SaveContainer(const IoBuffer& Payload)															   = 0;
+	virtual SaveAttachmentResult  SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&& Block) = 0;
+	virtual FinalizeResult		  FinalizeContainer(const IoHash& RawHash)															   = 0;
+	virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads)										   = 0;
+
+	virtual LoadContainerResult	  LoadContainer()										= 0;
+	virtual GetKnownBlocksResult  GetKnownBlocks()										= 0;
+	virtual LoadAttachmentResult  LoadAttachment(const IoHash& RawHash)					= 0;
+	virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0;
+};
+
+struct RemoteStoreOptions
+{
+	static const size_t DefaultMaxBlockSize		  = 64u * 1024u * 1024u;
+	static const size_t DefaultMaxChunksPerBlock  = 4u * 1000u;
+	static const size_t DefaultMaxChunkEmbedSize  = 3u * 512u * 1024u;
+	static const size_t DefaultChunkFileSizeLimit = 256u * 1024u * 1024u;
+
+	size_t MaxBlockSize		  = DefaultMaxBlockSize;
+	size_t MaxChunksPerBlock  = DefaultMaxChunksPerBlock;
+	size_t MaxChunkEmbedSize  = DefaultMaxChunkEmbedSize;
+	size_t ChunkFileSizeLimit = DefaultChunkFileSizeLimit;
+};
+
+typedef std::function<IoBuffer(const IoHash& AttachmentHash)> TGetAttachmentBufferFunc;
+
+RemoteProjectStore::LoadContainerResult BuildContainer(
+	CidStore&																	 ChunkStore,
+	ProjectStore::Project&														 Project,
+	ProjectStore::Oplog&														 Oplog,
+	size_t																		 MaxBlockSize,
+	size_t																		 MaxChunksPerBlock,
+	size_t																		 MaxChunkEmbedSize,
+	size_t																		 ChunkFileSizeLimit,
+	bool																		 BuildBlocks,
+	bool																		 IgnoreMissingAttachments,
+	bool																		 AllowChunking,
+	const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>&		 AsyncOnBlock,
+	const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>&		 OnLargeAttachment,
+	const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks,
+	bool																		 EmbedLooseFiles);
+
+class JobContext;
+
+RemoteProjectStore::Result SaveOplogContainer(ProjectStore::Oplog&									  Oplog,
+											  const CbObject&										  ContainerObject,
+											  const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments,
+											  const std::function<bool(const IoHash& RawHash)>&		  HasAttachment,
+											  const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock,
+											  const std::function<void(const IoHash& RawHash)>&		 OnNeedAttachment,
+											  const std::function<void(const ChunkedInfo& Chunked)>& OnChunkedAttachment,
+											  JobContext*											 OptionalContext);
+
+RemoteProjectStore::Result SaveOplog(CidStore&				ChunkStore,
+									 RemoteProjectStore&	RemoteStore,
+									 ProjectStore::Project& Project,
+									 ProjectStore::Oplog&	Oplog,
+									 size_t					MaxBlockSize,
+									 size_t					MaxChunksPerBlock,
+									 size_t					MaxChunkEmbedSize,
+									 size_t					ChunkFileSizeLimit,
+									 bool					EmbedLooseFiles,
+									 bool					ForceUpload,
+									 bool					IgnoreMissingAttachments,
+									 JobContext*			OptionalContext);
+
+RemoteProjectStore::Result LoadOplog(CidStore&			  ChunkStore,
+									 RemoteProjectStore&  RemoteStore,
+									 ProjectStore::Oplog& Oplog,
+									 bool				  ForceDownload,
+									 bool				  IgnoreMissingAttachments,
+									 bool				  CleanOplog,
+									 JobContext*		  OptionalContext);
+
+std::vector<IoHash>					   GetBlockHashesFromOplog(CbObjectView ContainerObject);
+std::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes);
+
+void remoteprojectstore_forcelink();
+
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h
new file mode 100644
index 000000000..d6b62c563
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h
@@ -0,0 +1,18 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenremotestore/projectstore/remoteprojectstore.h>
+
+namespace zen {
+
+struct ZenRemoteStoreOptions : RemoteStoreOptions
+{
+	std::string Url;
+	std::string ProjectId;
+	std::string OplogId;
+};
+
+std::shared_ptr<RemoteProjectStore> CreateZenRemoteStore(const ZenRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath);
+
+}  // namespace zen