moved source directories into `/src` (#264)

* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees

1 files changed, 244 insertions, 0 deletions

diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
new file mode 100644
index 000000000..66cf3c4f8
--- /dev/null
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -0,0 +1,244 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "jupiterremoteprojectstore.h"
+
+#include <zencore/compress.h>
+#include <zencore/fmtutils.h>
+
+#include <auth/authmgr.h>
+#include <upstream/jupiter.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <cpr/cpr.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+class JupiterRemoteStore : public RemoteProjectStore
+{
+public:
+	JupiterRemoteStore(Ref<CloudCacheClient>&& CloudClient,
+					   std::string_view		   Namespace,
+					   std::string_view		   Bucket,
+					   const IoHash&		   Key,
+					   bool					   ForceDisableBlocks,
+					   bool					   ForceDisableTempBlocks)
+	: m_CloudClient(CloudClient)
+	, m_Namespace(Namespace)
+	, m_Bucket(Bucket)
+	, m_Key(Key)
+	{
+		if (ForceDisableBlocks)
+		{
+			m_EnableBlocks = false;
+		}
+		if (ForceDisableTempBlocks)
+		{
+			m_UseTempBlocks = false;
+		}
+	}
+
+	virtual RemoteStoreInfo GetInfo() const override
+	{
+		return {.CreateBlocks	   = m_EnableBlocks,
+				.UseTempBlockFiles = m_UseTempBlocks,
+				.Description	   = fmt::format("[cloud] {} as {}/{}/{}"sv, m_CloudClient->ServiceUrl(), m_Namespace, m_Bucket, m_Key)};
+	}
+
+	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
+	{
+		const int32_t MaxAttempts = 3;
+		PutRefResult  Result;
+		{
+			CloudCacheSession Session(m_CloudClient.Get());
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			{
+				Result = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
+			}
+		}
+
+		return SaveResult{ConvertResult(Result), {Result.Needs.begin(), Result.Needs.end()} /*, {}*/, IoHash::HashBuffer(Payload)};
+	}
+
+	virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override
+	{
+		const int32_t	 MaxAttempts = 3;
+		CloudCacheResult Result;
+		{
+			CloudCacheSession Session(m_CloudClient.Get());
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			{
+				Result = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
+			}
+		}
+
+		return SaveAttachmentResult{ConvertResult(Result)};
+	}
+
+	virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override
+	{
+		SaveAttachmentsResult Result;
+		for (const SharedBuffer& Chunk : Chunks)
+		{
+			CompressedBuffer	 Compressed	 = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer());
+			SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash());
+			if (ChunkResult.ErrorCode)
+			{
+				return SaveAttachmentsResult{ChunkResult};
+			}
+		}
+		return Result;
+	}
+
+	virtual Result FinalizeContainer(const IoHash& RawHash) override
+	{
+		const int32_t	 MaxAttempts = 3;
+		CloudCacheResult Result;
+		{
+			CloudCacheSession Session(m_CloudClient.Get());
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			{
+				Result = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
+			}
+		}
+		return ConvertResult(Result);
+	}
+
+	virtual LoadContainerResult LoadContainer() override
+	{
+		const int32_t	 MaxAttempts = 3;
+		CloudCacheResult Result;
+		{
+			CloudCacheSession Session(m_CloudClient.Get());
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			{
+				Result = Session.GetRef(m_Namespace, m_Bucket, m_Key, ZenContentType::kCbObject);
+			}
+		}
+
+		if (Result.ErrorCode || !Result.Success)
+		{
+			return LoadContainerResult{ConvertResult(Result)};
+		}
+
+		CbObject ContainerObject = LoadCompactBinaryObject(Result.Response);
+		if (!ContainerObject)
+		{
+			return LoadContainerResult{
+				RemoteProjectStore::Result{
+					.ErrorCode		= gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
+					.ElapsedSeconds = Result.ElapsedSeconds,
+					.Reason = fmt::format("The ref {}/{}/{} is not formatted as a compact binary object"sv, m_Namespace, m_Bucket, m_Key)},
+				std::move(ContainerObject)};
+		}
+
+		return LoadContainerResult{ConvertResult(Result), std::move(ContainerObject)};
+	}
+
+	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
+	{
+		const int32_t	 MaxAttempts = 3;
+		CloudCacheResult Result;
+		{
+			CloudCacheSession Session(m_CloudClient.Get());
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			{
+				Result = Session.GetCompressedBlob(m_Namespace, RawHash);
+			}
+		}
+		return LoadAttachmentResult{ConvertResult(Result), std::move(Result.Response)};
+	}
+
+	virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override
+	{
+		LoadAttachmentsResult Result;
+		for (const IoHash& Hash : RawHashes)
+		{
+			LoadAttachmentResult ChunkResult = LoadAttachment(Hash);
+			if (ChunkResult.ErrorCode)
+			{
+				return LoadAttachmentsResult{ChunkResult};
+			}
+			ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000)));
+			Result.Chunks.emplace_back(
+				std::pair<IoHash, CompressedBuffer>{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))});
+		}
+		return Result;
+	}
+
+private:
+	static Result ConvertResult(const CloudCacheResult& Response)
+	{
+		std::string Text;
+		int32_t		ErrorCode = 0;
+		if (Response.ErrorCode != 0)
+		{
+			ErrorCode = Response.ErrorCode;
+		}
+		else if (!Response.Success)
+		{
+			ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+			if (Response.Response.GetContentType() == ZenContentType::kText)
+			{
+				Text =
+					std::string(reinterpret_cast<const std::string::value_type*>(Response.Response.GetData()), Response.Response.GetSize());
+			}
+		}
+		return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text};
+	}
+
+	Ref<CloudCacheClient> m_CloudClient;
+	const std::string	  m_Namespace;
+	const std::string	  m_Bucket;
+	const IoHash		  m_Key;
+	bool				  m_EnableBlocks  = true;
+	bool				  m_UseTempBlocks = true;
+};
+
+std::unique_ptr<RemoteProjectStore>
+CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options)
+{
+	std::string Url = Options.Url;
+	if (Url.find("://"sv) == std::string::npos)
+	{
+		// Assume https URL
+		Url = fmt::format("https://{}"sv, Url);
+	}
+	CloudCacheClientOptions ClientOptions{.Name			  = "Remote store"sv,
+										  .ServiceUrl	  = Url,
+										  .ConnectTimeout = std::chrono::milliseconds(2000),
+										  .Timeout		  = std::chrono::milliseconds(60000)};
+	// 1) Access token as parameter in request
+	// 2) Environment variable (different win vs linux/mac)
+	// 3) openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
+
+	std::unique_ptr<CloudCacheTokenProvider> TokenProvider;
+	if (!Options.AccessToken.empty())
+	{
+		TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = Options.AccessToken]() {
+			return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()};
+		});
+	}
+	else
+	{
+		TokenProvider =
+			CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() {
+				AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider);
+				return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
+			});
+	}
+
+	Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider)));
+
+	std::unique_ptr<RemoteProjectStore> RemoteStore = std::make_unique<JupiterRemoteStore>(std::move(CloudClient),
+																						   Options.Namespace,
+																						   Options.Bucket,
+																						   Options.Key,
+																						   Options.ForceDisableBlocks,
+																						   Options.ForceDisableTempBlocks);
+	return RemoteStore;
+}
+
+}  // namespace zen