Merge branch 'main' of https://github.com/EpicGames/zen

6 files changed, 44 insertions, 16 deletions

diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index c356b9f0b..904e613a4 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -146,6 +146,13 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
 
 	options.add_option("cache",
 					   "",
+					   "upstream-jupiter-use-legacy-ddc",
+					   "Whether to store derived data using the legacy endpoint",
+					   cxxopts::value<bool>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc)->default_value("false"),
+					   "");
+
+	options.add_option("cache",
+					   "",
 					   "upstream-zen-url",
 					   "URL to a remote Zen server instance",
 					   cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.ZenConfig.Url)->default_value(""),
@@ -278,6 +285,9 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv
 					ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings =
 						JupiterConfig->get_or("usedevelopmentsettings",
 											  ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings);
+
+					ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc =
+						JupiterConfig->get_or("uselegacyddc", ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc);
 				};
 
 				if (auto ZenConfig = UpstreamConfig->get<sol::optional<sol::table>>("zen"))
diff --git a/zenserver/config.h b/zenserver/config.h
index a5ef94fc4..92b7c9e31 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -24,7 +24,8 @@ struct ZenUpstreamJupiterConfig
 	std::string OAuthClientSecret;
 	std::string Namespace;
 	std::string DdcNamespace;
-	bool		UseDevelopmentSettings;
+	bool		UseDevelopmentSettings = false;
+	bool		UseLegacyDdc		   = false;
 };
 
 struct ZenUpstreamZenConfig
diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp
index 7e22b9af9..d204e8f0b 100644
--- a/zenserver/upstream/jupiter.cpp
+++ b/zenserver/upstream/jupiter.cpp
@@ -134,9 +134,10 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, const IoHash& Key)
 }
 
 CloudCacheResult
-CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key)
+CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenContentType RefType)
 {
-	std::string Auth;
+	const std::string ContentType = RefType == ZenContentType::kCbObject ? "application/x-ue-cb" : "application/octet-stream";
+	std::string		  Auth;
 	m_CacheClient->AcquireAccessToken(Auth);
 
 	ExtendableStringBuilder<256> Uri;
@@ -146,7 +147,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key)
 	cpr::Session& Session = m_SessionState->Session;
 
 	Session.SetOption(cpr::Url{Uri.c_str()});
-	Session.SetOption(cpr::Header{{"Authorization", Auth}, {"Accept", "application/x-ue-cb"}});
+	Session.SetOption(cpr::Header{{"Authorization", Auth}, {"Accept", ContentType}});
 
 	cpr::Response Response = Session.Get();
 	detail::Log(m_Log, "GET"sv, Response);
@@ -215,11 +216,12 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, const IoHash& Key,
 }
 
 CloudCacheResult
-CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Ref)
+CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType)
 {
 	IoHash Hash = IoHash::HashBuffer(Ref.Data(), Ref.Size());
 
-	std::string Auth;
+	const std::string ContentType = RefType == ZenContentType::kCbObject ? "application/x-ue-cb" : "application/octet-stream";
+	std::string		  Auth;
 	m_CacheClient->AcquireAccessToken(Auth);
 
 	ExtendableStringBuilder<256> Uri;
@@ -229,8 +231,7 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer
 	cpr::Session& Session = m_SessionState->Session;
 
 	Session.SetOption(cpr::Url{Uri.c_str()});
-	Session.SetOption(
-		cpr::Header{{"Authorization", Auth}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", "application/x-ue-cb"}});
+	Session.SetOption(cpr::Header{{"Authorization", Auth}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", ContentType}});
 	Session.SetBody(cpr::Body{(const char*)Ref.Data(), Ref.Size()});
 
 	cpr::Response Response = Session.Put();
diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h
index ba5e0a65a..f3af88e77 100644
--- a/zenserver/upstream/jupiter.h
+++ b/zenserver/upstream/jupiter.h
@@ -62,12 +62,12 @@ public:
 
 	CloudCacheResult GetDerivedData(std::string_view BucketId, std::string_view Key);
 	CloudCacheResult GetDerivedData(std::string_view BucketId, const IoHash& Key);
-	CloudCacheResult GetRef(std::string_view BucketId, const IoHash& Key);
+	CloudCacheResult GetRef(std::string_view BucketId, const IoHash& Key, ZenContentType RefType);
 	CloudCacheResult GetCompressedBlob(const IoHash& Key);
 
 	CloudCacheResult PutDerivedData(std::string_view BucketId, std::string_view Key, IoBuffer DerivedData);
 	CloudCacheResult PutDerivedData(std::string_view BucketId, const IoHash& Key, IoBuffer DerivedData);
-	CloudCacheResult PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Ref);
+	CloudCacheResult PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType);
 	CloudCacheResult PutCompressedBlob(const IoHash& Key, IoBuffer Blob);
 
 	std::vector<IoHash> Filter(std::string_view BucketId, const std::vector<IoHash>& ChunkHashes);
@@ -86,6 +86,7 @@ struct CloudCacheClientOptions
 	std::string_view OAuthProvider;
 	std::string_view OAuthClientId;
 	std::string_view OAuthSecret;
+	bool			 UseLegacyDdc = false;
 };
 
 /**
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index 6c9baf9b0..f7a91acb5 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -89,7 +89,7 @@ namespace detail {
 	class JupiterUpstreamEndpoint final : public zen::UpstreamEndpoint
 	{
 	public:
-		JupiterUpstreamEndpoint(const CloudCacheClientOptions& Options)
+		JupiterUpstreamEndpoint(const CloudCacheClientOptions& Options) : m_UseLegacyDdc(Options.UseLegacyDdc)
 		{
 			using namespace fmt::literals;
 			m_DisplayName = "Jupier - '{}'"_format(Options.ServiceUrl);
@@ -113,13 +113,13 @@ namespace detail {
 				zen::CloudCacheSession Session(m_Client);
 				CloudCacheResult	   Result;
 
-				if (Type == ZenContentType::kBinary)
+				if (m_UseLegacyDdc && Type == ZenContentType::kBinary)
 				{
 					Result = Session.GetDerivedData(CacheKey.Bucket, CacheKey.Hash);
 				}
 				else
 				{
-					Result = Session.GetRef(CacheKey.Bucket, CacheKey.Hash);
+					Result = Session.GetRef(CacheKey.Bucket, CacheKey.Hash, Type);
 				}
 
 				return {.Value = Result.Value, .Success = Result.Success};
@@ -160,7 +160,17 @@ namespace detail {
 					CloudCacheResult Result;
 					for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
 					{
-						Result = Session.PutDerivedData(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue);
+						if (m_UseLegacyDdc)
+						{
+							Result = Session.PutDerivedData(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue);
+						}
+						else
+						{
+							Result = Session.PutRef(CacheRecord.CacheKey.Bucket,
+													CacheRecord.CacheKey.Hash,
+													RecordValue,
+													ZenContentType::kBinary);
+						}
 					}
 
 					return {.Success = Result.Success};
@@ -185,7 +195,10 @@ namespace detail {
 						CloudCacheResult Result;
 						for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
 						{
-							Result = Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue);
+							Result = Session.PutRef(CacheRecord.CacheKey.Bucket,
+													CacheRecord.CacheKey.Hash,
+													RecordValue,
+													ZenContentType::kCbObject);
 						}
 
 						return {.Success = Result.Success};
@@ -199,6 +212,7 @@ namespace detail {
 		}
 
 	private:
+		bool					 m_UseLegacyDdc;
 		std::string				 m_DisplayName;
 		RefPtr<CloudCacheClient> m_Client;
 	};
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index be122baca..e465734e0 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -166,7 +166,7 @@ public:
 							.OAuthProvider		= "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv,
 							.OAuthClientId		= "0oao91lrhqPiAlaGD0x7"sv,
 							.OAuthSecret		= "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv,
-						};
+							.UseLegacyDdc		= false};
 					}
 
 					Options.ServiceUrl		   = ValueOrDefault(UpstreamConfig.JupiterConfig.Url, Options.ServiceUrl);
@@ -175,6 +175,7 @@ public:
 					Options.OAuthProvider	   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthProvider, Options.OAuthProvider);
 					Options.OAuthClientId	   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthClientId, Options.OAuthClientId);
 					Options.OAuthSecret		   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthClientSecret, Options.OAuthSecret);
+					Options.UseLegacyDdc |= UpstreamConfig.JupiterConfig.UseLegacyDdc;
 
 					if (!Options.ServiceUrl.empty())
 					{