Merge pull request #93 from EpicGames/de/namespaces-continuedv1.0.1.7actions_updates

De/namespaces continued

1 files changed, 71 insertions, 37 deletions

diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index c870e0773..52513abe9 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -160,6 +160,24 @@ namespace detail {
 			}
 		}
 
+		std::string_view GetActualDdcNamespace(CloudCacheSession& Session, std::string_view Namespace)
+		{
+			if (Namespace == ZenCacheStore::DefaultNamespace)
+			{
+				return Session.Client().DefaultDdcNamespace();
+			}
+			return Namespace;
+		}
+
+		std::string_view GetActualBlobStoreNamespace(CloudCacheSession& Session, std::string_view Namespace)
+		{
+			if (Namespace == ZenCacheStore::DefaultNamespace)
+			{
+				return Session.Client().DefaultBlobStoreNamespace();
+			}
+			return Namespace;
+		}
+
 		virtual UpstreamEndpointState GetState() override { return m_Status.EndpointState(); }
 
 		virtual UpstreamEndpointStatus GetStatus() override { return m_Status.EndpointStatus(); }
@@ -173,13 +191,16 @@ namespace detail {
 				CloudCacheSession Session(m_Client);
 				CloudCacheResult  Result;
 
+				std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, CacheKey.Namespace);
+
 				if (m_UseLegacyDdc && Type == ZenContentType::kBinary)
 				{
-					Result = Session.GetDerivedData(CacheKey.Bucket, CacheKey.Hash);
+					std::string_view DdcNamespace = GetActualDdcNamespace(Session, CacheKey.Namespace);
+					Result						  = Session.GetDerivedData(DdcNamespace, CacheKey.Bucket, CacheKey.Hash);
 				}
 				else if (Type == ZenContentType::kCompressedBinary)
 				{
-					Result = Session.GetRef(CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject);
+					Result = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject);
 
 					if (Result.Success)
 					{
@@ -190,23 +211,22 @@ namespace detail {
 							IoBuffer ContentBuffer;
 							int		 NumAttachments = 0;
 
-							CacheRecord.IterateAttachments(
-								[&Session, &Result, &ContentBuffer, &NumAttachments](CbFieldView AttachmentHash) {
-									CloudCacheResult AttachmentResult = Session.GetCompressedBlob(AttachmentHash.AsHash());
-									Result.Bytes += AttachmentResult.Bytes;
-									Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds;
-									Result.ErrorCode = AttachmentResult.ErrorCode;
+							CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) {
+								CloudCacheResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash());
+								Result.Bytes += AttachmentResult.Bytes;
+								Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds;
+								Result.ErrorCode = AttachmentResult.ErrorCode;
 
-									if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response)))
-									{
-										Result.Response = AttachmentResult.Response;
-										++NumAttachments;
-									}
-									else
-									{
-										Result.Success = false;
-									}
-								});
+								if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response)))
+								{
+									Result.Response = AttachmentResult.Response;
+									++NumAttachments;
+								}
+								else
+								{
+									Result.Success = false;
+								}
+							});
 							if (NumAttachments != 1)
 							{
 								Result.Success = false;
@@ -217,7 +237,7 @@ namespace detail {
 				else
 				{
 					const ZenContentType AcceptType = Type == ZenContentType::kCbPackage ? ZenContentType::kCbObject : Type;
-					Result							= Session.GetRef(CacheKey.Bucket, CacheKey.Hash, AcceptType);
+					Result							= Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, AcceptType);
 
 					if (Result.Success && Type == ZenContentType::kCbPackage)
 					{
@@ -228,8 +248,8 @@ namespace detail {
 						{
 							CbObject CacheRecord = LoadCompactBinaryObject(Result.Response);
 
-							CacheRecord.IterateAttachments([&Session, &Result, &Package](CbFieldView AttachmentHash) {
-								CloudCacheResult AttachmentResult = Session.GetCompressedBlob(AttachmentHash.AsHash());
+							CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) {
+								CloudCacheResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash());
 								Result.Bytes += AttachmentResult.Bytes;
 								Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds;
 								Result.ErrorCode = AttachmentResult.ErrorCode;
@@ -294,7 +314,9 @@ namespace detail {
 
 				if (!Result.Error)
 				{
-					CloudCacheResult RefResult = Session.GetRef(CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject);
+					std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, CacheKey.Namespace);
+					CloudCacheResult RefResult =
+						Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject);
 					AppendResult(RefResult, Result);
 
 					m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason);
@@ -305,8 +327,8 @@ namespace detail {
 						if (ValidationResult == CbValidateError::None)
 						{
 							Record = LoadCompactBinaryObject(RefResult.Response);
-							Record.IterateAttachments([this, &Session, &Result, &Package](CbFieldView AttachmentHash) {
-								CloudCacheResult BlobResult = Session.GetCompressedBlob(AttachmentHash.AsHash());
+							Record.IterateAttachments([&](CbFieldView AttachmentHash) {
+								CloudCacheResult BlobResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash());
 								AppendResult(BlobResult, Result);
 
 								m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason);
@@ -329,14 +351,15 @@ namespace detail {
 			return Result;
 		}
 
-		virtual GetUpstreamCacheResult GetCacheValue(const CacheKey&, const IoHash& ValueContentId) override
+		virtual GetUpstreamCacheResult GetCacheValue(const CacheKey& CacheKey, const IoHash& ValueContentId) override
 		{
 			ZEN_TRACE_CPU("Upstream::Horde::GetSingleCacheValue");
 
 			try
 			{
 				CloudCacheSession	   Session(m_Client);
-				const CloudCacheResult Result = Session.GetCompressedBlob(ValueContentId);
+				std::string_view	   BlobStoreNamespace = GetActualBlobStoreNamespace(Session, CacheKey.Namespace);
+				const CloudCacheResult Result			  = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId);
 
 				m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason);
 
@@ -376,8 +399,9 @@ namespace detail {
 				CompressedBuffer Compressed;
 				if (!Result.Error)
 				{
-					const CloudCacheResult BlobResult = Session.GetCompressedBlob(Request.ChunkId);
-					Payload							  = BlobResult.Response;
+					std::string_view	   BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Request.Key.Namespace);
+					const CloudCacheResult BlobResult		  = Session.GetCompressedBlob(BlobStoreNamespace, Request.ChunkId);
+					Payload									  = BlobResult.Response;
 
 					AppendResult(BlobResult, Result);
 
@@ -422,13 +446,18 @@ namespace detail {
 					CloudCacheResult Result;
 					for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
 					{
+						std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, CacheRecord.Key.Namespace);
 						if (m_UseLegacyDdc)
 						{
-							Result = Session.PutDerivedData(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue);
+							Result = Session.PutDerivedData(BlobStoreNamespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue);
 						}
 						else
 						{
-							Result = Session.PutRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, ZenContentType::kBinary);
+							Result = Session.PutRef(BlobStoreNamespace,
+													CacheRecord.Key.Bucket,
+													CacheRecord.Key.Hash,
+													RecordValue,
+													ZenContentType::kBinary);
 						}
 					}
 
@@ -527,7 +556,8 @@ namespace detail {
 			int64_t TotalBytes			= 0ull;
 			double	TotalElapsedSeconds = 0.0;
 
-			const auto PutBlobs = [&](std::span<IoHash> ValueContentIds, std::string& OutReason) -> bool {
+			std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Key.Namespace);
+			const auto		 PutBlobs			= [&](std::span<IoHash> ValueContentIds, std::string& OutReason) -> bool {
 				for (const IoHash& ValueContentId : ValueContentIds)
 				{
 					IoBuffer BlobBuffer;
@@ -539,7 +569,7 @@ namespace detail {
 					CloudCacheResult BlobResult;
 					for (int32_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++)
 					{
-						BlobResult = Session.PutCompressedBlob(ValueContentId, BlobBuffer);
+						BlobResult = Session.PutCompressedBlob(BlobStoreNamespace, ValueContentId, BlobBuffer);
 					}
 
 					m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason);
@@ -560,7 +590,7 @@ namespace detail {
 			PutRefResult RefResult;
 			for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++)
 			{
-				RefResult = Session.PutRef(Key.Bucket, Key.Hash, ObjectBuffer, ZenContentType::kCbObject);
+				RefResult = Session.PutRef(BlobStoreNamespace, Key.Bucket, Key.Hash, ObjectBuffer, ZenContentType::kCbObject);
 			}
 
 			m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason);
@@ -581,7 +611,7 @@ namespace detail {
 			}
 
 			const IoHash	  RefHash		 = IoHash::HashBuffer(ObjectBuffer);
-			FinalizeRefResult FinalizeResult = Session.FinalizeRef(Key.Bucket, Key.Hash, RefHash);
+			FinalizeRefResult FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash);
 
 			m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason);
 
@@ -599,7 +629,7 @@ namespace detail {
 					return {.Reason = std::move(Reason), .Success = false};
 				}
 
-				FinalizeResult = Session.FinalizeRef(Key.Bucket, Key.Hash, RefHash);
+				FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash);
 
 				m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason);
 
@@ -761,6 +791,7 @@ namespace detail {
 						const CacheKey& Key = Request->Key;
 						BatchRequest.BeginObject("Key"sv);
 						{
+							BatchRequest << "Namespace"sv << Key.Namespace;
 							BatchRequest << "Bucket"sv << Key.Bucket;
 							BatchRequest << "Hash"sv << Key.Hash;
 						}
@@ -871,6 +902,7 @@ namespace detail {
 						BatchRequest.BeginObject();
 						{
 							BatchRequest.BeginObject("Key"sv);
+							BatchRequest << "Namespace"sv << Request.Key.Namespace;
 							BatchRequest << "Bucket"sv << Request.Key.Bucket;
 							BatchRequest << "Hash"sv << Request.Key.Hash;
 							BatchRequest.EndObject();
@@ -1042,6 +1074,7 @@ namespace detail {
 								const CacheKey& Key = CacheRecord.Key;
 								BatchWriter.BeginObject("Key"sv);
 								{
+									BatchWriter << "Namespace"sv << Key.Namespace;
 									BatchWriter << "Bucket"sv << Key.Bucket;
 									BatchWriter << "Hash"sv << Key.Hash;
 								}
@@ -1517,7 +1550,7 @@ private:
 		ZenCacheValue		  CacheValue;
 		std::vector<IoBuffer> Payloads;
 
-		if (!m_CacheStore.Get(ZenCacheStore::DefaultNamespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue))
+		if (!m_CacheStore.Get(CacheRecord.Key.Namespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue))
 		{
 			ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", CacheRecord.Key.Bucket, CacheRecord.Key.Hash);
 			return;
@@ -1531,7 +1564,8 @@ private:
 			}
 			else
 			{
-				ZEN_WARN("process upstream FAILED, '{}/{}/{}', ValueContentId doesn't exist in CAS",
+				ZEN_WARN("process upstream FAILED, '{}/{}/{}/{}', ValueContentId doesn't exist in CAS",
+						 CacheRecord.Key.Namespace,
 						 CacheRecord.Key.Bucket,
 						 CacheRecord.Key.Hash,
 						 ValueContentId);