Merge branch 'jupiter-structured' of https://github.com/EpicGames/zen into jupiter-structured

5 files changed, 102 insertions, 110 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 0d7ad0f16..8753d50fc 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -276,9 +276,9 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
 		return m_CasStore.FindChunk(ChunkIt->second);
 	}
 
-	if (auto FileIt = m_ServerFileMap.find(ChunkId); FileIt != m_ServerFileMap.end())
+	if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end())
 	{
-		std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second;
+		std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath;
 
 		return IoBufferBuilder::MakeFromFile(FilePath.native().c_str());
 	}
@@ -296,20 +296,35 @@ ProjectStore::Oplog::IterateFileMap(std::function<void(const Oid&, const std::st
 {
 	for (const auto& Kv : m_FileMap)
 	{
-		Fn(Kv.first, Kv.second);
+		Fn(Kv.first, Kv.second.ClientPath);
 	}
 }
 
-void
-ProjectStore::Oplog::AddFileMapping(Oid FileId, std::string_view Path)
+bool
+ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath)
 {
-	m_FileMap.emplace(FileId, Path);
-}
+	if (ServerPath.empty() || ClientPath.empty())
+	{
+		return false;
+	}
 
-void
-ProjectStore::Oplog::AddServerFileMapping(Oid FileId, std::string_view Path)
-{
-	m_ServerFileMap.emplace(FileId, Path);
+	if (ServerPath[0] == '/')
+	{
+		ServerPath = ServerPath.substr(1);
+	}
+
+	FileMapEntry Entry;
+	Entry.ServerPath = ServerPath;
+	Entry.ClientPath = ClientPath;
+
+	m_FileMap.emplace(FileId, std::move(Entry));
+
+	if (Hash != IoHash::Zero)
+	{
+		m_ChunkMap.emplace(FileId, Hash);
+	}
+
+	return true;
 }
 
 void
@@ -356,93 +371,30 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 		Log().debug("bulkdata {} -> {}", BulkDataId, BulkDataHash);
 	}
 
-	if (CbFieldView FilesArray = Core["files"sv])
+	if (Core["files"sv])
 	{
-		int FileCount		= 0;
-		int ServerFileCount = 0;
-
-		std::atomic<bool> InvalidOp{false};
-
 		Stopwatch Timer;
+		int32_t	  FileCount = 0;
 
-		std::future<void> f0 = std::async(std::launch::async, [&] {
-			for (CbFieldView& Entry : FilesArray)
-			{
-				CbObjectView FileObj = Entry.AsObjectView();
-				const Oid	 FileId	 = FileObj["id"sv].AsObjectId();
-
-				if (auto PathField = FileObj["path"sv])
-				{
-					AddFileMapping(FileId, PathField.AsString());
-
-					// Log().debug("file {} -> {}", FileId, PathString);
-
-					++FileCount;
-				}
-				else
-				{
-					// Every file entry needs to specify a path
-					InvalidOp = true;
-					break;
-				}
-
-				if (InvalidOp.load(std::memory_order::relaxed))
-				{
-					break;
-				}
-			}
-		});
-
-		std::future<void> f1 = std::async(std::launch::async, [&] {
-			CbArrayView ServerFilesArray = Core["serverfiles"sv].AsArrayView();
+		for (CbFieldView& Entry : Core["files"sv])
+		{
+			CbObjectView	 FileObj	  = Entry.AsObjectView();
+			const Oid		 FileId		  = FileObj["id"sv].AsObjectId();
+			IoHash			 FileDataHash = FileObj["data"sv].AsBinaryAttachment();
+			std::string_view ServerPath	  = FileObj["serverpath"sv].AsString();
+			std::string_view ClientPath	  = FileObj["clientpath"sv].AsString();
 
-			for (CbFieldView& Entry : ServerFilesArray)
+			if (AddFileMapping(FileId, FileDataHash, ServerPath, ClientPath))
 			{
-				CbObjectView FileObj = Entry.AsObjectView();
-				const Oid	 FileId	 = FileObj["id"sv].AsObjectId();
-
-				if (auto PathField = FileObj["path"sv])
-				{
-					AddServerFileMapping(FileId, PathField.AsString());
-
-					// m_log.debug("file {} -> {}", FileId, PathString);
-
-					++ServerFileCount;
-				}
-				else
-				{
-					// Every file entry needs to specify a path
-					InvalidOp = true;
-					break;
-				}
-
-				if (InvalidOp.load(std::memory_order::relaxed))
-				{
-					break;
-				}
+				++FileCount;
 			}
-		});
-
-		f0.wait();
-		f1.wait();
-
-		if (InvalidOp)
-		{
-			return kInvalidOp;
-		}
-
-		if (FileCount || ServerFileCount)
-		{
-			Log().debug("{} files registered, {} server files (took {})",
-						FileCount,
-						ServerFileCount,
-						NiceTimeSpanMs(Timer.getElapsedTimeMs()));
-
-			if (FileCount != ServerFileCount)
+			else
 			{
-				Log().warn("client/server file list mismatch: {} vs {}", FileCount, ServerFileCount);
+				Log().warn("invalid file");
 			}
 		}
+
+		Log().debug("added {} file(s) in {}", FileCount, NiceTimeSpanMs(Timer.getElapsedTimeMs()));
 	}
 
 	for (CbFieldView& Entry : Core["meta"sv])
@@ -1114,7 +1066,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 			// the prep step rejected the chunk. This should be fixed since there's
 			// a performance cost associated with any file system activity
 
-			bool IsValid = true;
+			bool				IsValid = true;
 			std::vector<IoHash> MissingChunks;
 
 			CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index 38c53ea6e..72b8a1cd6 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -99,6 +99,12 @@ public:
 			uint64_t Size;
 		};
 
+		struct FileMapEntry
+		{
+			std::string ServerPath;
+			std::string ClientPath;
+		};
+
 		template<class V>
 		using OidMap = tsl::robin_map<Oid, V, Oid::Hasher>;
 
@@ -108,18 +114,17 @@ public:
 		std::filesystem::path m_BasePath;
 		std::filesystem::path m_TempPath;
 
-		OidMap<IoHash>					 m_ChunkMap;	   // output data chunk id -> CAS address
-		OidMap<IoHash>					 m_MetaMap;		   // meta chunk id -> CAS address
-		OidMap<std::string>				 m_FileMap;		   // file id -> client file
-		OidMap<std::string>				 m_ServerFileMap;  // file id -> server file
-		std::map<int, OplogEntryAddress> m_OpAddressMap;   // Index LSN -> op data in ops blob file
-		OidMap<int>						 m_LatestOpMap;	   // op key -> latest op LSN for key
+		OidMap<IoHash>					 m_ChunkMap;		 // output data chunk id -> CAS address
+		OidMap<IoHash>					 m_MetaMap;			 // meta chunk id -> CAS address
+		OidMap<FileMapEntry>			 m_FileMap;			 // file id -> file map entry
+		int32_t							 m_ManifestVersion;	 // File system manifest version
+		std::map<int, OplogEntryAddress> m_OpAddressMap;	 // Index LSN -> op data in ops blob file
+		OidMap<int>						 m_LatestOpMap;		 // op key -> latest op LSN for key
 
 		RefPtr<OplogStorage> m_Storage;
 		std::string			 m_OplogId;
 
-		void AddFileMapping(Oid FileId, std::string_view Path);
-		void AddServerFileMapping(Oid FileId, std::string_view Path);
+		bool AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath);
 		void AddChunkMapping(Oid ChunkId, IoHash Hash);
 		void AddMetaMapping(Oid ChunkId, IoHash Hash);
 	};
diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp
index 88a164fbd..363ebcd61 100644
--- a/zenserver/upstream/jupiter.cpp
+++ b/zenserver/upstream/jupiter.cpp
@@ -3,6 +3,7 @@
 #include "jupiter.h"
 
 #include <fmt/format.h>
+#include <zencore/compactbinary.h>
 #include <zencore/iobuffer.h>
 #include <zencore/iohash.h>
 #include <zencore/string.h>
@@ -108,26 +109,54 @@ CloudCacheSession::Put(std::string_view BucketId, std::string_view Key, IoBuffer
 	}
 }
 
-//////////////////////////////////////////////////////////////////////////
-
-IoBuffer
-CloudCacheSession::Get(std::string_view BucketId, const IoHash& Key)
+void
+CloudCacheSession::Put(std::string_view BucketId, std::string_view Key, CbObjectView Data)
 {
-	StringBuilder<64> KeyString;
-	Key.ToHexString(KeyString);
+	ExtendableStringBuilder<256> Uri;
+	Uri << m_CacheClient->ServiceUrl();
+	Uri << "/api/v1/c/ddc/" << m_CacheClient->Namespace() << "/" << BucketId << "/" TESTING_PREFIX << Key;
 
-	return Get(BucketId, KeyString);
+	auto& Session = m_SessionState->Session;
+
+	IoHash	   Hash		= Data.GetHash();
+	MemoryView DataView = Data.GetView();
+
+	std::string Auth;
+	m_CacheClient->AcquireAccessToken(Auth);
+	Session.SetOption(cpr::Url{Uri.c_str()});
+	Session.SetOption(
+		cpr::Header{{"Authorization", Auth}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", "application/x-ue-cb"}});
+	Session.SetOption(cpr::Body{(const char*)DataView.GetData(), DataView.GetSize()});
+
+	cpr::Response Response = Session.Put();
+
+	if (Response.error)
+	{
+		spdlog::warn("PUT failed: '{}'", Response.error.message);
+	}
 }
 
-void
-CloudCacheSession::Put(std::string_view BucketId, const IoHash& Key, IoBuffer Data, HttpContentType ContentType)
+std::vector<IoHash>
+CloudCacheSession::Filter(std::string_view BucketId, const std::vector<IoHash>& ChunkHashes)
 {
-	ZEN_UNUSED(ContentType);
+	ExtendableStringBuilder<256> Uri;
+	Uri << m_CacheClient->ServiceUrl();
+	Uri << "/api/v1/s/" << m_CacheClient->Namespace();
+
+	ZEN_UNUSED(BucketId, ChunkHashes);
 
+	return {};
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+IoBuffer
+CloudCacheSession::Get(std::string_view BucketId, const IoHash& Key)
+{
 	StringBuilder<64> KeyString;
 	Key.ToHexString(KeyString);
 
-	return Put(BucketId, KeyString, Data);
+	return Get(BucketId, KeyString);
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h
index c17ffb047..a0dbefa3c 100644
--- a/zenserver/upstream/jupiter.h
+++ b/zenserver/upstream/jupiter.h
@@ -9,6 +9,7 @@
 #include <atomic>
 #include <list>
 #include <memory>
+#include <vector>
 
 namespace zen {
 namespace detail {
@@ -18,6 +19,7 @@ namespace detail {
 class IoBuffer;
 class CloudCacheClient;
 struct IoHash;
+class CbObjectView;
 
 /**
  * Cached access token, for use with `Authorization:` header
@@ -54,7 +56,9 @@ public:
 
 	// Structured cache operations
 	IoBuffer Get(std::string_view BucketId, const IoHash& Key);
-	void	 Put(std::string_view BucketId, const IoHash& Key, IoBuffer Data, HttpContentType ContentType);
+	void	 Put(std::string_view BucketId, std::string_view Key, CbObjectView Data);
+
+	std::vector<IoHash> Filter(std::string_view BucketId, const std::vector<IoHash>& ChunkHashes);
 
 private:
 	RefPtr<CloudCacheClient>		m_CacheClient;
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index b47ec2f04..2ba6bd551 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -58,10 +58,12 @@
   <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <VcpkgEnableManifest>true</VcpkgEnableManifest>
     <VcpkgUseStatic>true</VcpkgUseStatic>
+    <VcpkgAdditionalInstallOptions>--overlay-ports=$(SolutionDir)vcpkg_overlay-ports</VcpkgAdditionalInstallOptions>
   </PropertyGroup>
   <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <VcpkgEnableManifest>true</VcpkgEnableManifest>
     <VcpkgUseStatic>true</VcpkgUseStatic>
+    <VcpkgAdditionalInstallOptions>--overlay-ports=$(SolutionDir)vcpkg_overlay-ports</VcpkgAdditionalInstallOptions>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>