allow download specification for zen builds download (#734)

- Feature: `zen builds download` now supports `--download-spec-path` to determine what content to download from a build
  - The unstructured format expects one line per file relative to the root with '/' as a path delimiter
  - The structured format uses JSon format and the `--download-spec-path` must have extension `.json` to enable structured input

        {
            "parts": {
                "default" : {
                    "files": [
                        "foo/bar",
                        "baz.exe"
                    ]
                },
                "symbols": {
                    "files": [
                        "baz.pdb"
                    ]
                }
            }
        }

5 files changed, 272 insertions, 108 deletions

diff --git a/src/zenremotestore/builds/buildmanifest.cpp b/src/zenremotestore/builds/buildmanifest.cpp
new file mode 100644
index 000000000..051436e96
--- /dev/null
+++ b/src/zenremotestore/builds/buildmanifest.cpp
@@ -0,0 +1,173 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildmanifest.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/fmtutils.h>
+
+#if ZEN_WITH_TESTS
+#	include <zencore/basicfile.h>
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#endif	// ZEN_WITH_TESTS
+
+namespace zen {
+
+using namespace std::literals;
+
+BuildManifest
+ParseBuildManifest(const std::filesystem::path& ManifestPath)
+{
+	BuildManifest Result;
+	{
+		IoBuffer ManifestContent = ReadFile(ManifestPath).Flatten();
+
+		if (ToLower(ManifestPath.extension().string()) == ".json")
+		{
+			IoBuffer		 MetaDataJson = ReadFile(ManifestPath).Flatten();
+			std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+			std::string		 JsonError;
+			CbObject		 Manifest = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+			if (!JsonError.empty())
+			{
+				throw std::runtime_error(fmt::format("Invalid manifest file at {}. '{}'", ManifestPath, JsonError));
+			}
+			CbObjectView PartsObject = Manifest["parts"sv].AsObjectView();
+			for (CbFieldView PartsField : PartsObject)
+			{
+				std::string_view PartName = PartsField.GetName();
+				if (PartName.empty())
+				{
+					throw std::runtime_error(fmt::format("Part {} in manifest file at {} does not have a name. '{}'",
+														 Result.Parts.size() + 1,
+														 ManifestPath,
+														 JsonError));
+				}
+				CbObjectView					   Part		  = PartsField.AsObjectView();
+				Oid								   PartId	  = Part["partId"sv].AsObjectId();
+				CbArrayView						   FilesArray = Part["files"sv].AsArrayView();
+				std::vector<std::filesystem::path> Files;
+				Files.reserve(FilesArray.Num());
+				for (CbFieldView FileField : FilesArray)
+				{
+					std::filesystem::path File(FileField.AsU8String());
+					Files.push_back(File);
+				}
+
+				Result.Parts.push_back(BuildManifest::Part{.PartId = PartId, .PartName = std::string(PartName), .Files = std::move(Files)});
+			}
+			return Result;
+		}
+		else
+		{
+			Result.Parts.resize(1);
+			BuildManifest::Part& SinglePart = Result.Parts.front();
+
+			std::string_view			ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize());
+			std::string_view::size_type Offset = 0;
+			while (Offset < ManifestContent.GetSize())
+			{
+				size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset);
+				if (PathBreakOffset == std::string_view::npos)
+				{
+					PathBreakOffset = ManifestContent.GetSize();
+				}
+				std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset);
+				if (!AssetPath.empty())
+				{
+					SinglePart.Files.push_back(std::filesystem::path(AssetPath));
+				}
+				Offset			 = PathBreakOffset;
+				size_t EolOffset = ManifestString.find_first_of("\r\n", Offset);
+				if (EolOffset == std::string_view::npos)
+				{
+					break;
+				}
+				Offset				   = EolOffset;
+				size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset);
+				if (LineBreakOffset == std::string_view::npos)
+				{
+					break;
+				}
+				Offset = LineBreakOffset;
+			}
+		}
+	}
+	return Result;
+}
+#if ZEN_WITH_TESTS
+
+TEST_CASE("buildmanifest.unstructured")
+{
+	ScopedTemporaryDirectory		   Root;
+	std::vector<std::filesystem::path> Files = {"fileA", "dirA/FileB", "dirB/FileC", "dirB/FileD"};
+
+	{
+		ExtendableStringBuilder<512> SB;
+		for (const std::filesystem::path& File : Files)
+		{
+			SB << File.generic_string() << "\n";
+		}
+		WriteFile(Root.Path() / "manifest.txt", IoBuffer(IoBuffer::Wrap, SB.ToView().data(), SB.ToView().length()));
+	}
+
+	BuildManifest Manifest = ParseBuildManifest(Root.Path() / "manifest.txt");
+	CHECK_EQ(Manifest.Parts.size(), 1u);
+	CHECK_EQ(Manifest.Parts[0].PartId, Oid::Zero);
+	CHECK_EQ(Manifest.Parts[0].PartName, "");
+	CHECK_EQ(Manifest.Parts[0].Files, Files);
+}
+
+TEST_CASE("buildmanifest.structured")
+{
+	ScopedTemporaryDirectory Root;
+
+	std::string Id = Oid::NewOid().ToString();
+
+	std::string ManifestString =
+		"{\n"
+		"    \"parts\": {\n"
+		"        \"default\": {\n"
+		"            \"partId\": \"098a2742d46c22a67ab57457\",\n"
+		"            \"files\": [\n"
+		"                \"foo/bar\",\n"
+		"                \"baz.exe\"\n"
+		"            ]\n"
+		"        },\n"
+		"        \"symbols\": {\n"
+		"            \"files\": [\n"
+		"                \"baz.pdb\"\n"
+		"            ]\n"
+		"        }\n"
+		"    }\n"
+		"}\n";
+
+	WriteFile(Root.Path() / "manifest.json", IoBuffer(IoBuffer::Wrap, ManifestString.data(), ManifestString.length()));
+
+	const Oid		  DefaultPartExpectedId	  = Oid::FromHexString("098a2742d46c22a67ab57457");
+	const std::string DefaultPartExpectedName = "default";
+	const Oid		  SymbolPartExpectedId	  = Oid::Zero;
+	const std::string SymbolsPartExpectedName = "symbols";
+
+	BuildManifest Manifest = ParseBuildManifest(Root.Path() / "manifest.json");
+	CHECK_EQ(Manifest.Parts.size(), 2u);
+	CHECK_EQ(Manifest.Parts[0].PartId, DefaultPartExpectedId);
+	CHECK_EQ(Manifest.Parts[0].PartName, DefaultPartExpectedName);
+	CHECK_EQ(Manifest.Parts[0].Files.size(), 2u);
+	CHECK_EQ(Manifest.Parts[0].Files[0].generic_string(), "foo/bar");
+	CHECK_EQ(Manifest.Parts[0].Files[1].generic_string(), "baz.exe");
+
+	CHECK_EQ(Manifest.Parts[1].PartId, SymbolPartExpectedId);
+	CHECK_EQ(Manifest.Parts[1].PartName, SymbolsPartExpectedName);
+	CHECK_EQ(Manifest.Parts[1].Files.size(), 1u);
+	CHECK_EQ(Manifest.Parts[1].Files[0].generic_string(), "baz.pdb");
+}
+
+void
+buildmanifest_forcelink()
+{
+}
+
+#endif	// ZEN_WITH_TESTS
+
+}  // namespace zen
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 76cbe6540..5368c7df4 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -3,6 +3,7 @@
 #include <zenremotestore/builds/buildstorageoperations.h>
 
 #include <zenremotestore/builds/buildcontent.h>
+#include <zenremotestore/builds/buildmanifest.h>
 #include <zenremotestore/builds/buildsavedstate.h>
 #include <zenremotestore/builds/buildstorage.h>
 #include <zenremotestore/builds/buildstoragecache.h>
@@ -4654,7 +4655,9 @@ BuildsOperationUploadFolder::ReadFolder()
 	tsl::robin_set<std::string> ExcludeAssetPaths;
 	if (IsFile(ExcludeManifestPath))
 	{
-		PartManifest							  Manifest	 = ParseManifest(m_Path, ExcludeManifestPath);
+		std::filesystem::path AbsoluteExcludeManifestPath =
+			MakeSafeAbsolutePath(ExcludeManifestPath.is_absolute() ? ExcludeManifestPath : m_Path / ExcludeManifestPath);
+		BuildManifest							  Manifest	 = ParseBuildManifest(AbsoluteExcludeManifestPath);
 		const std::vector<std::filesystem::path>& AssetPaths = Manifest.Parts.front().Files;
 		ExcludeAssetPaths.reserve(AssetPaths.size());
 		for (const std::filesystem::path& AssetPath : AssetPaths)
@@ -4700,7 +4703,8 @@ BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& Mani
 {
 	std::vector<UploadPart> UploadParts;
 	Stopwatch				ManifestParseTimer;
-	PartManifest			Manifest = ParseManifest(m_Path, ManifestPath);
+	std::filesystem::path	AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : m_Path / ManifestPath);
+	BuildManifest			Manifest			 = ParseBuildManifest(AbsoluteManifestPath);
 	if (Manifest.Parts.empty())
 	{
 		throw std::runtime_error(fmt::format("Manifest file at '{}' is invalid", ManifestPath));
@@ -4709,7 +4713,7 @@ BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& Mani
 	UploadParts.resize(Manifest.Parts.size());
 	for (size_t PartIndex = 0; PartIndex < Manifest.Parts.size(); PartIndex++)
 	{
-		PartManifest::Part& PartManifest = Manifest.Parts[PartIndex];
+		BuildManifest::Part& PartManifest = Manifest.Parts[PartIndex];
 		if (ManifestPath.is_relative())
 		{
 			PartManifest.Files.push_back(ManifestPath);
@@ -4869,92 +4873,6 @@ BuildsOperationUploadFolder::Execute(const Oid&					  BuildPartId,
 	}
 }
 
-BuildsOperationUploadFolder::PartManifest
-BuildsOperationUploadFolder::ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath)
-{
-	PartManifest Result;
-	{
-		std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath);
-		IoBuffer			  ManifestContent	   = ReadFile(AbsoluteManifestPath).Flatten();
-
-		if (ToLower(ManifestPath.extension().string()) == ".json")
-		{
-			IoBuffer		 MetaDataJson = ReadFile(ManifestPath).Flatten();
-			std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
-			std::string		 JsonError;
-			CbObject		 Manifest = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
-			if (!JsonError.empty())
-			{
-				throw std::runtime_error(fmt::format("Invalid manifest file at {}. '{}'", ManifestPath, JsonError));
-			}
-			CbObjectView PartsObject = Manifest["parts"sv].AsObjectView();
-			for (CbFieldView PartsField : PartsObject)
-			{
-				std::string_view PartName = PartsField.GetName();
-				if (PartName.empty())
-				{
-					throw std::runtime_error(fmt::format("Part {} in manifest file at {} does not have a name. '{}'",
-														 Result.Parts.size() + 1,
-														 ManifestPath,
-														 JsonError));
-				}
-				CbObjectView Part	= PartsField.AsObjectView();
-				Oid			 PartId = Part["partId"sv].AsObjectId();
-				if (PartId == Oid::Zero)
-				{
-					PartId = Oid::NewOid();
-				}
-				CbArrayView						   FilesArray = Part["files"sv].AsArrayView();
-				std::vector<std::filesystem::path> Files;
-				Files.reserve(FilesArray.Num());
-				for (CbFieldView FileField : FilesArray)
-				{
-					std::filesystem::path File(FileField.AsU8String());
-					Files.push_back(File);
-				}
-
-				Result.Parts.push_back(PartManifest::Part{.PartId = PartId, .PartName = std::string(PartName), .Files = std::move(Files)});
-			}
-			return Result;
-		}
-		else
-		{
-			Result.Parts.resize(1);
-			PartManifest::Part& SinglePart = Result.Parts.front();
-
-			std::string_view			ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize());
-			std::string_view::size_type Offset = 0;
-			while (Offset < ManifestContent.GetSize())
-			{
-				size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset);
-				if (PathBreakOffset == std::string_view::npos)
-				{
-					PathBreakOffset = ManifestContent.GetSize();
-				}
-				std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset);
-				if (!AssetPath.empty())
-				{
-					SinglePart.Files.emplace_back(std::filesystem::path(AssetPath));
-				}
-				Offset			 = PathBreakOffset;
-				size_t EolOffset = ManifestString.find_first_of("\r\n", Offset);
-				if (EolOffset == std::string_view::npos)
-				{
-					break;
-				}
-				Offset				   = EolOffset;
-				size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset);
-				if (LineBreakOffset == std::string_view::npos)
-				{
-					break;
-				}
-				Offset = LineBreakOffset;
-			}
-		}
-	}
-	return Result;
-}
-
 bool
 BuildsOperationUploadFolder::IsAcceptedFolder(const std::string_view& RelativePath) const
 {
@@ -7444,6 +7362,7 @@ GetRemoteContent(OperationLogOutput&							 Output,
 				 StorageInstance&								 Storage,
 				 const Oid&										 BuildId,
 				 const std::vector<std::pair<Oid, std::string>>& BuildParts,
+				 const BuildManifest&							 Manifest,
 				 std::span<const std::string>					 IncludeWildcards,
 				 std::span<const std::string>					 ExcludeWildcards,
 				 std::unique_ptr<ChunkingController>&			 OutChunkController,
@@ -7485,6 +7404,7 @@ GetRemoteContent(OperationLogOutput&							 Output,
 									  CbObject							  BuildPartManifest,
 									  std::span<const std::string>		  IncludeWildcards,
 									  std::span<const std::string>		  ExcludeWildcards,
+									  const BuildManifest::Part*		  OptionalManifest,
 									  ChunkedFolderContent&				  OutRemoteContent,
 									  std::vector<ChunkBlockDescription>& OutBlockDescriptions,
 									  std::vector<IoHash>&				  OutLooseChunkHashes) {
@@ -7529,29 +7449,78 @@ GetRemoteContent(OperationLogOutput&							 Output,
 								  OutRemoteContent.ChunkedContent.ChunkOrders,
 								  DoExtraContentVerify);
 
-		if (!IncludeWildcards.empty() || !ExcludeWildcards.empty())
+		std::vector<std::filesystem::path> DeletedPaths;
+
+		if (OptionalManifest)
 		{
-			std::vector<std::filesystem::path> DeletedPaths;
+			tsl::robin_set<std::string> PathsInManifest;
+			PathsInManifest.reserve(OptionalManifest->Files.size());
+			for (const std::filesystem::path& ManifestPath : OptionalManifest->Files)
+			{
+				PathsInManifest.insert(ToLower(ManifestPath.generic_string()));
+			}
 			for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
 			{
-				if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true))
+				if (!PathsInManifest.contains(ToLower(RemotePath.generic_string())))
 				{
 					DeletedPaths.push_back(RemotePath);
 				}
 			}
+		}
 
-			if (!DeletedPaths.empty())
+		if (!IncludeWildcards.empty() || !ExcludeWildcards.empty())
+		{
+			for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
 			{
-				OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths);
-				InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes);
+				if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true))
+				{
+					DeletedPaths.push_back(RemotePath);
+				}
 			}
 		}
 
+		if (!DeletedPaths.empty())
+		{
+			OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths);
+			InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes);
+		}
+
 #if ZEN_BUILD_DEBUG
 		ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards);
 #endif	// ZEN_BUILD_DEBUG
 	};
 
+	auto FindManifest = [&Manifest](const Oid& BuildPartId, std::string_view BuildPartName) -> const BuildManifest::Part* {
+		if (Manifest.Parts.empty())
+		{
+			return nullptr;
+		}
+		if (Manifest.Parts.size() == 1)
+		{
+			if (Manifest.Parts[0].PartId == Oid::Zero && Manifest.Parts[0].PartName.empty())
+			{
+				return &Manifest.Parts[0];
+			}
+		}
+
+		auto It = std::find_if(Manifest.Parts.begin(), Manifest.Parts.end(), [BuildPartId, BuildPartName](const BuildManifest::Part& Part) {
+			if (Part.PartId != Oid::Zero)
+			{
+				return Part.PartId == BuildPartId;
+			}
+			if (!Part.PartName.empty())
+			{
+				return Part.PartName == BuildPartName;
+			}
+			return false;
+		});
+		if (It != Manifest.Parts.end())
+		{
+			return &(*It);
+		}
+		return nullptr;
+	};
+
 	OutPartContents.resize(1);
 	ParseBuildPartManifest(Storage,
 						   BuildId,
@@ -7559,6 +7528,7 @@ GetRemoteContent(OperationLogOutput&							 Output,
 						   BuildPartManifest,
 						   IncludeWildcards,
 						   ExcludeWildcards,
+						   FindManifest(BuildPartId, BuildPartName),
 						   OutPartContents[0],
 						   OutBlockDescriptions,
 						   OutLooseChunkHashes);
@@ -7593,6 +7563,7 @@ GetRemoteContent(OperationLogOutput&							 Output,
 								   OverlayBuildPartManifest,
 								   IncludeWildcards,
 								   ExcludeWildcards,
+								   FindManifest(OverlayBuildPartId, OverlayBuildPartName),
 								   OverlayPartContent,
 								   OverlayPartBlockDescriptions,
 								   OverlayPartLooseChunkHashes);
@@ -7859,6 +7830,7 @@ namespace buildstorageoperations_testutils {
 																  AllBuildParts,
 																  {},
 																  {},
+																  {},
 																  ChunkController,
 																  PartContents,
 																  BlockDescriptions,
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildmanifest.h b/src/zenremotestore/include/zenremotestore/builds/buildmanifest.h
new file mode 100644
index 000000000..a0d9a7691
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/builds/buildmanifest.h
@@ -0,0 +1,27 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/filesystem.h>
+#include <zencore/uid.h>
+
+namespace zen {
+
+struct BuildManifest
+{
+	struct Part
+	{
+		Oid								   PartId = Oid::Zero;
+		std::string						   PartName;
+		std::vector<std::filesystem::path> Files;
+	};
+	std::vector<Part> Parts;
+};
+
+BuildManifest ParseBuildManifest(const std::filesystem::path& ManifestPath);
+
+#if ZEN_WITH_TESTS
+void buildmanifest_forcelink();
+#endif	// ZEN_WITH_TESTS
+
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 9f7d93398..47f402e15 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -577,17 +577,6 @@ private:
 
 	PrepareBuildResult PrepareBuild();
 
-	struct PartManifest
-	{
-		struct Part
-		{
-			Oid								   PartId;
-			std::string						   PartName;
-			std::vector<std::filesystem::path> Files;
-		};
-		std::vector<Part> Parts;
-	};
-
 	struct UploadPart
 	{
 		Oid						   PartId = Oid::Zero;
@@ -600,8 +589,6 @@ private:
 	std::vector<BuildsOperationUploadFolder::UploadPart> ReadFolder();
 	std::vector<UploadPart>								 ReadManifestParts(const std::filesystem::path& ManifestPath);
 
-	PartManifest ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath);
-
 	bool IsAcceptedFolder(const std::string_view& RelativePath) const;
 	bool IsAcceptedFile(const std::string_view& RelativePath) const;
 
@@ -811,10 +798,13 @@ std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(CbObjectView
 															   std::span<const std::string> BuildPartNames,
 															   std::uint64_t&				OutPreferredMultipartChunkSize);
 
+struct BuildManifest;
+
 ChunkedFolderContent GetRemoteContent(OperationLogOutput&							  Output,
 									  StorageInstance&								  Storage,
 									  const Oid&									  BuildId,
 									  const std::vector<std::pair<Oid, std::string>>& BuildParts,
+									  const BuildManifest&							  Manifest,
 									  std::span<const std::string>					  IncludeWildcards,
 									  std::span<const std::string>					  ExcludeWildcards,
 									  std::unique_ptr<ChunkingController>&			  OutChunkController,
diff --git a/src/zenremotestore/zenremotestore.cpp b/src/zenremotestore/zenremotestore.cpp
index 0d008ec40..a0bb17260 100644
--- a/src/zenremotestore/zenremotestore.cpp
+++ b/src/zenremotestore/zenremotestore.cpp
@@ -2,6 +2,7 @@
 
 #include <zenremotestore/zenremotestore.h>
 
+#include <zenremotestore/builds/buildmanifest.h>
 #include <zenremotestore/builds/buildsavedstate.h>
 #include <zenremotestore/builds/buildstorageoperations.h>
 #include <zenremotestore/chunking/chunkedcontent.h>
@@ -17,6 +18,7 @@ namespace zen {
 void
 zenremotestore_forcelinktests()
 {
+	buildmanifest_forcelink();
 	buildsavedstate_forcelink();
 	buildstorageoperations_forcelink();
 	chunkblock_forcelink();