allow download specification for zen builds download (#734)

- Feature: `zen builds download` now supports `--download-spec-path` to determine what content to download from a build
  - The unstructured format expects one line per file relative to the root with '/' as a path delimiter
  - The structured format uses JSon format and the `--download-spec-path` must have extension `.json` to enable structured input

        {
            "parts": {
                "default" : {
                    "files": [
                        "foo/bar",
                        "baz.exe"
                    ]
                },
                "symbols": {
                    "files": [
                        "baz.pdb"
                    ]
                }
            }
        }

1 files changed, 173 insertions, 0 deletions

diff --git a/src/zenremotestore/builds/buildmanifest.cpp b/src/zenremotestore/builds/buildmanifest.cpp
new file mode 100644
index 000000000..051436e96
--- /dev/null
+++ b/src/zenremotestore/builds/buildmanifest.cpp
@@ -0,0 +1,173 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildmanifest.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/fmtutils.h>
+
+#if ZEN_WITH_TESTS
+#	include <zencore/basicfile.h>
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#endif	// ZEN_WITH_TESTS
+
+namespace zen {
+
+using namespace std::literals;
+
+BuildManifest
+ParseBuildManifest(const std::filesystem::path& ManifestPath)
+{
+	BuildManifest Result;
+	{
+		IoBuffer ManifestContent = ReadFile(ManifestPath).Flatten();
+
+		if (ToLower(ManifestPath.extension().string()) == ".json")
+		{
+			IoBuffer		 MetaDataJson = ReadFile(ManifestPath).Flatten();
+			std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+			std::string		 JsonError;
+			CbObject		 Manifest = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+			if (!JsonError.empty())
+			{
+				throw std::runtime_error(fmt::format("Invalid manifest file at {}. '{}'", ManifestPath, JsonError));
+			}
+			CbObjectView PartsObject = Manifest["parts"sv].AsObjectView();
+			for (CbFieldView PartsField : PartsObject)
+			{
+				std::string_view PartName = PartsField.GetName();
+				if (PartName.empty())
+				{
+					throw std::runtime_error(fmt::format("Part {} in manifest file at {} does not have a name. '{}'",
+														 Result.Parts.size() + 1,
+														 ManifestPath,
+														 JsonError));
+				}
+				CbObjectView					   Part		  = PartsField.AsObjectView();
+				Oid								   PartId	  = Part["partId"sv].AsObjectId();
+				CbArrayView						   FilesArray = Part["files"sv].AsArrayView();
+				std::vector<std::filesystem::path> Files;
+				Files.reserve(FilesArray.Num());
+				for (CbFieldView FileField : FilesArray)
+				{
+					std::filesystem::path File(FileField.AsU8String());
+					Files.push_back(File);
+				}
+
+				Result.Parts.push_back(BuildManifest::Part{.PartId = PartId, .PartName = std::string(PartName), .Files = std::move(Files)});
+			}
+			return Result;
+		}
+		else
+		{
+			Result.Parts.resize(1);
+			BuildManifest::Part& SinglePart = Result.Parts.front();
+
+			std::string_view			ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize());
+			std::string_view::size_type Offset = 0;
+			while (Offset < ManifestContent.GetSize())
+			{
+				size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset);
+				if (PathBreakOffset == std::string_view::npos)
+				{
+					PathBreakOffset = ManifestContent.GetSize();
+				}
+				std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset);
+				if (!AssetPath.empty())
+				{
+					SinglePart.Files.push_back(std::filesystem::path(AssetPath));
+				}
+				Offset			 = PathBreakOffset;
+				size_t EolOffset = ManifestString.find_first_of("\r\n", Offset);
+				if (EolOffset == std::string_view::npos)
+				{
+					break;
+				}
+				Offset				   = EolOffset;
+				size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset);
+				if (LineBreakOffset == std::string_view::npos)
+				{
+					break;
+				}
+				Offset = LineBreakOffset;
+			}
+		}
+	}
+	return Result;
+}
+#if ZEN_WITH_TESTS
+
+TEST_CASE("buildmanifest.unstructured")
+{
+	ScopedTemporaryDirectory		   Root;
+	std::vector<std::filesystem::path> Files = {"fileA", "dirA/FileB", "dirB/FileC", "dirB/FileD"};
+
+	{
+		ExtendableStringBuilder<512> SB;
+		for (const std::filesystem::path& File : Files)
+		{
+			SB << File.generic_string() << "\n";
+		}
+		WriteFile(Root.Path() / "manifest.txt", IoBuffer(IoBuffer::Wrap, SB.ToView().data(), SB.ToView().length()));
+	}
+
+	BuildManifest Manifest = ParseBuildManifest(Root.Path() / "manifest.txt");
+	CHECK_EQ(Manifest.Parts.size(), 1u);
+	CHECK_EQ(Manifest.Parts[0].PartId, Oid::Zero);
+	CHECK_EQ(Manifest.Parts[0].PartName, "");
+	CHECK_EQ(Manifest.Parts[0].Files, Files);
+}
+
+TEST_CASE("buildmanifest.structured")
+{
+	ScopedTemporaryDirectory Root;
+
+	std::string Id = Oid::NewOid().ToString();
+
+	std::string ManifestString =
+		"{\n"
+		"    \"parts\": {\n"
+		"        \"default\": {\n"
+		"            \"partId\": \"098a2742d46c22a67ab57457\",\n"
+		"            \"files\": [\n"
+		"                \"foo/bar\",\n"
+		"                \"baz.exe\"\n"
+		"            ]\n"
+		"        },\n"
+		"        \"symbols\": {\n"
+		"            \"files\": [\n"
+		"                \"baz.pdb\"\n"
+		"            ]\n"
+		"        }\n"
+		"    }\n"
+		"}\n";
+
+	WriteFile(Root.Path() / "manifest.json", IoBuffer(IoBuffer::Wrap, ManifestString.data(), ManifestString.length()));
+
+	const Oid		  DefaultPartExpectedId	  = Oid::FromHexString("098a2742d46c22a67ab57457");
+	const std::string DefaultPartExpectedName = "default";
+	const Oid		  SymbolPartExpectedId	  = Oid::Zero;
+	const std::string SymbolsPartExpectedName = "symbols";
+
+	BuildManifest Manifest = ParseBuildManifest(Root.Path() / "manifest.json");
+	CHECK_EQ(Manifest.Parts.size(), 2u);
+	CHECK_EQ(Manifest.Parts[0].PartId, DefaultPartExpectedId);
+	CHECK_EQ(Manifest.Parts[0].PartName, DefaultPartExpectedName);
+	CHECK_EQ(Manifest.Parts[0].Files.size(), 2u);
+	CHECK_EQ(Manifest.Parts[0].Files[0].generic_string(), "foo/bar");
+	CHECK_EQ(Manifest.Parts[0].Files[1].generic_string(), "baz.exe");
+
+	CHECK_EQ(Manifest.Parts[1].PartId, SymbolPartExpectedId);
+	CHECK_EQ(Manifest.Parts[1].PartName, SymbolsPartExpectedName);
+	CHECK_EQ(Manifest.Parts[1].Files.size(), 1u);
+	CHECK_EQ(Manifest.Parts[1].Files[0].generic_string(), "baz.pdb");
+}
+
+void
+buildmanifest_forcelink()
+{
+}
+
+#endif	// ZEN_WITH_TESTS
+
+}  // namespace zen