move builds state functions to buildsavedstate.h/cpp (#577)

1 files changed, 443 insertions, 0 deletions

diff --git a/src/zenremotestore/builds/buildsavedstate.cpp b/src/zenremotestore/builds/buildsavedstate.cpp
new file mode 100644
index 000000000..933616856
--- /dev/null
+++ b/src/zenremotestore/builds/buildsavedstate.cpp
@@ -0,0 +1,443 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildsavedstate.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/trace.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#define EXTRA_VERIFY 0
+
+namespace zen {
+
+using namespace std::literals;
+
+namespace {
+	std::filesystem::path ZenStateDownloadFolder(const std::filesystem::path& SystemRootDir)
+	{
+		return SystemRootDir / "builds" / "downloads";
+	}
+}  // namespace
+
+CbObject
+CreateStateObject(const Oid&									  BuildId,
+				  const std::vector<std::pair<Oid, std::string>>& AllBuildParts,
+				  std::span<const ChunkedFolderContent>			  PartContents,
+				  const FolderContent&							  LocalFolderState,
+				  const std::filesystem::path&					  LocalPath)
+{
+	CbObjectWriter CurrentStateWriter;
+	CurrentStateWriter.AddString("path", (const char*)LocalPath.u8string().c_str());
+	CurrentStateWriter.BeginArray("builds"sv);
+	{
+		CurrentStateWriter.BeginObject();
+		{
+			CurrentStateWriter.AddObjectId("buildId"sv, BuildId);
+			CurrentStateWriter.BeginArray("parts"sv);
+			for (size_t PartIndex = 0; PartIndex < AllBuildParts.size(); PartIndex++)
+			{
+				const Oid BuildPartId = AllBuildParts[PartIndex].first;
+				CurrentStateWriter.BeginObject();
+				{
+					CurrentStateWriter.AddObjectId("partId"sv, BuildPartId);
+					CurrentStateWriter.AddString("partName"sv, AllBuildParts[PartIndex].second);
+					CurrentStateWriter.BeginObject("content");
+					{
+						SaveChunkedFolderContentToCompactBinary(PartContents[PartIndex], CurrentStateWriter);
+					}
+					CurrentStateWriter.EndObject();
+				}
+				CurrentStateWriter.EndObject();
+			}
+			CurrentStateWriter.EndArray();	// parts
+		}
+		CurrentStateWriter.EndObject();
+	}
+	CurrentStateWriter.EndArray();	// builds
+
+	CurrentStateWriter.BeginObject("localFolderState"sv);
+	{
+		SaveFolderContentToCompactBinary(LocalFolderState, CurrentStateWriter);
+	}
+	CurrentStateWriter.EndObject();	 // localFolderState
+
+	return CurrentStateWriter.Save();
+}
+
+void
+ReadBuildContentFromCompactBinary(CbObjectView						  BuildPartManifest,
+								  SourcePlatform&					  OutPlatform,
+								  std::vector<std::filesystem::path>& OutPaths,
+								  std::vector<IoHash>&				  OutRawHashes,
+								  std::vector<uint64_t>&			  OutRawSizes,
+								  std::vector<uint32_t>&			  OutAttributes,
+								  std::vector<IoHash>&				  OutSequenceRawHashes,
+								  std::vector<uint32_t>&			  OutChunkCounts,
+								  std::vector<uint32_t>&			  OutAbsoluteChunkOrders,
+								  std::vector<IoHash>&				  OutLooseChunkHashes,
+								  std::vector<uint64_t>&			  OutLooseChunkRawSizes,
+								  std::vector<IoHash>&				  OutBlockRawHashes)
+{
+	OutPlatform = FromString(BuildPartManifest["platform"sv].AsString(), SourcePlatform::_Count);
+
+	CbObjectView FilesObject = BuildPartManifest["files"sv].AsObjectView();
+
+	compactbinary_helpers::ReadArray("paths"sv, FilesObject, OutPaths);
+	compactbinary_helpers::ReadArray("rawhashes"sv, FilesObject, OutRawHashes);
+	compactbinary_helpers::ReadArray("rawsizes"sv, FilesObject, OutRawSizes);
+
+	uint64_t PathCount = OutPaths.size();
+	if (OutRawHashes.size() != PathCount)
+	{
+		throw std::runtime_error(fmt::format("Number of raw hashes entries does not match number of paths"));
+	}
+	if (OutRawSizes.size() != PathCount)
+	{
+		throw std::runtime_error(fmt::format("Number of raw sizes entries does not match number of paths"));
+	}
+
+	std::vector<uint32_t> ModeArray;
+	compactbinary_helpers::ReadArray("mode"sv, FilesObject, ModeArray);
+	if (ModeArray.size() != PathCount && ModeArray.size() != 0)
+	{
+		throw std::runtime_error(fmt::format("Number of attribute entries does not match number of paths"));
+	}
+
+	std::vector<uint32_t> AttributeArray;
+	compactbinary_helpers::ReadArray("attributes"sv, FilesObject, ModeArray);
+	if (AttributeArray.size() != PathCount && AttributeArray.size() != 0)
+	{
+		throw std::runtime_error(fmt::format("Number of attribute entries does not match number of paths"));
+	}
+
+	if (ModeArray.size() > 0)
+	{
+		if (OutPlatform == SourcePlatform::_Count)
+		{
+			OutPlatform = SourcePlatform::Linux;  // Best guess - under dev format
+		}
+		OutAttributes = std::move(ModeArray);
+	}
+	else if (AttributeArray.size() > 0)
+	{
+		if (OutPlatform == SourcePlatform::_Count)
+		{
+			OutPlatform = SourcePlatform::Windows;
+		}
+		OutAttributes = std::move(AttributeArray);
+	}
+	else
+	{
+		if (OutPlatform == SourcePlatform::_Count)
+		{
+			OutPlatform = GetSourceCurrentPlatform();
+		}
+	}
+
+	if (CbObjectView ChunkContentView = BuildPartManifest["chunkedContent"sv].AsObjectView(); ChunkContentView)
+	{
+		compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkContentView, OutSequenceRawHashes);
+		compactbinary_helpers::ReadArray("chunkcounts"sv, ChunkContentView, OutChunkCounts);
+		if (OutChunkCounts.size() != OutSequenceRawHashes.size())
+		{
+			throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths"));
+		}
+		compactbinary_helpers::ReadArray("chunkorders"sv, ChunkContentView, OutAbsoluteChunkOrders);
+	}
+	else if (FilesObject["chunkcounts"sv])
+	{
+		// Legacy zen style
+
+		std::vector<uint32_t> LegacyChunkCounts;
+		compactbinary_helpers::ReadArray("chunkcounts"sv, FilesObject, LegacyChunkCounts);
+		if (LegacyChunkCounts.size() != PathCount)
+		{
+			throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths"));
+		}
+		std::vector<uint32_t> LegacyAbsoluteChunkOrders;
+		compactbinary_helpers::ReadArray("chunkorders"sv, FilesObject, LegacyAbsoluteChunkOrders);
+
+		CbArrayView	   ChunkOrdersArray = BuildPartManifest["chunkorders"sv].AsArrayView();
+		const uint64_t ChunkOrdersCount = ChunkOrdersArray.Num();
+
+		tsl::robin_set<IoHash, IoHash::Hasher> FoundRawHashes;
+		FoundRawHashes.reserve(PathCount);
+
+		OutChunkCounts.reserve(PathCount);
+		OutAbsoluteChunkOrders.reserve(ChunkOrdersCount);
+
+		uint32_t OrderIndexOffset = 0;
+		for (uint32_t PathIndex = 0; PathIndex < OutPaths.size(); PathIndex++)
+		{
+			const IoHash& PathRawHash	   = OutRawHashes[PathIndex];
+			uint32_t	  LegacyChunkCount = LegacyChunkCounts[PathIndex];
+
+			if (FoundRawHashes.insert(PathRawHash).second)
+			{
+				OutSequenceRawHashes.push_back(PathRawHash);
+				OutChunkCounts.push_back(LegacyChunkCount);
+				std::span<uint32_t> AbsoluteChunkOrder =
+					std::span<uint32_t>(LegacyAbsoluteChunkOrders).subspan(OrderIndexOffset, LegacyChunkCount);
+				OutAbsoluteChunkOrders.insert(OutAbsoluteChunkOrders.end(), AbsoluteChunkOrder.begin(), AbsoluteChunkOrder.end());
+			}
+			OrderIndexOffset += LegacyChunkCounts[PathIndex];
+		}
+	}
+	else
+	{
+		// Legacy C# style
+
+		tsl::robin_set<IoHash, IoHash::Hasher> FoundRawHashes;
+		FoundRawHashes.reserve(PathCount);
+		uint32_t OrderIndexOffset = 0;
+		for (uint32_t PathIndex = 0; PathIndex < OutPaths.size(); PathIndex++)
+		{
+			if (OutRawSizes[PathIndex] > 0)
+			{
+				const IoHash& PathRawHash = OutRawHashes[PathIndex];
+				if (FoundRawHashes.insert(PathRawHash).second)
+				{
+					OutSequenceRawHashes.push_back(PathRawHash);
+					OutChunkCounts.push_back(1);
+					OutAbsoluteChunkOrders.push_back(OrderIndexOffset);
+					OutLooseChunkHashes.push_back(PathRawHash);
+					OutLooseChunkRawSizes.push_back(OutRawSizes[PathIndex]);
+					OrderIndexOffset += 1;
+				}
+			}
+		}
+	}
+
+	CbObjectView ChunkAttachmentsView = BuildPartManifest["chunkAttachments"sv].AsObjectView();
+	{
+		compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, ChunkAttachmentsView, OutLooseChunkHashes);
+		compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkAttachmentsView, OutLooseChunkRawSizes);
+		if (OutLooseChunkHashes.size() != OutLooseChunkRawSizes.size())
+		{
+			throw std::runtime_error(fmt::format("Number of attachment chunk hashes does not match number of attachemnt chunk raw sizes"));
+		}
+	}
+
+	CbObjectView BlocksView = BuildPartManifest["blockAttachments"sv].AsObjectView();
+	{
+		compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, BlocksView, OutBlockRawHashes);
+	}
+}
+
+void
+CalculateLocalChunkOrders(const std::span<const uint32_t>&				AbsoluteChunkOrders,
+						  const std::span<const IoHash>					LooseChunkHashes,
+						  const std::span<const uint64_t>				LooseChunkRawSizes,
+						  const std::span<const ChunkBlockDescription>& BlockDescriptions,
+						  std::vector<IoHash>&							OutLocalChunkHashes,
+						  std::vector<uint64_t>&						OutLocalChunkRawSizes,
+						  std::vector<uint32_t>&						OutLocalChunkOrders)
+{
+	ZEN_TRACE_CPU("CalculateLocalChunkOrders");
+
+	std::vector<IoHash>	  AbsoluteChunkHashes;
+	std::vector<uint64_t> AbsoluteChunkRawSizes;
+	AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), LooseChunkHashes.begin(), LooseChunkHashes.end());
+	AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), LooseChunkRawSizes.begin(), LooseChunkRawSizes.end());
+	for (const ChunkBlockDescription& Block : BlockDescriptions)
+	{
+		AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), Block.ChunkRawHashes.begin(), Block.ChunkRawHashes.end());
+		AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), Block.ChunkRawLengths.begin(), Block.ChunkRawLengths.end());
+	}
+	OutLocalChunkHashes.reserve(AbsoluteChunkHashes.size());
+	OutLocalChunkRawSizes.reserve(AbsoluteChunkRawSizes.size());
+	OutLocalChunkOrders.reserve(AbsoluteChunkOrders.size());
+
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+	ChunkHashToChunkIndex.reserve(AbsoluteChunkHashes.size());
+
+	for (uint32_t AbsoluteChunkOrderIndex = 0; AbsoluteChunkOrderIndex < AbsoluteChunkOrders.size(); AbsoluteChunkOrderIndex++)
+	{
+		const uint32_t AbsoluteChunkIndex	= AbsoluteChunkOrders[AbsoluteChunkOrderIndex];
+		const IoHash&  AbsoluteChunkHash	= AbsoluteChunkHashes[AbsoluteChunkIndex];
+		const uint64_t AbsoluteChunkRawSize = AbsoluteChunkRawSizes[AbsoluteChunkIndex];
+
+		if (auto It = ChunkHashToChunkIndex.find(AbsoluteChunkHash); It != ChunkHashToChunkIndex.end())
+		{
+			const uint32_t LocalChunkIndex = It->second;
+			OutLocalChunkOrders.push_back(LocalChunkIndex);
+		}
+		else
+		{
+			uint32_t LocalChunkIndex = gsl::narrow<uint32_t>(OutLocalChunkHashes.size());
+			OutLocalChunkHashes.push_back(AbsoluteChunkHash);
+			OutLocalChunkRawSizes.push_back(AbsoluteChunkRawSize);
+			OutLocalChunkOrders.push_back(LocalChunkIndex);
+			ChunkHashToChunkIndex.insert_or_assign(AbsoluteChunkHash, LocalChunkIndex);
+		}
+#if EXTRA_VERIFY
+		const uint32_t	LocalChunkIndex	  = OutLocalChunkOrders[AbsoluteChunkOrderIndex];
+		const IoHash&	LocalChunkHash	  = OutLocalChunkHashes[LocalChunkIndex];
+		const uint64_t& LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex];
+		ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
+		ZEN_ASSERT(LocalChunkRawSize == AbsoluteChunkRawSize);
+#endif	// EXTRA_VERIFY
+	}
+#if EXTRA_VERIFY
+	for (uint32_t OrderIndex = 0; OrderIndex < OutLocalChunkOrders.size(); OrderIndex++)
+	{
+		uint32_t	 LocalChunkIndex   = OutLocalChunkOrders[OrderIndex];
+		const IoHash LocalChunkHash	   = OutLocalChunkHashes[LocalChunkIndex];
+		uint64_t	 LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex];
+
+		uint32_t	 VerifyChunkIndex	= AbsoluteChunkOrders[OrderIndex];
+		const IoHash VerifyChunkHash	= AbsoluteChunkHashes[VerifyChunkIndex];
+		uint64_t	 VerifyChunkRawSize = AbsoluteChunkRawSizes[VerifyChunkIndex];
+
+		ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
+		ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
+	}
+#endif	// EXTRA_VERIFY
+}
+
+void
+ReadStateObject(CbObjectView					   StateView,
+				Oid&							   OutBuildId,
+				std::vector<Oid>&				   BuildPartsIds,
+				std::vector<std::string>&		   BuildPartsNames,
+				std::vector<ChunkedFolderContent>& OutPartContents,
+				FolderContent&					   OutLocalFolderState)
+{
+	CbObjectView BuildView = StateView["builds"sv].AsArrayView().CreateViewIterator().AsObjectView();
+	OutBuildId			   = BuildView["buildId"sv].AsObjectId();
+	for (CbFieldView PartView : BuildView["parts"sv].AsArrayView())
+	{
+		CbObjectView PartObjectView = PartView.AsObjectView();
+		BuildPartsIds.push_back(PartObjectView["partId"sv].AsObjectId());
+		BuildPartsNames.push_back(std::string(PartObjectView["partName"sv].AsString()));
+		OutPartContents.push_back(LoadChunkedFolderContentToCompactBinary(PartObjectView["content"sv].AsObjectView()));
+	}
+	OutLocalFolderState = LoadFolderContentToCompactBinary(StateView["localFolderState"sv].AsObjectView());
+}
+void
+ReadStateFile(const std::filesystem::path& StateFilePath, FolderContent& OutLocalFolderState, ChunkedFolderContent& OutLocalContent)
+{
+	ZEN_TRACE_CPU("ReadStateFile");
+
+	FileContents	FileData = ReadFile(StateFilePath);
+	CbValidateError ValidateError;
+	if (CbObject CurrentStateObject = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError);
+		ValidateError == CbValidateError::None)
+	{
+		if (CurrentStateObject)
+		{
+			Oid								  CurrentBuildId;
+			std::vector<Oid>				  SavedBuildPartIds;
+			std::vector<std::string>		  SavedBuildPartsNames;
+			std::vector<ChunkedFolderContent> SavedPartContents;
+			ReadStateObject(CurrentStateObject,
+							CurrentBuildId,
+							SavedBuildPartIds,
+							SavedBuildPartsNames,
+							SavedPartContents,
+							OutLocalFolderState);
+			if (!SavedPartContents.empty())
+			{
+				if (SavedPartContents.size() == 1)
+				{
+					OutLocalContent = std::move(SavedPartContents[0]);
+				}
+				else
+				{
+					OutLocalContent = MergeChunkedFolderContents(SavedPartContents[0],
+																 std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1));
+				}
+			}
+		}
+	}
+	else
+	{
+		throw std::runtime_error(
+			fmt::format("Failed to read compact binary object from '{}', reason: {}", StateFilePath, ToString(ValidateError)));
+	}
+}
+
+void
+AddDownloadedPath(const std::filesystem::path&					  SystemRootDir,
+				  const Oid&									  BuildId,
+				  const std::vector<std::pair<Oid, std::string>>& BuildParts,
+				  const std::filesystem::path&					  StateFilePath,
+				  const std::filesystem::path&					  LocalPath)
+{
+	ZEN_TRACE_CPU("AddDownloadedPath");
+
+	ZEN_ASSERT(!SystemRootDir.empty());
+	ZEN_ASSERT(!StateFilePath.empty());
+	ZEN_ASSERT(!LocalPath.empty());
+	const std::u8string			LocalPathString		= LocalPath.generic_u8string();
+	IoHash						PathHash			= IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length());
+	const std::filesystem::path StateDownloadFolder = ZenStateDownloadFolder(SystemRootDir);
+	CreateDirectories(StateDownloadFolder);
+	std::filesystem::path WritePath = StateDownloadFolder / (PathHash.ToHexString() + ".json");
+
+	CbObjectWriter Writer;
+	Writer.AddString("path", (const char*)LocalPath.u8string().c_str());
+	Writer.AddString("statePath", (const char*)StateFilePath.u8string().c_str());
+	Writer.AddDateTime("date", DateTime::Now());
+	Writer.BeginArray("builds"sv);
+	{
+		Writer.BeginObject();
+		{
+			Writer.AddObjectId("buildId", BuildId);
+			Writer.BeginArray("parts");
+			for (const auto& It : BuildParts)
+			{
+				Writer.BeginObject();
+				{
+					Writer.AddObjectId("partId", It.first);
+					Writer.AddString("partName", It.second);
+				}
+				Writer.EndObject();
+			}
+			Writer.EndArray();	// parts
+		}
+		Writer.EndObject();
+	}
+	Writer.EndArray();	// builds
+
+	CbObject					 Payload = Writer.Save();
+	ExtendableStringBuilder<512> SB;
+	CompactBinaryToJson(Payload.GetView(), SB);
+	MemoryView JsonPayload(SB.Data(), SB.Size());
+	TemporaryFile::SafeWriteFile(WritePath, JsonPayload);
+}
+
+std::vector<std::filesystem::path>
+GetDownloadedStatePaths(const std::filesystem::path& SystemRootDir)
+{
+	ZEN_ASSERT(!SystemRootDir.empty());
+
+	ZEN_TRACE_CPU("GetDownloadedPaths");
+
+	std::vector<std::filesystem::path> Result;
+
+	const std::filesystem::path StateDownloadFolder = ZenStateDownloadFolder(SystemRootDir);
+	if (IsDir(StateDownloadFolder))
+	{
+		DirectoryContent Content;
+		GetDirectoryContent(ZenStateDownloadFolder(SystemRootDir), DirectoryContentFlags::IncludeFiles, Content);
+		for (const std::filesystem::path& EntryPath : Content.Files)
+		{
+			IoHash EntryPathHash;
+			if (IoHash::TryParse(EntryPath.stem().string(), EntryPathHash))
+			{
+				Result.push_back(EntryPath);
+			}
+		}
+	}
+	return Result;
+}
+
+}  // namespace zen