update state when wildcard (#657)

* add --append option and improve state handling when using downloads for `zen builds download`

1 files changed, 551 insertions, 294 deletions

diff --git a/src/zenremotestore/builds/buildsavedstate.cpp b/src/zenremotestore/builds/buildsavedstate.cpp
index cf46668f9..5a86ee865 100644
--- a/src/zenremotestore/builds/buildsavedstate.cpp
+++ b/src/zenremotestore/builds/buildsavedstate.cpp
@@ -13,7 +13,10 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_set.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
-#define EXTRA_VERIFY 0
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#endif	// ZEN_WITH_TESTS
 
 namespace zen {
 
@@ -26,299 +29,181 @@ namespace {
 	}
 }  // namespace
 
-CbObject
-CreateStateObject(const Oid&									  BuildId,
-				  const std::vector<std::pair<Oid, std::string>>& AllBuildParts,
-				  std::span<const ChunkedFolderContent>			  PartContents,
-				  const FolderContent&							  LocalFolderState,
-				  const std::filesystem::path&					  LocalPath)
+///////////////////////// BuildsSelection
+
+void
+BuildsSelection::Write(const BuildsSelection& Selection, CbWriter& Output)
 {
-	CbObjectWriter CurrentStateWriter;
-	CurrentStateWriter.AddString("path", (const char*)LocalPath.u8string().c_str());
-	CurrentStateWriter.BeginArray("builds"sv);
+	Output.BeginArray("builds"sv);
+	for (const BuildsSelection::Build& Build : Selection.Builds)
 	{
-		CurrentStateWriter.BeginObject();
+		Output.BeginObject();
 		{
-			CurrentStateWriter.AddObjectId("buildId"sv, BuildId);
-			CurrentStateWriter.BeginArray("parts"sv);
-			for (size_t PartIndex = 0; PartIndex < AllBuildParts.size(); PartIndex++)
+			ZEN_ASSERT(Build.Id != Oid::Zero);
+			Output.AddObjectId("buildId"sv, Build.Id);
+			compactbinary_helpers::WriteArray(Build.IncludeWildcards, "includeWildcards"sv, Output);
+			compactbinary_helpers::WriteArray(Build.ExcludeWildcards, "excludeWildcards"sv, Output);
+
+			Output.BeginArray("parts"sv);
+			for (const BuildsSelection::BuildPart& BuildPart : Build.Parts)
 			{
-				const Oid BuildPartId = AllBuildParts[PartIndex].first;
-				CurrentStateWriter.BeginObject();
+				Output.BeginObject();
 				{
-					CurrentStateWriter.AddObjectId("partId"sv, BuildPartId);
-					CurrentStateWriter.AddString("partName"sv, AllBuildParts[PartIndex].second);
-					CurrentStateWriter.BeginObject("content");
-					{
-						SaveChunkedFolderContentToCompactBinary(PartContents[PartIndex], CurrentStateWriter);
-					}
-					CurrentStateWriter.EndObject();
+					ZEN_ASSERT(BuildPart.Id != Oid::Zero);
+					Output.AddObjectId("partId"sv, BuildPart.Id);
+					Output.AddString("partName"sv, BuildPart.Name);
 				}
-				CurrentStateWriter.EndObject();
+				Output.EndObject();
 			}
-			CurrentStateWriter.EndArray();	// parts
+			Output.EndArray();	// parts
 		}
-		CurrentStateWriter.EndObject();
+		Output.EndObject();
 	}
-	CurrentStateWriter.EndArray();	// builds
-
-	CurrentStateWriter.BeginObject("localFolderState"sv);
-	{
-		SaveFolderContentToCompactBinary(LocalFolderState, CurrentStateWriter);
-	}
-	CurrentStateWriter.EndObject();	 // localFolderState
-
-	return CurrentStateWriter.Save();
+	Output.EndArray();	// builds
 }
 
-void
-ReadBuildContentFromCompactBinary(CbObjectView						  BuildPartManifest,
-								  SourcePlatform&					  OutPlatform,
-								  std::vector<std::filesystem::path>& OutPaths,
-								  std::vector<IoHash>&				  OutRawHashes,
-								  std::vector<uint64_t>&			  OutRawSizes,
-								  std::vector<uint32_t>&			  OutAttributes,
-								  std::vector<IoHash>&				  OutSequenceRawHashes,
-								  std::vector<uint32_t>&			  OutChunkCounts,
-								  std::vector<uint32_t>&			  OutAbsoluteChunkOrders,
-								  std::vector<IoHash>&				  OutLooseChunkHashes,
-								  std::vector<uint64_t>&			  OutLooseChunkRawSizes,
-								  std::vector<IoHash>&				  OutBlockRawHashes)
+BuildsSelection
+BuildsSelection::Read(CbObjectView& Input, std::vector<ChunkedFolderContent>* OptionalOutLegacyPartsContent)
 {
-	OutPlatform = FromString(BuildPartManifest["platform"sv].AsString(), SourcePlatform::_Count);
-
-	CbObjectView FilesObject = BuildPartManifest["files"sv].AsObjectView();
-
-	OutPaths	 = compactbinary_helpers::ReadArray<std::filesystem::path>("paths"sv, FilesObject);
-	OutRawHashes = compactbinary_helpers::ReadArray<IoHash>("rawhashes"sv, FilesObject);
-	OutRawSizes	 = compactbinary_helpers::ReadArray<uint64_t>("rawsizes"sv, FilesObject);
-
-	uint64_t PathCount = OutPaths.size();
-	if (OutRawHashes.size() != PathCount)
-	{
-		throw std::runtime_error(fmt::format("Number of raw hashes entries does not match number of paths"));
-	}
-	if (OutRawSizes.size() != PathCount)
-	{
-		throw std::runtime_error(fmt::format("Number of raw sizes entries does not match number of paths"));
-	}
+	std::vector<BuildsSelection::Build> Builds;
 
-	std::vector<uint32_t> ModeArray = compactbinary_helpers::ReadArray<uint32_t>("mode"sv, FilesObject);
-	if (ModeArray.size() != PathCount && ModeArray.size() != 0)
+	CbArrayView BuildsArray = Input["builds"sv].AsArrayView();
+	Builds.reserve(BuildsArray.Num());
+	for (CbFieldView BuildView : BuildsArray)
 	{
-		throw std::runtime_error(fmt::format("Number of attribute entries does not match number of paths"));
-	}
-
-	std::vector<uint32_t> AttributeArray = compactbinary_helpers::ReadArray<uint32_t>("attributes"sv, FilesObject);
-	if (AttributeArray.size() != PathCount && AttributeArray.size() != 0)
-	{
-		throw std::runtime_error(fmt::format("Number of attribute entries does not match number of paths"));
-	}
-
-	if (ModeArray.size() > 0)
-	{
-		if (OutPlatform == SourcePlatform::_Count)
-		{
-			OutPlatform = SourcePlatform::Linux;  // Best guess - under dev format
-		}
-		OutAttributes = std::move(ModeArray);
-	}
-	else if (AttributeArray.size() > 0)
-	{
-		if (OutPlatform == SourcePlatform::_Count)
+		std::vector<BuildsSelection::BuildPart> Parts;
+		CbObjectView							BuildObjectView = BuildView.AsObjectView();
+		Oid										BuildId			= BuildObjectView["buildId"sv].AsObjectId();
+		if (BuildId == Oid::Zero)
 		{
-			OutPlatform = SourcePlatform::Windows;
+			throw std::runtime_error(fmt::format("BuildsSelection build id is invalid '{}'", BuildId));
 		}
-		OutAttributes = std::move(AttributeArray);
-	}
-	else
-	{
-		if (OutPlatform == SourcePlatform::_Count)
-		{
-			OutPlatform = GetSourceCurrentPlatform();
-		}
-	}
 
-	if (CbObjectView ChunkContentView = BuildPartManifest["chunkedContent"sv].AsObjectView(); ChunkContentView)
-	{
-		OutSequenceRawHashes = compactbinary_helpers::ReadArray<IoHash>("sequenceRawHashes"sv, ChunkContentView);
-		OutChunkCounts		 = compactbinary_helpers::ReadArray<uint32_t>("chunkcounts"sv, ChunkContentView);
-		if (OutChunkCounts.size() != OutSequenceRawHashes.size())
-		{
-			throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths"));
-		}
-		OutAbsoluteChunkOrders = compactbinary_helpers::ReadArray<uint32_t>("chunkorders"sv, ChunkContentView);
-	}
-	else if (FilesObject["chunkcounts"sv])
-	{
-		// Legacy zen style
-
-		std::vector<uint32_t> LegacyChunkCounts = compactbinary_helpers::ReadArray<uint32_t>("chunkcounts"sv, FilesObject);
-		if (LegacyChunkCounts.size() != PathCount)
-		{
-			throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths"));
-		}
-		std::vector<uint32_t> LegacyAbsoluteChunkOrders = compactbinary_helpers::ReadArray<uint32_t>("chunkorders"sv, FilesObject);
-
-		CbArrayView	   ChunkOrdersArray = BuildPartManifest["chunkorders"sv].AsArrayView();
-		const uint64_t ChunkOrdersCount = ChunkOrdersArray.Num();
-
-		tsl::robin_set<IoHash, IoHash::Hasher> FoundRawHashes;
-		FoundRawHashes.reserve(PathCount);
-
-		OutChunkCounts.reserve(PathCount);
-		OutAbsoluteChunkOrders.reserve(ChunkOrdersCount);
+		std::vector<std::string> IncludeWildcards = compactbinary_helpers::ReadArray<std::string>("includeWildcards"sv, BuildObjectView);
+		std::vector<std::string> ExcludeWildcards = compactbinary_helpers::ReadArray<std::string>("excludeWildcards"sv, BuildObjectView);
 
-		uint32_t OrderIndexOffset = 0;
-		for (uint32_t PathIndex = 0; PathIndex < OutPaths.size(); PathIndex++)
+		CbArrayView BuildPartsArray = BuildObjectView["parts"sv].AsArrayView();
+		Parts.reserve(BuildPartsArray.Num());
+		for (CbFieldView PartView : BuildPartsArray)
 		{
-			const IoHash& PathRawHash	   = OutRawHashes[PathIndex];
-			uint32_t	  LegacyChunkCount = LegacyChunkCounts[PathIndex];
-
-			if (FoundRawHashes.insert(PathRawHash).second)
+			CbObjectView PartObjectView = PartView.AsObjectView();
+			Oid			 BuildPartId	= PartObjectView["partId"sv].AsObjectId();
+			if (BuildPartId == Oid::Zero)
 			{
-				OutSequenceRawHashes.push_back(PathRawHash);
-				OutChunkCounts.push_back(LegacyChunkCount);
-				std::span<uint32_t> AbsoluteChunkOrder =
-					std::span<uint32_t>(LegacyAbsoluteChunkOrders).subspan(OrderIndexOffset, LegacyChunkCount);
-				OutAbsoluteChunkOrders.insert(OutAbsoluteChunkOrders.end(), AbsoluteChunkOrder.begin(), AbsoluteChunkOrder.end());
+				throw std::runtime_error(fmt::format("BuildsSelection build part id in build '{}' is invalid '{}'", BuildId, BuildPartId));
 			}
-			OrderIndexOffset += LegacyChunkCounts[PathIndex];
-		}
-	}
-	else
-	{
-		// Legacy C# style
+			std::string_view BuildPartName = PartObjectView["partName"sv].AsString();
 
-		tsl::robin_set<IoHash, IoHash::Hasher> FoundRawHashes;
-		FoundRawHashes.reserve(PathCount);
-		uint32_t OrderIndexOffset = 0;
-		for (uint32_t PathIndex = 0; PathIndex < OutPaths.size(); PathIndex++)
-		{
-			if (OutRawSizes[PathIndex] > 0)
+			if (OptionalOutLegacyPartsContent != nullptr)
 			{
-				const IoHash& PathRawHash = OutRawHashes[PathIndex];
-				if (FoundRawHashes.insert(PathRawHash).second)
+				CbObjectView PartContentObjectView = PartObjectView["content"sv].AsObjectView();
+				if (PartContentObjectView)
 				{
-					OutSequenceRawHashes.push_back(PathRawHash);
-					OutChunkCounts.push_back(1);
-					OutAbsoluteChunkOrders.push_back(OrderIndexOffset);
-					OutLooseChunkHashes.push_back(PathRawHash);
-					OutLooseChunkRawSizes.push_back(OutRawSizes[PathIndex]);
-					OrderIndexOffset += 1;
+					OptionalOutLegacyPartsContent->push_back(LoadChunkedFolderContentFromCompactBinary(PartContentObjectView));
 				}
 			}
+			Parts.push_back(BuildsSelection::BuildPart{.Id = BuildPartId, .Name = std::string(BuildPartName)});
 		}
-	}
 
-	CbObjectView ChunkAttachmentsView = BuildPartManifest["chunkAttachments"sv].AsObjectView();
-	{
-		OutLooseChunkHashes	  = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, ChunkAttachmentsView);
-		OutLooseChunkRawSizes = compactbinary_helpers::ReadArray<uint64_t>("chunkRawSizes"sv, ChunkAttachmentsView);
-		if (OutLooseChunkHashes.size() != OutLooseChunkRawSizes.size())
-		{
-			throw std::runtime_error(fmt::format("Number of attachment chunk hashes does not match number of attachemnt chunk raw sizes"));
-		}
-	}
-
-	CbObjectView BlocksView = BuildPartManifest["blockAttachments"sv].AsObjectView();
-	{
-		OutBlockRawHashes = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, BlocksView);
+		Builds.push_back(BuildsSelection::Build{.Id				  = BuildId,
+												.Parts			  = std::move(Parts),
+												.IncludeWildcards = std::move(IncludeWildcards),
+												.ExcludeWildcards = std::move(ExcludeWildcards)});
 	}
+	return BuildsSelection{.Builds = std::move(Builds)};
 }
 
+///////////////////////// BuildState
+
 void
-CalculateLocalChunkOrders(const std::span<const uint32_t>&				AbsoluteChunkOrders,
-						  const std::span<const IoHash>					LooseChunkHashes,
-						  const std::span<const uint64_t>				LooseChunkRawSizes,
-						  const std::span<const ChunkBlockDescription>& BlockDescriptions,
-						  std::vector<IoHash>&							OutLocalChunkHashes,
-						  std::vector<uint64_t>&						OutLocalChunkRawSizes,
-						  std::vector<uint32_t>&						OutLocalChunkOrders)
+BuildState::Write(const BuildState& State, CbWriter& Output)
 {
-	ZEN_TRACE_CPU("CalculateLocalChunkOrders");
+	BuildsSelection::Write(State.Selection, Output);
 
-	std::vector<IoHash>	  AbsoluteChunkHashes;
-	std::vector<uint64_t> AbsoluteChunkRawSizes;
-	AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), LooseChunkHashes.begin(), LooseChunkHashes.end());
-	AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), LooseChunkRawSizes.begin(), LooseChunkRawSizes.end());
-	for (const ChunkBlockDescription& Block : BlockDescriptions)
+	Output.BeginObject("content");
 	{
-		AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), Block.ChunkRawHashes.begin(), Block.ChunkRawHashes.end());
-		AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), Block.ChunkRawLengths.begin(), Block.ChunkRawLengths.end());
+		SaveChunkedFolderContentToCompactBinary(State.ChunkedContent, Output);
 	}
-	OutLocalChunkHashes.reserve(AbsoluteChunkHashes.size());
-	OutLocalChunkRawSizes.reserve(AbsoluteChunkRawSizes.size());
-	OutLocalChunkOrders.reserve(AbsoluteChunkOrders.size());
+	Output.EndObject();	 // content
+}
 
-	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
-	ChunkHashToChunkIndex.reserve(AbsoluteChunkHashes.size());
+BuildState
+BuildState::Read(CbObjectView& Input)
+{
+	std::vector<ChunkedFolderContent> LegacyPartsContent;
+	BuildsSelection					  Selection = BuildsSelection::Read(Input, &LegacyPartsContent);
 
-	for (uint32_t AbsoluteChunkOrderIndex = 0; AbsoluteChunkOrderIndex < AbsoluteChunkOrders.size(); AbsoluteChunkOrderIndex++)
+	ChunkedFolderContent ChunkedContent;
+	CbObjectView		 ContentObjectView = Input["content"sv].AsObjectView();
+	if (ContentObjectView)
 	{
-		const uint32_t AbsoluteChunkIndex	= AbsoluteChunkOrders[AbsoluteChunkOrderIndex];
-		const IoHash&  AbsoluteChunkHash	= AbsoluteChunkHashes[AbsoluteChunkIndex];
-		const uint64_t AbsoluteChunkRawSize = AbsoluteChunkRawSizes[AbsoluteChunkIndex];
-
-		if (auto It = ChunkHashToChunkIndex.find(AbsoluteChunkHash); It != ChunkHashToChunkIndex.end())
+		ZEN_ASSERT(LegacyPartsContent.empty());
+		ChunkedContent = LoadChunkedFolderContentFromCompactBinary(ContentObjectView);
+	}
+	else
+	{
+		if (LegacyPartsContent.size() == 1)
 		{
-			const uint32_t LocalChunkIndex = It->second;
-			OutLocalChunkOrders.push_back(LocalChunkIndex);
+			ChunkedContent = std::move(LegacyPartsContent.front());
 		}
 		else
 		{
-			uint32_t LocalChunkIndex = gsl::narrow<uint32_t>(OutLocalChunkHashes.size());
-			OutLocalChunkHashes.push_back(AbsoluteChunkHash);
-			OutLocalChunkRawSizes.push_back(AbsoluteChunkRawSize);
-			OutLocalChunkOrders.push_back(LocalChunkIndex);
-			ChunkHashToChunkIndex.insert_or_assign(AbsoluteChunkHash, LocalChunkIndex);
-		}
-#if EXTRA_VERIFY
-		const uint32_t	LocalChunkIndex	  = OutLocalChunkOrders[AbsoluteChunkOrderIndex];
-		const IoHash&	LocalChunkHash	  = OutLocalChunkHashes[LocalChunkIndex];
-		const uint64_t& LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex];
-		ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
-		ZEN_ASSERT(LocalChunkRawSize == AbsoluteChunkRawSize);
-#endif	// EXTRA_VERIFY
+			ChunkedContent =
+				MergeChunkedFolderContents(LegacyPartsContent[0], std::span<const ChunkedFolderContent>(LegacyPartsContent).subspan(1));
+		}
 	}
-#if EXTRA_VERIFY
-	for (uint32_t OrderIndex = 0; OrderIndex < OutLocalChunkOrders.size(); OrderIndex++)
-	{
-		uint32_t	 LocalChunkIndex   = OutLocalChunkOrders[OrderIndex];
-		const IoHash LocalChunkHash	   = OutLocalChunkHashes[LocalChunkIndex];
-		uint64_t	 LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex];
 
-		uint32_t	 VerifyChunkIndex	= AbsoluteChunkOrders[OrderIndex];
-		const IoHash VerifyChunkHash	= AbsoluteChunkHashes[VerifyChunkIndex];
-		uint64_t	 VerifyChunkRawSize = AbsoluteChunkRawSizes[VerifyChunkIndex];
+	return BuildState{.Selection = std::move(Selection), .ChunkedContent = std::move(ChunkedContent)};
+}
+
+///////////////////////// BuildSaveState
 
-		ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
-		ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
+void
+BuildSaveState::Write(const BuildSaveState& SaveState, CbWriter& Output)
+{
+	ZEN_ASSERT(!SaveState.LocalPath.empty());
+
+	Output.AddString("path", (const char*)SaveState.LocalPath.u8string().c_str());
+	BuildsSelection::Write(SaveState.State.Selection, Output);
+
+	Output.BeginObject("content");
+	{
+		SaveChunkedFolderContentToCompactBinary(SaveState.State.ChunkedContent, Output);
 	}
-#endif	// EXTRA_VERIFY
+	Output.EndObject();	 // content
+
+	Output.BeginObject("localFolderState"sv);
+	{
+		SaveFolderContentToCompactBinary(SaveState.FolderState, Output);
+	}
+	Output.EndObject();	 // localFolderState
 }
 
-void
-ReadStateObject(CbObjectView					   StateView,
-				Oid&							   OutBuildId,
-				std::vector<Oid>&				   BuildPartsIds,
-				std::vector<std::string>&		   BuildPartsNames,
-				std::vector<ChunkedFolderContent>& OutPartContents,
-				FolderContent&					   OutLocalFolderState)
+BuildSaveState
+BuildSaveState::Read(CbObjectView& Input)
 {
-	CbObjectView BuildView = StateView["builds"sv].AsArrayView().CreateViewIterator().AsObjectView();
-	OutBuildId			   = BuildView["buildId"sv].AsObjectId();
-	for (CbFieldView PartView : BuildView["parts"sv].AsArrayView())
+	BuildState			  State					 = BuildState::Read(Input);
+	CbObjectView		  LocalFolderStateObject = Input["localFolderState"sv].AsObjectView();
+	FolderContent		  FolderState			 = LoadFolderContentToCompactBinary(LocalFolderStateObject);
+	std::filesystem::path LocalPath				 = Input["path"sv].AsU8String();
+	if (LocalPath.empty())
 	{
-		CbObjectView PartObjectView = PartView.AsObjectView();
-		BuildPartsIds.push_back(PartObjectView["partId"sv].AsObjectId());
-		BuildPartsNames.push_back(std::string(PartObjectView["partName"sv].AsString()));
-		OutPartContents.push_back(LoadChunkedFolderContentToCompactBinary(PartObjectView["content"sv].AsObjectView()));
+		throw std::runtime_error("BuildSaveState is invalid, 'path' field is empty");
 	}
-	OutLocalFolderState = LoadFolderContentToCompactBinary(StateView["localFolderState"sv].AsObjectView());
+
+	return BuildSaveState{.State = std::move(State), .FolderState = std::move(FolderState), .LocalPath = std::move(LocalPath)};
 }
-void
-ReadStateFile(const std::filesystem::path& StateFilePath, FolderContent& OutLocalFolderState, ChunkedFolderContent& OutLocalContent)
+
+CbObject
+CreateBuildSaveStateObject(const BuildSaveState& SaveState)
+{
+	CbObjectWriter CurrentStateWriter;
+	BuildSaveState::Write(SaveState, CurrentStateWriter);
+	return CurrentStateWriter.Save();
+}
+
+BuildSaveState
+ReadBuildSaveStateFile(const std::filesystem::path& StateFilePath)
 {
 	ZEN_TRACE_CPU("ReadStateFile");
 
@@ -329,28 +214,11 @@ ReadStateFile(const std::filesystem::path& StateFilePath, FolderContent& OutLoca
 	{
 		if (CurrentStateObject)
 		{
-			Oid								  CurrentBuildId;
-			std::vector<Oid>				  SavedBuildPartIds;
-			std::vector<std::string>		  SavedBuildPartsNames;
-			std::vector<ChunkedFolderContent> SavedPartContents;
-			ReadStateObject(CurrentStateObject,
-							CurrentBuildId,
-							SavedBuildPartIds,
-							SavedBuildPartsNames,
-							SavedPartContents,
-							OutLocalFolderState);
-			if (!SavedPartContents.empty())
-			{
-				if (SavedPartContents.size() == 1)
-				{
-					OutLocalContent = std::move(SavedPartContents[0]);
-				}
-				else
-				{
-					OutLocalContent = MergeChunkedFolderContents(SavedPartContents[0],
-																 std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1));
-				}
-			}
+			return BuildSaveState::Read(CurrentStateObject);
+		}
+		else
+		{
+			throw std::runtime_error(fmt::format("Compact binary object read from '{}' is empty", StateFilePath));
 		}
 	}
 	else
@@ -360,48 +228,61 @@ ReadStateFile(const std::filesystem::path& StateFilePath, FolderContent& OutLoca
 	}
 }
 
+///////////////////////// BuildsDownloadInfo
+
+void
+BuildsDownloadInfo::Write(const BuildsDownloadInfo& Info, CbWriter& Output)
+{
+	ZEN_ASSERT(!Info.LocalPath.empty());
+	ZEN_ASSERT(!Info.StateFilePath.empty());
+	ZEN_ASSERT(!Info.Iso8601Date.empty());
+
+	Output.AddString("path", (const char*)Info.LocalPath.u8string().c_str());
+	Output.AddString("statePath", (const char*)Info.StateFilePath.u8string().c_str());
+	Output.AddString("date", Info.Iso8601Date);
+
+	BuildsSelection::Write(Info.Selection, Output);
+}
+
+BuildsDownloadInfo
+BuildsDownloadInfo::Read(CbObjectView& Input)
+{
+	BuildsSelection Selection = BuildsSelection::Read(Input, /*OptionalOutLegacyPartsContent*/ nullptr);
+
+	std::filesystem::path LocalPath = Input["path"sv].AsU8String();
+	if (LocalPath.empty())
+	{
+		throw std::runtime_error("BuildsDownloadInfo is invalid, 'path' field is empty");
+	}
+	std::filesystem::path StateFilePath = Input["statePath"sv].AsU8String();
+	if (StateFilePath.empty())
+	{
+		throw std::runtime_error("BuildsDownloadInfo is invalid, 'statePath' field is empty");
+	}
+	std::string_view Iso8601Date = Input["date"sv].AsString();
+
+	return BuildsDownloadInfo{.Selection	 = std::move(Selection),
+							  .LocalPath	 = std::move(LocalPath),
+							  .StateFilePath = std::move(StateFilePath),
+							  .Iso8601Date	 = std::string(Iso8601Date)};
+}
+
 void
-AddDownloadedPath(const std::filesystem::path&					  SystemRootDir,
-				  const Oid&									  BuildId,
-				  const std::vector<std::pair<Oid, std::string>>& BuildParts,
-				  const std::filesystem::path&					  StateFilePath,
-				  const std::filesystem::path&					  LocalPath)
+AddDownloadedPath(const std::filesystem::path& SystemRootDir, const BuildsDownloadInfo& Info)
 {
 	ZEN_TRACE_CPU("AddDownloadedPath");
 
 	ZEN_ASSERT(!SystemRootDir.empty());
-	ZEN_ASSERT(!StateFilePath.empty());
-	ZEN_ASSERT(!LocalPath.empty());
-	const std::u8string			LocalPathString		= LocalPath.generic_u8string();
+	ZEN_ASSERT(!Info.StateFilePath.empty());
+	ZEN_ASSERT(!Info.LocalPath.empty());
+	const std::u8string			LocalPathString		= Info.LocalPath.generic_u8string();
 	IoHash						PathHash			= IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length());
 	const std::filesystem::path StateDownloadFolder = ZenStateDownloadFolder(SystemRootDir);
 	CreateDirectories(StateDownloadFolder);
 	std::filesystem::path WritePath = StateDownloadFolder / (PathHash.ToHexString() + ".json");
 
 	CbObjectWriter Writer;
-	Writer.AddString("path", (const char*)LocalPath.u8string().c_str());
-	Writer.AddString("statePath", (const char*)StateFilePath.u8string().c_str());
-	Writer.AddDateTime("date", DateTime::Now());
-	Writer.BeginArray("builds"sv);
-	{
-		Writer.BeginObject();
-		{
-			Writer.AddObjectId("buildId", BuildId);
-			Writer.BeginArray("parts");
-			for (const auto& It : BuildParts)
-			{
-				Writer.BeginObject();
-				{
-					Writer.AddObjectId("partId", It.first);
-					Writer.AddString("partName", It.second);
-				}
-				Writer.EndObject();
-			}
-			Writer.EndArray();	// parts
-		}
-		Writer.EndObject();
-	}
-	Writer.EndArray();	// builds
+	BuildsDownloadInfo::Write(Info, Writer);
 
 	CbObject					 Payload = Writer.Save();
 	ExtendableStringBuilder<512> SB;
@@ -410,6 +291,21 @@ AddDownloadedPath(const std::filesystem::path&					  SystemRootDir,
 	TemporaryFile::SafeWriteFile(WritePath, JsonPayload);
 }
 
+BuildsDownloadInfo
+ReadDownloadedInfoFile(const std::filesystem::path& DownloadInfoPath)
+{
+	IoBuffer		 MetaDataJson = ReadFile(DownloadInfoPath).Flatten();
+	std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+	std::string		 JsonError;
+	CbObject		 DownloadInfo = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+	if (!JsonError.empty())
+	{
+		throw std::runtime_error(fmt::format("Invalid download state file at {}. '{}'", DownloadInfoPath, JsonError));
+	}
+
+	return BuildsDownloadInfo::Read(DownloadInfo);
+}
+
 std::vector<std::filesystem::path>
 GetDownloadedStatePaths(const std::filesystem::path& SystemRootDir)
 {
@@ -436,4 +332,365 @@ GetDownloadedStatePaths(const std::filesystem::path& SystemRootDir)
 	return Result;
 }
 
+#if ZEN_WITH_TESTS
+
+void
+buildsavedstate_forcelink()
+{
+}
+
+namespace buildsavestate_test {
+	bool Equal(const BuildsSelection& Lhs, const BuildsSelection& Rhs)
+	{
+		if (Lhs.Builds.size() != Rhs.Builds.size())
+		{
+			return false;
+		}
+		for (size_t BuildIndex = 0; BuildIndex < Lhs.Builds.size(); BuildIndex++)
+		{
+			const BuildsSelection::Build LhsBuild = Lhs.Builds[BuildIndex];
+			const BuildsSelection::Build RhsBuild = Rhs.Builds[BuildIndex];
+			if (LhsBuild.Id != RhsBuild.Id)
+			{
+				return false;
+			}
+			if (LhsBuild.Parts.size() != RhsBuild.Parts.size())
+			{
+				return false;
+			}
+			for (size_t PartIndex = 0; PartIndex < LhsBuild.Parts.size(); PartIndex++)
+			{
+				const BuildsSelection::BuildPart LhsPart = LhsBuild.Parts[PartIndex];
+				const BuildsSelection::BuildPart RhsPart = RhsBuild.Parts[PartIndex];
+				if (LhsPart.Id != RhsPart.Id)
+				{
+					return false;
+				}
+				if (LhsPart.Name != RhsPart.Name)
+				{
+					return false;
+				}
+			}
+			if (LhsBuild.IncludeWildcards != RhsBuild.IncludeWildcards)
+			{
+				return false;
+			}
+			if (LhsBuild.ExcludeWildcards != RhsBuild.ExcludeWildcards)
+			{
+				return false;
+			}
+		}
+		return true;
+	}
+	bool Equal(const ChunkedContentData& Lhs, const ChunkedContentData& Rhs)
+	{
+		if (Lhs.SequenceRawHashes != Rhs.SequenceRawHashes)
+		{
+			return false;
+		}
+		if (Lhs.ChunkCounts != Rhs.ChunkCounts)
+		{
+			return false;
+		}
+		if (Lhs.ChunkOrders != Rhs.ChunkOrders)
+		{
+			return false;
+		}
+		if (Lhs.ChunkHashes != Rhs.ChunkHashes)
+		{
+			return false;
+		}
+		if (Lhs.ChunkRawSizes != Rhs.ChunkRawSizes)
+		{
+			return false;
+		}
+		return true;
+	}
+
+	bool Equal(const ChunkedFolderContent& Lhs, const ChunkedFolderContent& Rhs)
+	{
+		if (Lhs.Platform != Rhs.Platform)
+		{
+			return false;
+		}
+		if (Lhs.Paths != Rhs.Paths)
+		{
+			return false;
+		}
+		if (Lhs.RawSizes != Rhs.RawSizes)
+		{
+			return false;
+		}
+		if (Lhs.Attributes != Rhs.Attributes)
+		{
+			return false;
+		}
+		if (Lhs.RawHashes != Rhs.RawHashes)
+		{
+			return false;
+		}
+		return Equal(Lhs.ChunkedContent, Rhs.ChunkedContent);
+	}
+
+	bool Equal(const BuildState& Lhs, const BuildState& Rhs)
+	{
+		if (!Equal(Lhs.Selection, Rhs.Selection))
+		{
+			return false;
+		}
+		if (!Equal(Lhs.ChunkedContent, Rhs.ChunkedContent))
+		{
+			return false;
+		}
+		return true;
+	}
+
+	bool Equal(const FolderContent& Lhs, const FolderContent& Rhs)
+	{
+		if (Lhs.Platform != Rhs.Platform)
+		{
+			return false;
+		}
+		if (Lhs.Paths != Rhs.Paths)
+		{
+			return false;
+		}
+		if (Lhs.RawSizes != Rhs.RawSizes)
+		{
+			return false;
+		}
+		if (Lhs.Attributes != Rhs.Attributes)
+		{
+			return false;
+		}
+		if (Lhs.ModificationTicks != Rhs.ModificationTicks)
+		{
+			return false;
+		}
+		return true;
+	}
+
+	bool Equal(const BuildSaveState& Lhs, const BuildSaveState& Rhs)
+	{
+		if (!Equal(Lhs.State, Rhs.State))
+		{
+			return false;
+		}
+		if (!Equal(Lhs.FolderState, Rhs.FolderState))
+		{
+			return false;
+		}
+		if (Lhs.LocalPath != Rhs.LocalPath)
+		{
+			return false;
+		}
+		return true;
+	}
+
+	bool Equal(const BuildsDownloadInfo& Lhs, const BuildsDownloadInfo& Rhs)
+	{
+		if (!Equal(Lhs.Selection, Rhs.Selection))
+		{
+			return false;
+		}
+		if (Lhs.LocalPath != Rhs.LocalPath)
+		{
+			return false;
+		}
+		if (Lhs.StateFilePath != Rhs.StateFilePath)
+		{
+			return false;
+		}
+		if (Lhs.Iso8601Date != Rhs.Iso8601Date)
+		{
+			return false;
+		}
+		return true;
+	}
+
+	BuildsSelection MakeBuildsSelectionA()
+	{
+		return BuildsSelection{
+			.Builds = std::vector<BuildsSelection::Build>{
+				BuildsSelection::Build{
+					.Id	   = Oid::NewOid(),
+					.Parts = std::vector<BuildsSelection::BuildPart>{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}}},
+				BuildsSelection::Build{
+					.Id = Oid::NewOid(),
+					.Parts =
+						std::vector<BuildsSelection::BuildPart>{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "first_part"},
+																BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "second_part"}},
+					.IncludeWildcards = std::vector<std::string>{std::string{"*.exe"}, std::string{"*.dll"}},
+					.ExcludeWildcards = std::vector<std::string>{std::string{"*.pdb"}}},
+				BuildsSelection::Build{
+					.Id	   = Oid::NewOid(),
+					.Parts = std::vector<BuildsSelection::BuildPart>{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}},
+					.IncludeWildcards = std::vector<std::string>{std::string{"*.ucas"}, std::string{"*.utok"}},
+					.ExcludeWildcards = std::vector<std::string>{std::string{"*.pak"}}}}};
+	}
+
+	BuildsSelection MakeBuildsSelectionB()
+	{
+		return BuildsSelection{
+			.Builds = std::vector<BuildsSelection::Build>{
+				BuildsSelection::Build{
+					.Id	   = Oid::NewOid(),
+					.Parts = std::vector<BuildsSelection::BuildPart>{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}}},
+				BuildsSelection::Build{
+					.Id	   = Oid::NewOid(),
+					.Parts = std::vector<BuildsSelection::BuildPart>{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}},
+					.IncludeWildcards = std::vector<std::string>{std::string{"*.exe"}, std::string{"*.dll"}},
+					.ExcludeWildcards = std::vector<std::string>{std::string{"*.pdb"}, std::string{"*.xSYM"}}}}};
+	}
+
+	BuildState MakeBuildStateA(FastRandom& Random)
+	{
+		BuildsSelection		  Selection = MakeBuildsSelectionA();
+		std::vector<IoBuffer> Chunks;
+
+		ChunkedFolderContent Content = chunkedcontent_testutils::CreateChunkedFolderContent(
+			Random,
+			std::vector<std::pair<const std::string, uint64_t>>{
+				std::pair<std::string, uint64_t>{"file_1", 6u * 1024u},
+				std::pair<std::string, uint64_t>{"file_2.exe", 0},
+				std::pair<std::string, uint64_t>{"file_3.txt", 798},
+				std::pair<std::string, uint64_t>{"dir_1/dir1_file_1.exe", 19u * 1024u},
+				std::pair<std::string, uint64_t>{"dir_1/dir1_file_2.pdb", 7u * 1024u},
+				std::pair<std::string, uint64_t>{"dir_1/dir1_file_3.txt", 93},
+				std::pair<std::string, uint64_t>{"dir_2/dir2_dir1/dir2_dir1_file_1.exe", 31u * 1024u},
+				std::pair<std::string, uint64_t>{"dir_2/dir2_dir1/dir2_dir1_file_2.pdb", 17u * 1024u},
+				std::pair<std::string, uint64_t>{"dir_2/dir2_dir1/dir2_dir1_file_3.dll", 13u * 1024u},
+				std::pair<std::string, uint64_t>{"dir_2/dir2_dir2/dir2_dir2_file_1.txt", 2u * 1024u},
+				std::pair<std::string, uint64_t>{"dir_2/dir2_dir2/dir2_dir2_file_2.json", 3u * 1024u}},
+			4u * 1024u,
+			Chunks);
+		return BuildState{.Selection = std::move(Selection), .ChunkedContent = std::move(Content)};
+	}
+
+	FolderContent MakeFolderContent(ChunkedFolderContent& Content)
+	{
+		FolderContent Result = {.Platform = Content.Platform};
+		Result.Paths		 = Content.Paths;
+		Result.RawSizes		 = Content.RawSizes;
+		Result.Attributes	 = Content.Attributes;
+		Result.ModificationTicks.resize(Result.Paths.size(), 0);
+		uint64_t AttributeIndex = 0;
+		for (uint64_t& ModificationTick : Result.ModificationTicks)
+		{
+			ModificationTick = ++AttributeIndex;
+		}
+		return Result;
+	}
+}  // namespace buildsavestate_test
+
+TEST_CASE("buildsavestate.BuildsSelection")
+{
+	using namespace buildsavestate_test;
+
+	const BuildsSelection Selection = MakeBuildsSelectionA();
+
+	CbObjectWriter Writer;
+	BuildsSelection::Write(Selection, Writer);
+	CbObject SavedObject = Writer.Save();
+
+	BuildsSelection ReadbackSelection = BuildsSelection::Read(SavedObject, nullptr);
+
+	CHECK(Equal(Selection, ReadbackSelection));
+}
+
+TEST_CASE("buildsavestate.BuildsState")
+{
+	using namespace buildsavestate_test;
+
+	FastRandom Random;
+
+	BuildState State = MakeBuildStateA(Random);
+
+	CbObjectWriter Writer;
+	BuildState::Write(State, Writer);
+	CbObject SavedObject = Writer.Save();
+
+	BuildState ReadbackState = BuildState::Read(SavedObject);
+
+	CHECK(Equal(State, ReadbackState));
+}
+
+TEST_CASE("buildsavestate.BuildsSaveState")
+{
+	using namespace buildsavestate_test;
+
+	FastRandom Random;
+
+	BuildState	   State	   = MakeBuildStateA(Random);
+	FolderContent  FolderState = MakeFolderContent(State.ChunkedContent);
+	BuildSaveState SaveState   = {.State	   = std::move(State),
+								  .FolderState = std::move(FolderState),
+								  .LocalPath   = "\\\\?\\E:\\temp\\localpath"};
+
+	CbObjectWriter Writer;
+	BuildSaveState::Write(SaveState, Writer);
+	CbObject SavedObject = Writer.Save();
+
+	BuildSaveState ReadbackSavedState = BuildSaveState::Read(SavedObject);
+
+	CHECK(Equal(SaveState, ReadbackSavedState));
+}
+
+TEST_CASE("buildsavestate.BuildsDownloadInfo")
+{
+	using namespace buildsavestate_test;
+
+	BuildsDownloadInfo DownloadInfo = {.Selection	  = MakeBuildsSelectionA(),
+									   .LocalPath	  = "\\\\?\\E:\\temp\\localpath",
+									   .StateFilePath = "\\\\?\\E:\\temp\\localpath\\.zen\\state.cbo",
+									   .Iso8601Date	  = DateTime::Now().ToIso8601()};
+
+	CbObjectWriter Writer;
+	BuildsDownloadInfo::Write(DownloadInfo, Writer);
+	CbObject SavedObject = Writer.Save();
+
+	BuildsDownloadInfo ReadbackDownloadInfo = BuildsDownloadInfo::Read(SavedObject);
+
+	CHECK(Equal(DownloadInfo, ReadbackDownloadInfo));
+}
+
+TEST_CASE("buildsavestate.DownloadedPaths")
+{
+	using namespace buildsavestate_test;
+
+	ScopedTemporaryDirectory SystemDir;
+
+	BuildsDownloadInfo DownloadInfoA = {.Selection	   = MakeBuildsSelectionA(),
+										.LocalPath	   = "\\\\?\\E:\\temp\\localpathA",
+										.StateFilePath = "\\\\?\\E:\\temp\\localpathA\\.zen\\state.cbo",
+										.Iso8601Date   = DateTime(DateTime::Now().GetTicks() - 200).ToIso8601()};
+
+	BuildsDownloadInfo DownloadInfoB = {.Selection	   = MakeBuildsSelectionB(),
+										.LocalPath	   = "\\\\?\\E:\\temp\\localpathB",
+										.StateFilePath = "\\\\?\\E:\\temp\\localpathB\\.zen\\state.cbo",
+										.Iso8601Date   = DateTime(DateTime::Now().GetTicks() - 100).ToIso8601()};
+
+	AddDownloadedPath(SystemDir.Path(), DownloadInfoA);
+	AddDownloadedPath(SystemDir.Path(), DownloadInfoB);
+
+	std::vector<std::filesystem::path> DownloadedPaths = GetDownloadedStatePaths(SystemDir.Path());
+
+	CHECK_EQ(2u, DownloadedPaths.size());
+	BuildsDownloadInfo ReadBackDownloadInfo0 = ReadDownloadedInfoFile(DownloadedPaths[0]);
+	BuildsDownloadInfo ReadBackDownloadInfo1 = ReadDownloadedInfoFile(DownloadedPaths[1]);
+
+	if (DownloadInfoA.LocalPath == ReadBackDownloadInfo0.LocalPath)
+	{
+		CHECK(Equal(DownloadInfoA, ReadBackDownloadInfo0));
+		CHECK(Equal(DownloadInfoB, ReadBackDownloadInfo1));
+	}
+	else
+	{
+		CHECK(Equal(DownloadInfoA, ReadBackDownloadInfo1));
+		CHECK(Equal(DownloadInfoB, ReadBackDownloadInfo0));
+	}
+}
+
+#endif	// ZEN_WITH_TESTS
+
 }  // namespace zen