update state when wildcard (#657)

* add --append option and improve state handling when using downloads for `zen builds download`

1 files changed, 95 insertions, 166 deletions

diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 798ef4dae..92017e98d 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -2,6 +2,7 @@
 
 #include <zenremotestore/builds/buildstorageoperations.h>
 
+#include <zenremotestore/builds/buildcontent.h>
 #include <zenremotestore/builds/buildsavedstate.h>
 #include <zenremotestore/builds/buildstorage.h>
 #include <zenremotestore/builds/buildstoragecache.h>
@@ -30,8 +31,6 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_set.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
-#define EXTRA_VERIFY 0
-
 namespace zen {
 
 using namespace std::literals;
@@ -1104,6 +1103,14 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 								NeededBlockChunkIndexes.push_back(ChunkBlockIndex);
 							}
 						}
+						else
+						{
+							ZEN_ASSERT(!RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]);
+						}
+					}
+					else
+					{
+						ZEN_DEBUG("Chunk {} not found in block {}", ChunkHash, BlockDescription.BlockHash);
 					}
 				}
 				return NeededBlockChunkIndexes;
@@ -1973,9 +1980,6 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 
 			if (!m_Options.PrimeCacheOnly)
 			{
-				ZEN_ASSERT(m_WrittenChunkByteCount == BytesToWrite);
-				ZEN_ASSERT(m_ValidatedChunkByteCount == BytesToValidate);
-
 				uint32_t RawSequencesMissingWriteCount = 0;
 				for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++)
 				{
@@ -1999,6 +2003,8 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 					}
 				}
 				ZEN_ASSERT(RawSequencesMissingWriteCount == 0);
+				ZEN_ASSERT(m_WrittenChunkByteCount == BytesToWrite);
+				ZEN_ASSERT(m_ValidatedChunkByteCount == BytesToValidate);
 			}
 
 			const uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() +
@@ -2702,46 +2708,43 @@ BuildsOperationUpdateFolder::FindScavengeSources()
 	std::vector<ScavengeSource> Result;
 	for (const std::filesystem::path& EntryPath : StatePaths)
 	{
-		bool DeleteEntry = false;
-
-		// Read state and verify that it is valid
-		IoBuffer		 MetaDataJson = ReadFile(EntryPath).Flatten();
-		std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
-		std::string		 JsonError;
-		CbObject		 DownloadInfo = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
-		if (JsonError.empty())
+		if (IsFile(EntryPath))
 		{
-			std::filesystem::path StateFilePath = DownloadInfo["statePath"].AsU8String();
-			if (IsFile(StateFilePath))
+			bool DeleteEntry = false;
+
+			try
 			{
-				std::filesystem::path Path = DownloadInfo["path"].AsU8String();
-				if (!std::filesystem::equivalent(Path, m_Path))
+				BuildsDownloadInfo Info = ReadDownloadedInfoFile(EntryPath);
+				if (!Info.LocalPath.empty())
 				{
-					if (IsDir(Path))
-					{
-						Result.push_back({.StateFilePath = std::move(StateFilePath), .Path = std::move(Path)});
-					}
-					else
+					if (!std::filesystem::equivalent(Info.LocalPath, m_Path))
 					{
-						DeleteEntry = true;
+						if (IsDir(Info.LocalPath) && IsFile(Info.StateFilePath))
+						{
+							Result.push_back({.StateFilePath = std::move(Info.StateFilePath), .Path = std::move(Info.LocalPath)});
+						}
+						else
+						{
+							DeleteEntry = true;
+						}
 					}
 				}
+				else
+				{
+					DeleteEntry = true;
+				}
 			}
-			else
+			catch (const std::exception& Ex)
 			{
+				ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", Ex.what());
 				DeleteEntry = true;
 			}
-		}
-		else
-		{
-			ZEN_OPERATION_LOG_WARN(m_LogOutput, "Invalid download state file at {}. '{}'", EntryPath, JsonError);
-			DeleteEntry = true;
-		}
 
-		if (DeleteEntry)
-		{
-			std::error_code DummyEc;
-			std::filesystem::remove(EntryPath, DummyEc);
+			if (DeleteEntry)
+			{
+				std::error_code DummyEc;
+				std::filesystem::remove(EntryPath, DummyEc);
+			}
 		}
 	}
 	return Result;
@@ -2828,7 +2831,9 @@ BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source,
 	FolderContent LocalFolderState;
 	try
 	{
-		ReadStateFile(Source.StateFilePath, LocalFolderState, OutScavengedLocalContent);
+		BuildSaveState SavedState = ReadBuildSaveStateFile(Source.StateFilePath);
+		OutScavengedLocalContent  = std::move(SavedState.State.ChunkedContent);
+		LocalFolderState		  = std::move(SavedState.FolderState);
 	}
 	catch (const std::exception& Ex)
 	{
@@ -5127,51 +5132,53 @@ BuildsOperationUploadFolder::Execute()
 			{
 				AllChunkBlockHashes.push_back(BlockDescription.BlockHash);
 			}
-#if EXTRA_VERIFY
-			tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex;
-			std::vector<IoHash>							   AbsoluteChunkHashes;
-			AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size());
-			for (uint32_t ChunkIndex : LooseChunkIndexes)
+			std::vector<IoHash> AbsoluteChunkHashes;
+			if (m_Options.DoExtraContentValidation)
 			{
-				ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()});
-				AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
-			}
-			for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions)
-			{
-				for (const IoHash& ChunkHash : Block.ChunkRawHashes)
+				tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex;
+				AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size());
+				for (uint32_t ChunkIndex : LooseChunkIndexes)
 				{
-					ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()});
-					AbsoluteChunkHashes.push_back(ChunkHash);
+					ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()});
+					AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+				}
+				for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions)
+				{
+					for (const IoHash& ChunkHash : Block.ChunkRawHashes)
+					{
+						ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()});
+						AbsoluteChunkHashes.push_back(ChunkHash);
+					}
+				}
+				for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes)
+				{
+					ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash);
+					ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash);
+				}
+				for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders)
+				{
+					ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] ==
+							   LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+					ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex);
 				}
 			}
-			for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes)
-			{
-				ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash);
-				ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash);
-			}
-			for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders)
-			{
-				ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] ==
-						   LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
-				ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex);
-			}
-#endif	// EXTRA_VERIFY
 			std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes,
 																					 LocalContent.ChunkedContent.ChunkOrders,
 																					 LocalLookup.ChunkHashToChunkIndex,
 																					 LooseChunkIndexes,
 																					 AllChunkBlockDescriptions);
 
-#if EXTRA_VERIFY
-			for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++)
+			if (m_Options.DoExtraContentValidation)
 			{
-				uint32_t	  LocalChunkIndex	 = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex];
-				uint32_t	  AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex];
-				const IoHash& LocalChunkHash	 = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
-				const IoHash& AbsoluteChunkHash	 = AbsoluteChunkHashes[AbsoluteChunkIndex];
-				ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
+				for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++)
+				{
+					uint32_t	  LocalChunkIndex	 = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex];
+					uint32_t	  AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex];
+					const IoHash& LocalChunkHash	 = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+					const IoHash& AbsoluteChunkHash	 = AbsoluteChunkHashes[AbsoluteChunkIndex];
+					ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
+				}
 			}
-#endif	// EXTRA_VERIFY
 
 			WriteBuildContentToCompactBinary(PartManifestWriter,
 											 LocalContent.Platform,
@@ -5187,7 +5194,7 @@ BuildsOperationUploadFolder::Execute()
 											 LooseChunkIndexes,
 											 AllChunkBlockHashes);
 
-#if EXTRA_VERIFY
+			if (m_Options.DoExtraContentValidation)
 			{
 				ChunkedFolderContent VerifyFolderContent;
 
@@ -5226,7 +5233,8 @@ BuildsOperationUploadFolder::Execute()
 										  AllChunkBlockDescriptions,
 										  VerifyFolderContent.ChunkedContent.ChunkHashes,
 										  VerifyFolderContent.ChunkedContent.ChunkRawSizes,
-										  VerifyFolderContent.ChunkedContent.ChunkOrders);
+										  VerifyFolderContent.ChunkedContent.ChunkOrders,
+										  m_Options.DoExtraContentValidation);
 
 				ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths);
 				ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes);
@@ -5249,7 +5257,6 @@ BuildsOperationUploadFolder::Execute()
 					ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
 				}
 			}
-#endif	// EXTRA_VERIFY
 			PartManifest = PartManifestWriter.Save();
 		}
 
@@ -6097,19 +6104,21 @@ BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders(
 {
 	ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders");
 
-#if EXTRA_VERIFY
 	std::vector<IoHash> TmpAbsoluteChunkHashes;
-	TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size());
-#endif	// EXTRA_VERIFY
+	if (m_Options.DoExtraContentValidation)
+	{
+		TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size());
+	}
 	std::vector<uint32_t> LocalChunkIndexToAbsoluteChunkIndex;
 	LocalChunkIndexToAbsoluteChunkIndex.resize(LocalChunkHashes.size(), (uint32_t)-1);
 	std::uint32_t AbsoluteChunkCount = 0;
 	for (uint32_t ChunkIndex : LooseChunkIndexes)
 	{
 		LocalChunkIndexToAbsoluteChunkIndex[ChunkIndex] = AbsoluteChunkCount;
-#if EXTRA_VERIFY
-		TmpAbsoluteChunkHashes.push_back(LocalChunkHashes[ChunkIndex]);
-#endif	// EXTRA_VERIFY
+		if (m_Options.DoExtraContentValidation)
+		{
+			TmpAbsoluteChunkHashes.push_back(LocalChunkHashes[ChunkIndex]);
+		}
 		AbsoluteChunkCount++;
 	}
 	for (const ChunkBlockDescription& Block : BlockDescriptions)
@@ -6122,9 +6131,10 @@ BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders(
 				ZEN_ASSERT_SLOW(LocalChunkHashes[LocalChunkIndex] == ChunkHash);
 				LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex] = AbsoluteChunkCount;
 			}
-#if EXTRA_VERIFY
-			TmpAbsoluteChunkHashes.push_back(ChunkHash);
-#endif	// EXTRA_VERIFY
+			if (m_Options.DoExtraContentValidation)
+			{
+				TmpAbsoluteChunkHashes.push_back(ChunkHash);
+			}
 			AbsoluteChunkCount++;
 		}
 	}
@@ -6133,12 +6143,13 @@ BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders(
 	for (const uint32_t LocalChunkIndex : LocalChunkOrder)
 	{
 		const uint32_t AbsoluteChunkIndex = LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex];
-#if EXTRA_VERIFY
-		ZEN_ASSERT(LocalChunkHashes[LocalChunkIndex] == TmpAbsoluteChunkHashes[AbsoluteChunkIndex]);
-#endif	// EXTRA_VERIFY
+		if (m_Options.DoExtraContentValidation)
+		{
+			ZEN_ASSERT(LocalChunkHashes[LocalChunkIndex] == TmpAbsoluteChunkHashes[AbsoluteChunkIndex]);
+		}
 		AbsoluteChunkOrder.push_back(AbsoluteChunkIndex);
 	}
-#if EXTRA_VERIFY
+	if (m_Options.DoExtraContentValidation)
 	{
 		uint32_t OrderIndex = 0;
 		while (OrderIndex < LocalChunkOrder.size())
@@ -6151,91 +6162,9 @@ BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders(
 			OrderIndex++;
 		}
 	}
-#endif	// EXTRA_VERIFY
 	return AbsoluteChunkOrder;
 }
 
-void
-BuildsOperationUploadFolder::WriteBuildContentToCompactBinary(CbObjectWriter&						 PartManifestWriter,
-															  const SourcePlatform					 Platform,
-															  std::span<const std::filesystem::path> Paths,
-															  std::span<const IoHash>				 RawHashes,
-															  std::span<const uint64_t>				 RawSizes,
-															  std::span<const uint32_t>				 Attributes,
-															  std::span<const IoHash>				 SequenceRawHashes,
-															  std::span<const uint32_t>				 ChunkCounts,
-															  std::span<const IoHash>				 LocalChunkHashes,
-															  std::span<const uint64_t>				 LocalChunkRawSizes,
-															  const std::vector<uint32_t>&			 AbsoluteChunkOrders,
-															  const std::span<const uint32_t>		 LooseLocalChunkIndexes,
-															  const std::span<IoHash>				 BlockHashes)
-{
-	ZEN_ASSERT(Platform != SourcePlatform::_Count);
-	PartManifestWriter.AddString("platform"sv, ToString(Platform));
-
-	uint64_t TotalSize = 0;
-	for (const uint64_t Size : RawSizes)
-	{
-		TotalSize += Size;
-	}
-	PartManifestWriter.AddInteger("totalSize", TotalSize);
-
-	PartManifestWriter.BeginObject("files"sv);
-	{
-		compactbinary_helpers::WriteArray(Paths, "paths"sv, PartManifestWriter);
-		compactbinary_helpers::WriteArray(RawHashes, "rawhashes"sv, PartManifestWriter);
-		compactbinary_helpers::WriteArray(RawSizes, "rawsizes"sv, PartManifestWriter);
-		if (Platform == SourcePlatform::Windows)
-		{
-			compactbinary_helpers::WriteArray(Attributes, "attributes"sv, PartManifestWriter);
-		}
-		if (Platform == SourcePlatform::Linux || Platform == SourcePlatform::MacOS)
-		{
-			compactbinary_helpers::WriteArray(Attributes, "mode"sv, PartManifestWriter);
-		}
-	}
-	PartManifestWriter.EndObject();	 // files
-
-	PartManifestWriter.BeginObject("chunkedContent");
-	{
-		compactbinary_helpers::WriteArray(SequenceRawHashes, "sequenceRawHashes"sv, PartManifestWriter);
-		compactbinary_helpers::WriteArray(ChunkCounts, "chunkcounts"sv, PartManifestWriter);
-		compactbinary_helpers::WriteArray(AbsoluteChunkOrders, "chunkorders"sv, PartManifestWriter);
-	}
-	PartManifestWriter.EndObject();	 // chunkedContent
-
-	size_t LooseChunkCount = LooseLocalChunkIndexes.size();
-	if (LooseChunkCount > 0)
-	{
-		PartManifestWriter.BeginObject("chunkAttachments");
-		{
-			PartManifestWriter.BeginArray("rawHashes"sv);
-			for (uint32_t ChunkIndex : LooseLocalChunkIndexes)
-			{
-				PartManifestWriter.AddBinaryAttachment(LocalChunkHashes[ChunkIndex]);
-			}
-			PartManifestWriter.EndArray();	// rawHashes
-
-			PartManifestWriter.BeginArray("chunkRawSizes"sv);
-			for (uint32_t ChunkIndex : LooseLocalChunkIndexes)
-			{
-				PartManifestWriter.AddInteger(LocalChunkRawSizes[ChunkIndex]);
-			}
-			PartManifestWriter.EndArray();	// chunkSizes
-		}
-		PartManifestWriter.EndObject();	 //
-	}
-
-	if (BlockHashes.size() > 0)
-	{
-		PartManifestWriter.BeginObject("blockAttachments");
-		{
-			compactbinary_helpers::WriteBinaryAttachmentArray(BlockHashes, "rawHashes"sv, PartManifestWriter);
-		}
-		PartManifestWriter.EndObject();	 // blocks
-	}
-}
-
 CompositeBuffer
 BuildsOperationUploadFolder::FetchChunk(const ChunkedFolderContent& Content,
 										const ChunkedContentLookup& Lookup,