move builds state functions to buildsavedstate.h/cpp (#577)

1 files changed, 33 insertions, 183 deletions

diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index ca927e643..469d29e0d 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -2,6 +2,7 @@
 
 #include <zenremotestore/builds/buildstorageoperations.h>
 
+#include <zenremotestore/builds/buildsavedstate.h>
 #include <zenremotestore/builds/buildstorage.h>
 #include <zenremotestore/builds/buildstoragecache.h>
 #include <zenremotestore/chunking/chunkblock.h>
@@ -343,84 +344,6 @@ namespace {
 
 }  // namespace
 
-bool
-ReadStateObject(BuildOpLogOutput&				   LogOutput,
-				CbObjectView					   StateView,
-				Oid&							   OutBuildId,
-				std::vector<Oid>&				   BuildPartsIds,
-				std::vector<std::string>&		   BuildPartsNames,
-				std::vector<ChunkedFolderContent>& OutPartContents,
-				FolderContent&					   OutLocalFolderState)
-{
-	try
-	{
-		CbObjectView BuildView = StateView["builds"sv].AsArrayView().CreateViewIterator().AsObjectView();
-		OutBuildId			   = BuildView["buildId"sv].AsObjectId();
-		for (CbFieldView PartView : BuildView["parts"sv].AsArrayView())
-		{
-			CbObjectView PartObjectView = PartView.AsObjectView();
-			BuildPartsIds.push_back(PartObjectView["partId"sv].AsObjectId());
-			BuildPartsNames.push_back(std::string(PartObjectView["partName"sv].AsString()));
-			OutPartContents.push_back(LoadChunkedFolderContentToCompactBinary(PartObjectView["content"sv].AsObjectView()));
-		}
-		OutLocalFolderState = LoadFolderContentToCompactBinary(StateView["localFolderState"sv].AsObjectView());
-		return true;
-	}
-	catch (const std::exception& Ex)
-	{
-		LOG_OUTPUT_WARN(LogOutput, "Unable to read local state: ", Ex.what());
-		return false;
-	}
-}
-
-bool
-ReadStateFile(BuildOpLogOutput&			   LogOutput,
-			  const std::filesystem::path& StateFilePath,
-			  FolderContent&			   OutLocalFolderState,
-			  ChunkedFolderContent&		   OutLocalContent)
-{
-	ZEN_TRACE_CPU("ReadStateFile");
-	bool HasLocalState = false;
-	try
-	{
-		CbObject CurrentStateObject = LoadCompactBinaryObject(StateFilePath).Object;
-		if (CurrentStateObject)
-		{
-			Oid								  CurrentBuildId;
-			std::vector<Oid>				  SavedBuildPartIds;
-			std::vector<std::string>		  SavedBuildPartsNames;
-			std::vector<ChunkedFolderContent> SavedPartContents;
-			if (ReadStateObject(LogOutput,
-								CurrentStateObject,
-								CurrentBuildId,
-								SavedBuildPartIds,
-								SavedBuildPartsNames,
-								SavedPartContents,
-								OutLocalFolderState))
-			{
-				if (!SavedPartContents.empty())
-				{
-					if (SavedPartContents.size() == 1)
-					{
-						OutLocalContent = std::move(SavedPartContents[0]);
-					}
-					else
-					{
-						OutLocalContent = MergeChunkedFolderContents(SavedPartContents[0],
-																	 std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1));
-					}
-					HasLocalState = true;
-				}
-			}
-		}
-	}
-	catch (const std::exception& Ex)
-	{
-		LOG_OUTPUT_WARN(LogOutput, "Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
-	}
-	return HasLocalState;
-}
-
 struct BlockRangeDescriptor
 {
 	uint32_t BlockIndex			  = (uint32_t)-1;
@@ -3405,49 +3328,46 @@ BuildsOperationUpdateFolder::FindScavengeSources()
 {
 	ZEN_TRACE_CPU("FindScavengeSources");
 
+	std::vector<std::filesystem::path> StatePaths = GetDownloadedStatePaths(m_Options.SystemRootDir);
+
 	std::vector<ScavengeSource> Result;
-	DirectoryContent			Content;
-	GetDirectoryContent(m_Options.SystemRootDir / "builds" / "downloads", DirectoryContentFlags::IncludeFiles, Content);
-	for (const std::filesystem::path& EntryPath : Content.Files)
-	{
-		bool   DeleteEntry = false;
-		IoHash EntryPathHash;
-		if (IoHash::TryParse(EntryPath.stem().string(), EntryPathHash))
-		{
-			// Read state and verify that it is valid
-			IoBuffer		 MetaDataJson = ReadFile(EntryPath).Flatten();
-			std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
-			std::string		 JsonError;
-			CbObject		 DownloadInfo = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
-			if (JsonError.empty())
+	for (const std::filesystem::path& EntryPath : StatePaths)
+	{
+		bool DeleteEntry = false;
+
+		// Read state and verify that it is valid
+		IoBuffer		 MetaDataJson = ReadFile(EntryPath).Flatten();
+		std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+		std::string		 JsonError;
+		CbObject		 DownloadInfo = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+		if (JsonError.empty())
+		{
+			std::filesystem::path StateFilePath = DownloadInfo["statePath"].AsU8String();
+			if (IsFile(StateFilePath))
 			{
-				std::filesystem::path StateFilePath = DownloadInfo["statePath"].AsU8String();
-				if (IsFile(StateFilePath))
+				std::filesystem::path Path = DownloadInfo["path"].AsU8String();
+				if (!std::filesystem::equivalent(Path, m_Path))
 				{
-					std::filesystem::path Path = DownloadInfo["path"].AsU8String();
-					if (!std::filesystem::equivalent(Path, m_Path))
+					if (IsDir(Path))
 					{
-						if (IsDir(Path))
-						{
-							Result.push_back({.StateFilePath = std::move(StateFilePath), .Path = std::move(Path)});
-						}
-						else
-						{
-							DeleteEntry = true;
-						}
+						Result.push_back({.StateFilePath = std::move(StateFilePath), .Path = std::move(Path)});
+					}
+					else
+					{
+						DeleteEntry = true;
 					}
-				}
-				else
-				{
-					DeleteEntry = true;
 				}
 			}
 			else
 			{
-				LOG_OUTPUT_WARN(m_LogOutput, "Invalid download state file at {}. '{}'", EntryPath, JsonError);
 				DeleteEntry = true;
 			}
 		}
+		else
+		{
+			LOG_OUTPUT_WARN(m_LogOutput, "Invalid download state file at {}. '{}'", EntryPath, JsonError);
+			DeleteEntry = true;
+		}
 
 		if (DeleteEntry)
 		{
@@ -3525,12 +3445,13 @@ BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source,
 	ZEN_TRACE_CPU("FindScavengeContent");
 
 	FolderContent LocalFolderState;
-	if (!ReadStateFile(m_LogOutput, Source.StateFilePath, LocalFolderState, OutScavengedLocalContent))
+	try
 	{
-		return false;
+		ReadStateFile(Source.StateFilePath, LocalFolderState, OutScavengedLocalContent);
 	}
-	if (!IsDir(Source.Path))
+	catch (const std::exception& Ex)
 	{
+		ZEN_CONSOLE_DEBUG("Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what());
 		return false;
 	}
 
@@ -6066,77 +5987,6 @@ BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders(
 }
 
 void
-BuildsOperationUploadFolder::CalculateLocalChunkOrders(const std::span<const uint32_t>&				 AbsoluteChunkOrders,
-													   const std::span<const IoHash>				 LooseChunkHashes,
-													   const std::span<const uint64_t>				 LooseChunkRawSizes,
-													   const std::span<const ChunkBlockDescription>& BlockDescriptions,
-													   std::vector<IoHash>&							 OutLocalChunkHashes,
-													   std::vector<uint64_t>&						 OutLocalChunkRawSizes,
-													   std::vector<uint32_t>&						 OutLocalChunkOrders)
-{
-	ZEN_TRACE_CPU("CalculateLocalChunkOrders");
-
-	std::vector<IoHash>	  AbsoluteChunkHashes;
-	std::vector<uint64_t> AbsoluteChunkRawSizes;
-	AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), LooseChunkHashes.begin(), LooseChunkHashes.end());
-	AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), LooseChunkRawSizes.begin(), LooseChunkRawSizes.end());
-	for (const ChunkBlockDescription& Block : BlockDescriptions)
-	{
-		AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), Block.ChunkRawHashes.begin(), Block.ChunkRawHashes.end());
-		AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), Block.ChunkRawLengths.begin(), Block.ChunkRawLengths.end());
-	}
-	OutLocalChunkHashes.reserve(AbsoluteChunkHashes.size());
-	OutLocalChunkRawSizes.reserve(AbsoluteChunkRawSizes.size());
-	OutLocalChunkOrders.reserve(AbsoluteChunkOrders.size());
-
-	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
-	ChunkHashToChunkIndex.reserve(AbsoluteChunkHashes.size());
-
-	for (uint32_t AbsoluteChunkOrderIndex = 0; AbsoluteChunkOrderIndex < AbsoluteChunkOrders.size(); AbsoluteChunkOrderIndex++)
-	{
-		const uint32_t AbsoluteChunkIndex	= AbsoluteChunkOrders[AbsoluteChunkOrderIndex];
-		const IoHash&  AbsoluteChunkHash	= AbsoluteChunkHashes[AbsoluteChunkIndex];
-		const uint64_t AbsoluteChunkRawSize = AbsoluteChunkRawSizes[AbsoluteChunkIndex];
-
-		if (auto It = ChunkHashToChunkIndex.find(AbsoluteChunkHash); It != ChunkHashToChunkIndex.end())
-		{
-			const uint32_t LocalChunkIndex = It->second;
-			OutLocalChunkOrders.push_back(LocalChunkIndex);
-		}
-		else
-		{
-			uint32_t LocalChunkIndex = gsl::narrow<uint32_t>(OutLocalChunkHashes.size());
-			OutLocalChunkHashes.push_back(AbsoluteChunkHash);
-			OutLocalChunkRawSizes.push_back(AbsoluteChunkRawSize);
-			OutLocalChunkOrders.push_back(LocalChunkIndex);
-			ChunkHashToChunkIndex.insert_or_assign(AbsoluteChunkHash, LocalChunkIndex);
-		}
-#if EXTRA_VERIFY
-		const uint32_t	LocalChunkIndex	  = OutLocalChunkOrders[AbsoluteChunkOrderIndex];
-		const IoHash&	LocalChunkHash	  = OutLocalChunkHashes[LocalChunkIndex];
-		const uint64_t& LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex];
-		ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
-		ZEN_ASSERT(LocalChunkRawSize == AbsoluteChunkRawSize);
-#endif	// EXTRA_VERIFY
-	}
-#if EXTRA_VERIFY
-	for (uint32_t OrderIndex = 0; OrderIndex < OutLocalChunkOrders.size(); OrderIndex++)
-	{
-		uint32_t	 LocalChunkIndex   = OutLocalChunkOrders[OrderIndex];
-		const IoHash LocalChunkHash	   = OutLocalChunkHashes[LocalChunkIndex];
-		uint64_t	 LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex];
-
-		uint32_t	 VerifyChunkIndex	= AbsoluteChunkOrders[OrderIndex];
-		const IoHash VerifyChunkHash	= AbsoluteChunkHashes[VerifyChunkIndex];
-		uint64_t	 VerifyChunkRawSize = AbsoluteChunkRawSizes[VerifyChunkIndex];
-
-		ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
-		ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
-	}
-#endif	// EXTRA_VERIFY
-}
-
-void
 BuildsOperationUploadFolder::WriteBuildContentToCompactBinary(CbObjectWriter&						 PartManifestWriter,
 															  const SourcePlatform					 Platform,
 															  std::span<const std::filesystem::path> Paths,