save global download info file for scavenging (#349)

* save global download info file for scavenging
* don't let test code write to official state folder

1 files changed, 189 insertions, 105 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 2f4a2921e..ebcbb59b7 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -1292,10 +1292,10 @@ namespace {
 		}
 	}
 
-	CbObject CreateStateObject(const Oid&								BuildId,
-							   std::vector<std::pair<Oid, std::string>> AllBuildParts,
-							   std::span<const ChunkedFolderContent>	PartContents,
-							   const FolderContent&						LocalFolderState)
+	CbObject CreateStateObject(const Oid&									   BuildId,
+							   const std::vector<std::pair<Oid, std::string>>& AllBuildParts,
+							   std::span<const ChunkedFolderContent>		   PartContents,
+							   const FolderContent&							   LocalFolderState)
 	{
 		CbObjectWriter CurrentStateWriter;
 		CurrentStateWriter.BeginArray("builds"sv);
@@ -1334,6 +1334,51 @@ namespace {
 		return CurrentStateWriter.Save();
 	}
 
+	void AddDownloadedPath(const std::filesystem::path&					   SystemRootDir,
+						   const Oid&									   BuildId,
+						   const std::vector<std::pair<Oid, std::string>>& BuildParts,
+						   const std::filesystem::path&					   StateFilePath,
+						   const std::filesystem::path&					   LocalPath)
+	{
+		ZEN_ASSERT(!SystemRootDir.empty());
+		ZEN_ASSERT(!StateFilePath.empty());
+		ZEN_ASSERT(!LocalPath.empty());
+		const std::u8string	  LocalPathString = LocalPath.generic_u8string();
+		IoHash				  PathHash		  = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length());
+		std::filesystem::path WritePath		  = SystemRootDir / "builds" / "downloads" / (PathHash.ToHexString() + ".json");
+		CreateDirectories(WritePath.parent_path());
+		CbObjectWriter Writer;
+		Writer.AddString("path", (const char*)LocalPath.u8string().c_str());
+		Writer.AddString("statePath", (const char*)StateFilePath.u8string().c_str());
+		Writer.AddDateTime("date", DateTime::Now());
+		Writer.BeginArray("builds"sv);
+		{
+			Writer.BeginObject();
+			{
+				Writer.AddObjectId("buildId", BuildId);
+				Writer.BeginArray("parts");
+				for (const auto& It : BuildParts)
+				{
+					Writer.BeginObject();
+					{
+						Writer.AddObjectId("partId", It.first);
+						Writer.AddString("partName", It.second);
+					}
+					Writer.EndObject();
+				}
+				Writer.EndArray();	// parts
+			}
+			Writer.EndObject();
+		}
+		Writer.EndArray();	// builds
+
+		CbObject					 Payload = Writer.Save();
+		ExtendableStringBuilder<512> SB;
+		CompactBinaryToJson(Payload.GetView(), SB);
+		MemoryView JsonPayload(SB.Data(), SB.Size());
+		TemporaryFile::SafeWriteFile(WritePath, JsonPayload);
+	}
+
 	class BufferedOpenFile
 	{
 	public:
@@ -7882,6 +7927,7 @@ namespace {
 						std::span<const std::string> BuildPartNames,
 						const std::filesystem::path& Path,
 						const std::filesystem::path& ZenFolderPath,
+						const std::filesystem::path	 SystemRootDir,
 						bool						 AllowMultiparts,
 						bool						 AllowPartialBlockRequests,
 						bool						 WipeTargetFolder,
@@ -8004,6 +8050,8 @@ namespace {
 			CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
 			TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
 			ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+
+			AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
 		}
 		else
 		{
@@ -8055,6 +8103,8 @@ namespace {
 					TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
 					ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
 
+					AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
+
 #if 0
 					ExtendableStringBuilder<1024> SB;
 					CompactBinaryToJson(StateObject, SB);
@@ -8285,9 +8335,11 @@ BuildsCommand::BuildsCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
 
-	auto AddAuthOptions = [this](cxxopts::Options& Ops) {
+	auto AddSystemOptions = [this](cxxopts::Options& Ops) {
 		Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value<std::string>(m_SystemRootDir), "<systemdir>");
+	};
 
+	auto AddAuthOptions = [this](cxxopts::Options& Ops) {
 		// Direct access token (may expire)
 		Ops.add_option("auth-token",
 					   "",
@@ -8428,6 +8480,7 @@ BuildsCommand::BuildsCommand()
 	m_Options.positional_help("verb");
 
 	// list
+	AddSystemOptions(m_ListOptions);
 	AddCloudOptions(m_ListOptions);
 	AddFileOptions(m_ListOptions);
 	AddOutputOptions(m_ListOptions);
@@ -8449,6 +8502,7 @@ BuildsCommand::BuildsCommand()
 	m_ListOptions.positional_help("query-path result-path");
 
 	// upload
+	AddSystemOptions(m_UploadOptions);
 	AddCloudOptions(m_UploadOptions);
 	AddFileOptions(m_UploadOptions);
 	AddOutputOptions(m_UploadOptions);
@@ -8521,6 +8575,7 @@ BuildsCommand::BuildsCommand()
 	m_UploadOptions.positional_help("local-path build-id");
 
 	// download
+	AddSystemOptions(m_DownloadOptions);
 	AddCloudOptions(m_DownloadOptions);
 	AddFileOptions(m_DownloadOptions);
 	AddOutputOptions(m_DownloadOptions);
@@ -8585,6 +8640,7 @@ BuildsCommand::BuildsCommand()
 	m_DiffOptions.parse_positional({"local-path", "compare-path"});
 	m_DiffOptions.positional_help("local-path compare-path");
 
+	AddSystemOptions(m_TestOptions);
 	AddCloudOptions(m_TestOptions);
 	AddFileOptions(m_TestOptions);
 	AddOutputOptions(m_TestOptions);
@@ -8607,6 +8663,7 @@ BuildsCommand::BuildsCommand()
 	m_TestOptions.parse_positional({"local-path"});
 	m_TestOptions.positional_help("local-path");
 
+	AddSystemOptions(m_FetchBlobOptions);
 	AddCloudOptions(m_FetchBlobOptions);
 	AddFileOptions(m_FetchBlobOptions);
 	AddOutputOptions(m_FetchBlobOptions);
@@ -8618,6 +8675,7 @@ BuildsCommand::BuildsCommand()
 	m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"});
 	m_FetchBlobOptions.positional_help("build-id blob-hash");
 
+	AddSystemOptions(m_ValidateBuildPartOptions);
 	AddCloudOptions(m_ValidateBuildPartOptions);
 	AddFileOptions(m_ValidateBuildPartOptions);
 	AddOutputOptions(m_ValidateBuildPartOptions);
@@ -8640,6 +8698,7 @@ BuildsCommand::BuildsCommand()
 	m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"});
 	m_ValidateBuildPartOptions.positional_help("build-id build-part-id");
 
+	AddSystemOptions(m_MultiTestDownloadOptions);
 	AddCloudOptions(m_MultiTestDownloadOptions);
 	AddFileOptions(m_MultiTestDownloadOptions);
 	AddOutputOptions(m_MultiTestDownloadOptions);
@@ -8684,6 +8743,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		return 0;
 	}
 
+	std::filesystem::path SystemRootDir;
+	auto				  ParseSystemOptions = [&]() {
+		 SystemRootDir = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir);
+	};
+
 	auto ParseStorageOptions = [&]() {
 		if (!m_OverrideHost.empty() || !m_Host.empty())
 		{
@@ -8710,10 +8774,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 											.RetryCount	 = 2};
 
 	auto CreateAuthMgr = [&]() {
+		ZEN_ASSERT(!SystemRootDir.empty());
 		if (!Auth)
 		{
-			std::filesystem::path DataRoot =
-				m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir);
 			if (m_EncryptionKey.empty())
 			{
 				m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456";
@@ -8726,7 +8789,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				ZEN_CONSOLE("Warning: Using default encryption initialization vector");
 			}
 
-			AuthConfig AuthMgrConfig = {.RootDirectory = DataRoot / "auth",
+			AuthConfig AuthMgrConfig = {.RootDirectory = SystemRootDir / "auth",
 										.EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey),
 										.EncryptionIV  = AesIV128Bit::FromString(m_EncryptionIV)};
 			if (!AuthMgrConfig.EncryptionKey.IsValid())
@@ -8742,6 +8805,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	auto ParseAuthOptions = [&]() {
+		ParseSystemOptions();
 		if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty())
 		{
 			CreateAuthMgr();
@@ -9300,6 +9364,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		{
 			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
 
+			ParseSystemOptions();
+
 			if (m_Path.empty())
 			{
 				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
@@ -9362,6 +9428,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						   m_BuildPartNames,
 						   Path,
 						   ZenFolderPath,
+						   SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
 						   m_Clean,
@@ -9407,8 +9474,109 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			return AbortFlag ? 11 : 0;
 		}
 
+		if (SubOption == &m_FetchBlobOptions)
+		{
+			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
+			if (m_BlobHash.empty())
+			{
+				throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help()));
+			}
+
+			IoHash BlobHash;
+			if (!IoHash::TryParse(m_BlobHash, BlobHash))
+			{
+				throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help()));
+			}
+
+			const Oid BuildId = Oid::FromHexString(m_BuildId);
+
+			std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
+															? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
+															: MakeSafeAbsolutePath(m_ZenFolderPath);
+			CreateDirectories(ZenFolderPath);
+			auto _ = MakeGuard([ZenFolderPath]() {
+				if (CleanDirectory(ZenFolderPath, {}))
+				{
+					std::error_code DummyEc;
+					RemoveDir(ZenFolderPath, DummyEc);
+				}
+			});
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+
+			uint64_t CompressedSize;
+			uint64_t DecompressedSize;
+			ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
+			if (AbortFlag)
+			{
+				return 11;
+			}
+			ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
+						BlobHash,
+						CompressedSize,
+						DecompressedSize);
+			return 0;
+		}
+
+		if (SubOption == &m_ValidateBuildPartOptions)
+		{
+			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
+
+			if (m_BuildId.empty())
+			{
+				throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));
+			}
+			Oid BuildId = Oid::TryFromHexString(m_BuildId);
+			if (BuildId == Oid::Zero)
+			{
+				throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help()));
+			}
+
+			if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
+			{
+				throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
+			}
+
+			std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
+															? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
+															: MakeSafeAbsolutePath(m_ZenFolderPath);
+			CreateDirectories(ZenFolderPath);
+			auto _ = MakeGuard([ZenFolderPath]() {
+				if (CleanDirectory(ZenFolderPath, {}))
+				{
+					std::error_code DummyEc;
+					RemoveDir(ZenFolderPath, DummyEc);
+				}
+			});
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+
+			Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId);
+
+			ValidateStatistics ValidateStats;
+			DownloadStatistics DownloadStats;
+			ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats);
+
+			return AbortFlag ? 13 : 0;
+		}
+
 		if (SubOption == &m_MultiTestDownloadOptions)
 		{
+			SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+			CreateDirectories(SystemRootDir);
+			CleanDirectory(SystemRootDir, {});
+			auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); });
+
 			if (m_Path.empty())
 			{
 				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
@@ -9448,6 +9616,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 							   {},
 							   Path,
 							   ZenFolderPath,
+							   SystemRootDir,
 							   m_AllowMultiparts,
 							   m_AllowPartialBlockRequests,
 							   BuildIdString == m_BuildIds.front(),
@@ -9466,6 +9635,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 		if (SubOption == &m_TestOptions)
 		{
+			SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+			CreateDirectories(SystemRootDir);
+			CleanDirectory(SystemRootDir, {});
+			auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); });
+
 			if (m_Path.empty())
 			{
 				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
@@ -9490,7 +9664,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				CleanDirectory(StoragePath, {});
 				m_StoragePath = StoragePath.generic_string();
 			}
-			auto _ = MakeGuard([&]() {
+			auto __ = MakeGuard([&]() {
 				if (m_OverrideHost.empty() && StoragePath.empty())
 				{
 					DeleteDirectories(StoragePath);
@@ -9554,6 +9728,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						   {},
 						   DownloadPath,
 						   ZenFolderPath,
+						   SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   true,
@@ -9576,6 +9751,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						   {},
 						   DownloadPath,
 						   ZenFolderPath,
+						   SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   false,
@@ -9693,6 +9869,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						   {},
 						   DownloadPath,
 						   ZenFolderPath,
+						   SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   false,
@@ -9743,6 +9920,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						   {},
 						   DownloadPath,
 						   ZenFolderPath,
+						   SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   false,
@@ -9761,6 +9939,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						   {},
 						   DownloadPath,
 						   ZenFolderPath,
+						   SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   false,
@@ -9779,6 +9958,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						   {},
 						   DownloadPath,
 						   ZenFolderPath,
+						   SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   false,
@@ -9792,102 +9972,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			return 0;
 		}
-
-		if (SubOption == &m_FetchBlobOptions)
-		{
-			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
-			if (m_BlobHash.empty())
-			{
-				throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help()));
-			}
-
-			IoHash BlobHash;
-			if (!IoHash::TryParse(m_BlobHash, BlobHash))
-			{
-				throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help()));
-			}
-
-			const Oid BuildId = Oid::FromHexString(m_BuildId);
-
-			std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
-
-			BuildStorage::Statistics	  StorageStats;
-			BuildStorageCache::Statistics StorageCacheStats;
-
-			const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
-															? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
-															: MakeSafeAbsolutePath(m_ZenFolderPath);
-			CreateDirectories(ZenFolderPath);
-			auto _ = MakeGuard([ZenFolderPath]() {
-				if (CleanDirectory(ZenFolderPath, {}))
-				{
-					std::error_code DummyEc;
-					RemoveDir(ZenFolderPath, DummyEc);
-				}
-			});
-
-			StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
-
-			uint64_t CompressedSize;
-			uint64_t DecompressedSize;
-			ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
-			if (AbortFlag)
-			{
-				return 11;
-			}
-			ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
-						BlobHash,
-						CompressedSize,
-						DecompressedSize);
-			return 0;
-		}
-
-		if (SubOption == &m_ValidateBuildPartOptions)
-		{
-			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
-
-			if (m_BuildId.empty())
-			{
-				throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));
-			}
-			Oid BuildId = Oid::TryFromHexString(m_BuildId);
-			if (BuildId == Oid::Zero)
-			{
-				throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help()));
-			}
-
-			if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
-			{
-				throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
-			}
-
-			std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
-
-			BuildStorage::Statistics	  StorageStats;
-			BuildStorageCache::Statistics StorageCacheStats;
-
-			const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
-															? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
-															: MakeSafeAbsolutePath(m_ZenFolderPath);
-			CreateDirectories(ZenFolderPath);
-			auto _ = MakeGuard([ZenFolderPath]() {
-				if (CleanDirectory(ZenFolderPath, {}))
-				{
-					std::error_code DummyEc;
-					RemoveDir(ZenFolderPath, DummyEc);
-				}
-			});
-
-			StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
-
-			Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId);
-
-			ValidateStatistics ValidateStats;
-			DownloadStatistics DownloadStats;
-			ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats);
-
-			return AbortFlag ? 13 : 0;
-		}
 	}
 	catch (const ParallellWorkException& Ex)
 	{