path: root/src/zen/cmds/builds_cmd.h

// Copyright Epic Games, Inc. All Rights Reserved.

#pragma once

#include <zencore/scopeguard.h>
#include <zenremotestore/builds/buildstoragecache.h>
#include <zenremotestore/builds/buildstorageutil.h>
#include <zenremotestore/partialblockrequestmode.h>
#include <zenremotestore/transferthreadworkers.h>

#include "authutils.h"
#include "consoleprogress.h"

#include <optional>

namespace zen {

class ProgressBase;
class AuthMgr;

struct CreateBuildStorageOptions
{
	bool RequireNamespace			 = true;
	bool RequireBucket				 = true;
	bool BoostCacheBackgroundWorkers = false;
};

//////////////////////////////////////////////////////////////////////////

struct BuildsConfiguration
{
	std::filesystem::path SystemRootDir;
	bool				  UseSparseFiles = true;

	bool				PlainProgress	  = false;
	bool				LogProgress		  = false;
	bool				Verbose			  = false;
	bool				Quiet			  = false;
	ConsoleProgressMode ProgressMode	  = ConsoleProgressMode::Pretty;
	bool				BoostWorkerCount  = false;
	bool				BoostWorkerMemory = false;
	bool				BoostWorkers	  = false;

	std::string OverrideHost;
	std::string Host;
	std::string Url;
	bool		AssumeHttp2	  = false;
	bool		VerboseHttp	  = false;
	bool		AllowRedirect = false;
	std::string Namespace;
	std::string Bucket;

	std::filesystem::path StoragePath;
	bool				  WriteMetadataAsJson = false;

	std::string ZenCacheHost;

	AuthCommandLineOptions AuthOptions;

	std::string IncludeWildcard;
	std::string ExcludeWildcard;
	std::string ExcludeFolders;
	std::string ExcludeExtensions;

	std::filesystem::path ChunkingCachePath;

	bool		AllowMultiparts			  = true;
	std::string AllowPartialBlockRequests = "true";

	bool AppendNewContent = false;

	std::filesystem::path ZenFolderPath;

	void AddSystemOptions(cxxopts::Options& Ops);
	void AddCloudOptions(cxxopts::Options& Ops);
	void AddFileOptions(cxxopts::Options& Ops);
	void AddCacheOptions(cxxopts::Options& Ops);
	void AddOutputOptions(cxxopts::Options& Ops);
	void AddWorkerOptions(cxxopts::Options& Ops);
	void AddZenFolderOptions(cxxopts::Options& Ops);
	void AddChunkingCacheOptions(cxxopts::Options& Ops);
	void AddWildcardOptions(cxxopts::Options& Ops);
	void AddExcludeFolderOption(cxxopts::Options& Ops);
	void AddExcludeExtensionsOption(cxxopts::Options& Ops);
	void AddMultipartOptions(cxxopts::Options& Ops);
	void AddPartialBlockRequestOptions(cxxopts::Options& Ops);
	void AddAppendNewContentOptions(cxxopts::Options& Ops);
};

//////////////////////////////////////////////////////////////////////////

class BuildsSubCmdBase : public ZenSubCmdBase
{
public:
	BuildsSubCmdBase(BuildsConfiguration& Config, std::string_view Name, std::string_view Description)
	: ZenSubCmdBase(Name, Description)
	, m_Config(Config)
	{
	}

protected:
	const BuildsConfiguration& m_Config;
	std::filesystem::path	   m_ResolvedZenFolderPath;
	// Set by EnsureZenFolderExists: true if this command newly created the zen folder
	// (so CleanZenFolder may remove the whole folder), false if it already existed
	// (in which case only the temp subfolder is wiped, to preserve any pre-existing state).
	bool m_CreatedZenFolder = false;

	struct ResolvedStorage
	{
		std::filesystem::path SystemRootDir;
		std::string			  Host;
		std::string			  Namespace;
		std::string			  Bucket;
		std::filesystem::path StoragePath;
	};

	void LogBanner();
	void LogWorkersInfo(const TransferThreadWorkers& Workers);

	// SystemRootDirOverride / StoragePathOverride: empty = fall back to m_Config.
	ResolvedStorage ParseStorageOptions(std::string&					 BuildId,
										const CreateBuildStorageOptions& Options,
										cxxopts::Options&				 SubOpts,
										const std::filesystem::path&	 SystemRootDirOverride = {},
										const std::filesystem::path&	 StoragePathOverride   = {});

	// Builds the storage instance using the supplied Resolved values. ZenFolder is the final zen folder
	// path (caller resolves; pass GetZenFolderPath() after ResolveZenFolderPath). If ZenFolder is empty
	// the storage runs without a temp directory and keeps all downloads in memory.
	// Caller owns Stats/Auth lifetime. Stats must outlive the returned StorageInstance.
	StorageInstance			 CreateBuildStorage(const std::filesystem::path&	 ZenFolder,
												const CreateBuildStorageOptions& Options,
												cxxopts::Options&				 SubOpts,
												BuildStorageBase::Statistics&	 OutStorageStats,
												BuildStorageCache::Statistics&	 OutCacheStats,
												std::unique_ptr<AuthMgr>&		 OutAuth,
												const ResolvedStorage&			 Resolved);
	Oid						 ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts);
	Oid						 ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts);
	std::vector<Oid>		 ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts);
	std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts);
	CbObject				 ParseBuildMetadata(bool				   CreateBuild,
												std::filesystem::path& BuildMetadataPath,
												const std::string&	   BuildMetadata,
												cxxopts::Options&	   SubOpts);
	void					 ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts);
	IoHash					 ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts);
	EPartialBlockRequestMode ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts);
	void					 ParseZenProcessId(int& ZenProcessId);
	void					 ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards);
	void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions);

	// Resolves the zen folder using this chain: --zen-folder-path; else LocalPath/ZenFolderName
	// if LocalPath is non-empty; else cwd/ZenFolderName. Read-only commands that do not need a
	// zen folder should skip this and pass an empty path to CreateBuildStorage.
	void ResolveZenFolderPath(const std::filesystem::path& LocalPath);
	// Assigns the zen folder to --zen-folder-path if given, else to Fallback directly (no
	// ZenFolderName appended). For commands that want a specific scratch location rather than
	// the standard LocalPath/.zen chain.
	void						 SetZenFolderPath(const std::filesystem::path& Fallback);
	const std::filesystem::path& GetZenFolderPath() const { return m_ResolvedZenFolderPath; }
	// Creates the resolved zen folder and records whether it had to be created.
	// Use together with CleanZenFolder via MakeGuard.
	void EnsureZenFolderExists();

	std::atomic<bool>&			  AbortFlag() const;
	std::atomic<bool>&			  PauseFlag() const;
	std::unique_ptr<ProgressBase> CreateProgress() const;

	// Wipes temp state produced by the command. If EnsureZenFolderExists created the zen folder
	// it is removed outright; otherwise only the temp subfolder is wiped so pre-existing state
	// (e.g. current_state.cbo from a prior download in the same folder) is preserved.
	void CleanZenFolder();
};

//////////////////////////////////////////////////////////////////////////

class BuildsListNamespacesSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsListNamespacesSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	bool				  m_Recursive = false;
	std::filesystem::path m_ResultPath;
};

//////////////////////////////////////////////////////////////////////////

class BuildsListSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsListSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::filesystem::path m_QueryPath;
	std::filesystem::path m_ResultPath;
};

//////////////////////////////////////////////////////////////////////////

class BuildsListBlocksSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsListBlocksSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::string			  m_BuildId;
	std::filesystem::path m_ResultPath;
	uint32_t			  m_MaxCount = 16;
};

//////////////////////////////////////////////////////////////////////////

class BuildsUploadSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsUploadSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::filesystem::path m_Path;
	std::string			  m_BuildId;
	std::string			  m_BuildPartId;
	std::string			  m_BuildPartName;
	bool				  m_CreateBuild = false;
	std::filesystem::path m_BuildMetadataPath;
	std::string			  m_BuildMetadata;
	bool				  m_Clean					  = false;
	uint8_t				  m_BlockReuseMinPercentLimit = 85;
	uint64_t			  m_FindBlockMaxCount		  = 10000;
	bool				  m_PostUploadVerify		  = false;
	std::filesystem::path m_ManifestPath;
	bool				  m_UploadToZenCache = true;
};

//////////////////////////////////////////////////////////////////////////

class BuildsDownloadSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsDownloadSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::filesystem::path	 m_Path;
	std::string				 m_BuildId;
	std::vector<std::string> m_BuildPartIds;
	std::vector<std::string> m_BuildPartNames;
	bool					 m_Clean			  = false;
	bool					 m_Force			  = false;
	bool					 m_PostDownloadVerify = false;
	bool					 m_EnableScavenging	  = true;
	std::filesystem::path	 m_DownloadSpecPath;
	bool					 m_UploadToZenCache = true;
	bool					 m_AllowFileClone	= true;
};

//////////////////////////////////////////////////////////////////////////

class BuildsLsSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsLsSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::string				 m_BuildId;
	std::vector<std::string> m_BuildPartIds;
	std::vector<std::string> m_BuildPartNames;
	std::filesystem::path	 m_ResultPath;
};

//////////////////////////////////////////////////////////////////////////

class BuildsDiffSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsDiffSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::filesystem::path m_Path;
	std::filesystem::path m_DiffPath;
	bool				  m_OnlyChunked = false;
};

//////////////////////////////////////////////////////////////////////////

class BuildsFetchBlobSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsFetchBlobSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::string m_BuildId;
	std::string m_BlobHash;
};

//////////////////////////////////////////////////////////////////////////

class BuildsPrimeCacheSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsPrimeCacheSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::string				 m_BuildId;
	std::vector<std::string> m_BuildPartIds;
	std::vector<std::string> m_BuildPartNames;
	bool					 m_Force = false;
};

//////////////////////////////////////////////////////////////////////////

class BuildsPauseSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsPauseSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	int m_ZenProcessId = -1;
};

//////////////////////////////////////////////////////////////////////////

class BuildsResumeSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsResumeSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	int m_ZenProcessId = -1;
};

//////////////////////////////////////////////////////////////////////////

class BuildsAbortSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsAbortSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	int m_ZenProcessId = -1;
};

//////////////////////////////////////////////////////////////////////////

class BuildsValidatePartSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsValidatePartSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	std::string m_BuildId;
	std::string m_BuildPartId;
	std::string m_BuildPartName;
};

//////////////////////////////////////////////////////////////////////////

class BuildsTestSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsTestSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	// Fixture-only overrides of SystemRootDir/StoragePath; passed to ParseStorageOptions explicitly.
	std::filesystem::path m_TestSystemRootDir;
	std::filesystem::path m_TestStoragePath;

	std::filesystem::path m_Path;
	std::string			  m_BuildPartName;
	std::string			  m_BuildId;
	std::string			  m_BuildPartId;
	bool				  m_CreateBuild				  = false;
	uint64_t			  m_FindBlockMaxCount		  = 10000;
	uint8_t				  m_BlockReuseMinPercentLimit = 85;
	bool				  m_UploadToZenCache		  = true;
	bool				  m_EnableScavenging		  = true;
	bool				  m_AllowFileClone			  = true;
};

//////////////////////////////////////////////////////////////////////////

class BuildsMultiTestDownloadSubCmd : public BuildsSubCmdBase
{
public:
	explicit BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config);
	void Run(const ZenCliOptions& GlobalOptions) override;

private:
	// Fixture-only override of SystemRootDir; passed to CreateStorage explicitly.
	std::filesystem::path m_TestSystemRootDir;

	std::filesystem::path	 m_Path;
	std::vector<std::string> m_BuildIds;
	bool					 m_EnableScavenging = true;
	bool					 m_AllowFileClone	= true;
};

//////////////////////////////////////////////////////////////////////////

class BuildsCommand : public ZenCmdWithSubCommands
{
public:
	static constexpr char Name[] = "builds";
	static constexpr char Description[] =
		"Manage builds - list, list-namespaces, ls, upload, download, prime-cache, diff, fetch-blob, validate-part, pause, resume, abort";

	BuildsCommand();
	~BuildsCommand();

	cxxopts::Options& Options() override { return m_Options; }

	BuildsConfiguration&	   GetConfiguration() { return m_Configuration; }
	const BuildsConfiguration& GetConfiguration() const { return m_Configuration; }

private:
	cxxopts::Options	m_Options{Name, Description};
	std::string			m_SubCommand;
	BuildsConfiguration m_Configuration;

	BuildsListNamespacesSubCmd	  m_ListNamespacesSubCmd;
	BuildsListSubCmd			  m_ListSubCmd;
	BuildsListBlocksSubCmd		  m_ListBlocksSubCmd;
	BuildsUploadSubCmd			  m_UploadSubCmd;
	BuildsDownloadSubCmd		  m_DownloadSubCmd;
	BuildsLsSubCmd				  m_LsSubCmd;
	BuildsDiffSubCmd			  m_DiffSubCmd;
	BuildsFetchBlobSubCmd		  m_FetchBlobSubCmd;
	BuildsPrimeCacheSubCmd		  m_PrimeCacheSubCmd;
	BuildsPauseSubCmd			  m_PauseSubCmd;
	BuildsResumeSubCmd			  m_ResumeSubCmd;
	BuildsAbortSubCmd			  m_AbortSubCmd;
	BuildsValidatePartSubCmd	  m_ValidatePartSubCmd;
	BuildsTestSubCmd			  m_TestSubCmd;
	BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd;

	std::optional<ScopedSignalHandler> m_SigIntGuard;
#if ZEN_PLATFORM_WINDOWS
	std::optional<ScopedSignalHandler> m_SigBreakGuard;
#endif

	bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override;
};

}  // namespace zen