// Copyright Epic Games, Inc. All Rights Reserved.

#include "builds_cmd.h"

#include <zencore/basicfile.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinaryfile.h>
#include <zencore/compactbinaryfmt.h>
#include <zencore/compactbinaryvalue.h>
#include <zencore/compress.h>
#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/parallelwork.h>
#include <zencore/scopeguard.h>
#include <zencore/session.h>
#include <zencore/stream.h>
#include <zencore/string.h>
#include <zencore/trace.h>
#include <zencore/uid.h>
#include <zenhttp/formatters.h>
#include <zenhttp/httpclient.h>
#include <zenhttp/httpclientauth.h>
#include <zenhttp/httpcommon.h>
#include <zenremotestore/builds/buildcontent.h>
#include <zenremotestore/builds/buildmanifest.h>
#include <zenremotestore/builds/buildsavedstate.h>
#include <zenremotestore/builds/buildstoragecache.h>
#include <zenremotestore/builds/buildstorageoperations.h>
#include <zenremotestore/builds/buildstorageutil.h>
#include <zenremotestore/builds/filebuildstorage.h>
#include <zenremotestore/builds/jupiterbuildstorage.h>
#include <zenremotestore/chunking/chunkblock.h>
#include <zenremotestore/chunking/chunkedcontent.h>
#include <zenremotestore/chunking/chunkedfile.h>
#include <zenremotestore/chunking/chunkingcache.h>
#include <zenremotestore/chunking/chunkingcontroller.h>
#include <zenremotestore/filesystemutils.h>
#include <zenremotestore/jupiter/jupiterhost.h>
#include <zenremotestore/operationlogoutput.h>
#include <zenremotestore/transferthreadworkers.h>
#include <zenutil/wildcard.h>
#include <zenutil/workerpools.h>
#include <zenutil/zenserverprocess.h>

#include "../progressbar.h"

#include <signal.h>
#include <memory>
#include <numeric>

ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
#include <tsl/robin_set.h>
#include <json11.hpp>
ZEN_THIRD_PARTY_INCLUDES_END

#if ZEN_PLATFORM_WINDOWS
#	include <zencore/windows.h>
#else
#	include <fcntl.h>
#	include <sys/file.h>
#	include <sys/stat.h>
#	include <unistd.h>
#endif

static const bool DoExtraContentVerify = false;

namespace zen {

using namespace std::literals;

namespace builds_impl {
	static std::atomic<bool> AbortFlag = false;
	static std::atomic<bool> PauseFlag = false;

	static void SignalCallbackHandler(int SigNum)
	{
		if (SigNum == SIGINT)
		{
			PauseFlag = false;
			AbortFlag = true;
		}
#if ZEN_PLATFORM_WINDOWS
		if (SigNum == SIGBREAK)
		{
			PauseFlag = false;
			AbortFlag = true;
		}
#endif	// ZEN_PLATFORM_WINDOWS
	}

	struct ZenStateSharedData
	{
		static constexpr uint64_t kMagicV1 = 0x3176646d636e657a;  // zencmdv1

		uint64_t Magic = 0;	 // Implies the size and layout of this struct - changes to the data requires change to Magic constant
		std::atomic<uint32_t> Pid;
		uint8_t				  SessionId[12];
		uint8_t				  Padding1[4];
		std::atomic<uint8_t>  Abort;
		std::atomic<uint8_t>  Pause;
		uint8_t				  Padding2[2];

		void Initialize()
		{
			Magic = kMagicV1;
			Pid.store(0);
			memset(SessionId, 0, sizeof(SessionId));
			Padding1[0] = Padding1[1] = Padding1[2] = Padding1[3] = 0;
			Abort.store(0);
			Pause.store(0);
			Padding2[0] = Padding2[1] = 0;
		}
	};

	struct MemMap
	{
		void*		Handle = nullptr;
		void*		Data   = nullptr;
		size_t		Size   = 0;
		std::string Name;
	};

	class ZenState
	{
	public:
		ZenState(const ZenState&) = delete;
		ZenState& operator=(const ZenState&) = delete;

		ZenState();
		explicit ZenState(uint32_t Pid);
		~ZenState();

		const ZenStateSharedData& StateData() const
		{
			ZEN_ASSERT(m_Data);
			return *m_Data;
		}
		ZenStateSharedData& StateData()
		{
			ZEN_ASSERT(m_Data);
			return *m_Data;
		}

	private:
		static constexpr std::string_view MapBaseName = "UnrealEngineZenCmd_"sv;
		static constexpr size_t			  MapSize	  = sizeof(ZenStateSharedData);

		bool						  m_Created = false;
		std::unique_ptr<SharedMemory> m_MemMap;
		ZenStateSharedData*			  m_Data = nullptr;

		std::thread m_StateMonitor;
		Event		m_ExitStateMonitorEvent;
	};

	ZenState::ZenState(uint32_t Pid)
	{
		const std::string ZenStateMapName = fmt::format("{}{}", MapBaseName, Pid);

		if (!IsProcessRunning(Pid))
		{
			throw std::runtime_error(fmt::format("The process {} is not running", Pid));
		}
		std::unique_ptr<SharedMemory> MemMap = OpenSharedMemory(ZenStateMapName, MapSize, false);
		if (!MemMap)
		{
			throw std::runtime_error(fmt::format("The process {} is not a running zen process", Pid));
		}
		ZenStateSharedData* data = (ZenStateSharedData*)MemMap->GetData();
		if (uint64_t MemMagic = data->Magic; MemMagic != ZenStateSharedData::kMagicV1)
		{
			throw std::runtime_error(fmt::format("The mem map for process {} has an unsupported magic {:x}, expected {:x}",
												 Pid,
												 MemMagic,
												 ZenStateSharedData::kMagicV1));
		}
		if (uint32_t MemPid = data->Pid.load(); MemPid != Pid)
		{
			throw std::runtime_error(
				fmt::format("The mem map for process {} has an missmatching pid of {}, expected {}", Pid, MemPid, Pid));
		}
		m_MemMap = std::move(MemMap);
		m_Data	 = data;
	}

	ZenState::ZenState()
	{
		int Pid = GetCurrentProcessId();

		const std::string ZenStateMapName = fmt::format("{}{}", MapBaseName, Pid);

		std::unique_ptr<SharedMemory> MemMap = CreateSharedMemory(ZenStateMapName, MapSize, false);
		if (!MemMap)
		{
			throw std::runtime_error(fmt::format("The mem map for process {} could not be created", Pid));
		}

		ZenStateSharedData* data = (ZenStateSharedData*)MemMap->GetData();
		data->Initialize();
		data->Pid.store(gsl::narrow<uint32_t>(Pid));
		const Oid SessionId = GetSessionId();
		memcpy(data->SessionId, &SessionId, sizeof SessionId);

		m_MemMap = std::move(MemMap);
		m_Data	 = data;

		m_StateMonitor = std::thread([this]() {
			while (!m_ExitStateMonitorEvent.Wait(500))
			{
				if (m_Data->Abort.load())
				{
					AbortFlag.store(true);
				}
				PauseFlag.store(m_Data->Pause.load());
			}
		});
	}

	ZenState::~ZenState()
	{
		try
		{
			if (m_StateMonitor.joinable())
			{
				m_ExitStateMonitorEvent.Set();
				m_StateMonitor.join();
			}
			m_MemMap.reset();
		}
		catch (const std::exception& Ex)
		{
			ZEN_CONSOLE_ERROR("ZenState::~ZenState threw exception: {}", Ex.what());
		}
	}

	const std::string	  ZenFolderName = ".zen";
	std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; }
	// std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json";
	// }

	std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path)
	{
		const std::u8string LocalPathString = Path.generic_u8string();
		IoHash				PathHash		= IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length());
		return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash);
	}

	const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt";

	const std::string UnsyncFolderName = ".unsync";

	const std::string UGSFolderName			  = ".ugs";
	const std::string LegacyZenTempFolderName = ".zen-tmp";

	const std::vector<std::string> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName});
	const std::vector<std::string> DefaultExcludeExtensions({});

	const double DefaultLatency		  = 0;	// .0010;
	const double DefaultDelayPerKBSec = 0;	// 0.00005;

	const bool SingleThreaded = false;
	bool	   UseSparseFiles = false;

	static bool				 IsVerbose	  = false;
	static bool				 IsQuiet	  = false;
	static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;

#undef ZEN_CONSOLE_VERBOSE
#define ZEN_CONSOLE_VERBOSE(fmtstr, ...)                            \
	if (IsVerbose)                                                  \
	{                                                               \
		ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \
	}

	const std::string DefaultAccessTokenEnvVariableName(
#if ZEN_PLATFORM_WINDOWS
		"UE-CloudDataCacheAccessToken"sv
#endif
#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
		"UE_CloudDataCacheAccessToken"sv
#endif

	);

	static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory)
	{
		return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u;
	}

	class FilteredRate
	{
	public:
		FilteredRate() {}

		void Start()
		{
			if (StartTimeUS == (uint64_t)-1)
			{
				uint64_t Expected = (uint64_t)-1;
				if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()))
				{
					LastTimeUS = StartTimeUS.load();
				}
			}
		}
		void Stop()
		{
			if (EndTimeUS == (uint64_t)-1)
			{
				uint64_t Expected = (uint64_t)-1;
				EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs());
			}
		}

		void Update(uint64_t Count)
		{
			if (LastTimeUS == (uint64_t)-1)
			{
				return;
			}
			uint64_t TimeUS		 = Timer.GetElapsedTimeUs();
			uint64_t TimeDeltaUS = TimeUS - LastTimeUS;
			if (TimeDeltaUS >= 2000000)
			{
				uint64_t Delta	   = Count - LastCount;
				uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS;

				LastPerSecond = PerSecond;

				LastCount = Count;

				FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8;

				LastTimeUS = TimeUS;
			}
		}

		uint64_t GetCurrent() const	 // If Stopped - return total count / total time
		{
			if (LastTimeUS == (uint64_t)-1)
			{
				return 0;
			}
			return FilteredPerSecond;
		}

		uint64_t GetElapsedTimeUS() const
		{
			if (StartTimeUS == (uint64_t)-1)
			{
				return 0;
			}
			if (EndTimeUS == (uint64_t)-1)
			{
				return 0;
			}
			uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS;
			return TimeDeltaUS;
		}

		bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); }

	private:
		Stopwatch			  Timer;
		std::atomic<uint64_t> StartTimeUS		= (uint64_t)-1;
		std::atomic<uint64_t> EndTimeUS			= (uint64_t)-1;
		std::atomic<uint64_t> LastTimeUS		= (uint64_t)-1;
		uint64_t			  LastCount			= 0;
		uint64_t			  LastPerSecond		= 0;
		uint64_t			  FilteredPerSecond = 0;
	};

	uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count)
	{
		if (ElapsedWallTimeUS == 0)
		{
			return 0;
		}
		return Count * 1000000 / ElapsedWallTimeUS;
	}

	bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory)
	{
		return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory);
	}

	void ValidateBuildPart(OperationLogOutput&	  Output,
						   TransferThreadWorkers& Workers,
						   BuildStorageBase&	  Storage,
						   const Oid&			  BuildId,
						   Oid					  BuildPartId,
						   const std::string_view BuildPartName)
	{
		ZEN_TRACE_CPU("ValidateBuildPart");

		ProgressBar::SetLogOperationName(ProgressMode, "Validate Part");

		BuildsOperationValidateBuildPart ValidateOp(Output,
													Storage,
													AbortFlag,
													PauseFlag,
													Workers.GetIOWorkerPool(),
													Workers.GetNetworkPool(),
													BuildId,
													BuildPartId,
													BuildPartName,
													BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose});

		ValidateOp.Execute();

		const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount;
		const uint64_t DownloadedByteCount =
			ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount;
		ZEN_CONSOLE("Verified:      {:>8} ({}), {}B/sec, {}",
					DownloadedCount,
					NiceBytes(DownloadedByteCount),
					NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)),
					NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000));
	}

	struct UploadFolderOptions
	{
		std::filesystem::path			TempDir;
		uint64_t						FindBlockMaxCount;
		uint8_t							BlockReuseMinPercentLimit;
		bool							AllowMultiparts;
		bool							CreateBuild;
		bool							IgnoreExistingBlocks;
		bool							UploadToZenCache;
		const std::vector<std::string>& ExcludeFolders	  = DefaultExcludeFolders;
		const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions;
	};

	std::vector<std::pair<Oid, std::string>> UploadFolder(OperationLogOutput&		   Output,
														  TransferThreadWorkers&	   Workers,
														  StorageInstance&			   Storage,
														  const Oid&				   BuildId,
														  const Oid&				   BuildPartId,
														  const std::string_view	   BuildPartName,
														  const std::filesystem::path& Path,
														  const std::filesystem::path& ManifestPath,
														  const CbObject&			   MetaData,
														  ChunkingController&		   ChunkController,
														  ChunkingCache&			   ChunkCache,
														  const UploadFolderOptions&   Options)
	{
		ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder");

		Stopwatch UploadTimer;

		BuildsOperationUploadFolder UploadOp(
			Output,
			Storage,
			AbortFlag,
			PauseFlag,
			Workers.GetIOWorkerPool(),
			Workers.GetNetworkPool(),
			BuildId,
			Path,
			Options.CreateBuild,
			std::move(MetaData),
			BuildsOperationUploadFolder::Options{.IsQuiet					= IsQuiet,
												 .IsVerbose					= IsVerbose,
												 .DoExtraContentValidation	= DoExtraContentVerify,
												 .FindBlockMaxCount			= Options.FindBlockMaxCount,
												 .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit,
												 .AllowMultiparts			= Options.AllowMultiparts,
												 .IgnoreExistingBlocks		= Options.IgnoreExistingBlocks,
												 .TempDir					= Options.TempDir,
												 .ExcludeFolders			= Options.ExcludeFolders,
												 .ExcludeExtensions			= Options.ExcludeExtensions,
												 .ZenExcludeManifestName	= ZenExcludeManifestName,
												 .NonCompressableExtensions = DefaultSplitOnlyExtensions,
												 .PopulateCache				= Options.UploadToZenCache});

		std::vector<std::pair<Oid, std::string>> UploadedParts =
			UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache);
		if (AbortFlag)
		{
			return {};
		}

		ZEN_CONSOLE_VERBOSE(
			"Folder scanning stats:"
			"\n    FoundFileCount:        {}"
			"\n    FoundFileByteCount:    {}"
			"\n    AcceptedFileCount:     {}"
			"\n    AcceptedFileByteCount: {}"
			"\n    ElapsedWallTimeUS:     {}",
			UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
			NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
			UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(),
			NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()),
			NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000));

		ZEN_CONSOLE_VERBOSE(
			"Chunking stats:"
			"\n    FilesProcessed:       {}"
			"\n    FilesChunked:         {}"
			"\n    BytesHashed:          {}"
			"\n    UniqueChunksFound:    {}"
			"\n    UniqueSequencesFound: {}"
			"\n    UniqueBytesFound:     {}"
			"\n    FilesFoundInCache:    {}"
			"\n    ChunksFoundInCache:   {}"
			"\n    FilesStoredInCache:   {}"
			"\n    ChunksStoredInCache:  {}"
			"\n    ElapsedWallTimeUS:    {}",
			UploadOp.m_ChunkingStats.FilesProcessed.load(),
			UploadOp.m_ChunkingStats.FilesChunked.load(),
			NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()),
			UploadOp.m_ChunkingStats.UniqueChunksFound.load(),
			UploadOp.m_ChunkingStats.UniqueSequencesFound.load(),
			NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()),
			UploadOp.m_ChunkingStats.FilesFoundInCache.load(),
			UploadOp.m_ChunkingStats.ChunksFoundInCache.load(),
			NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()),
			UploadOp.m_ChunkingStats.FilesStoredInCache.load(),
			UploadOp.m_ChunkingStats.ChunksStoredInCache.load(),
			NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()),
			NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000));

		ZEN_CONSOLE_VERBOSE(
			"Find block stats:"
			"\n    FindBlockTimeMS:             {}"
			"\n    PotentialChunkCount:         {}"
			"\n    PotentialChunkByteCount:     {}"
			"\n    FoundBlockCount:             {}"
			"\n    FoundBlockChunkCount:        {}"
			"\n    FoundBlockByteCount:         {}"
			"\n    AcceptedBlockCount:          {}"
			"\n    NewBlocksCount:              {}"
			"\n    NewBlocksChunkCount:         {}"
			"\n    NewBlocksChunkByteCount:     {}",
			NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS),
			UploadOp.m_FindBlocksStats.PotentialChunkCount,
			NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount),
			UploadOp.m_FindBlocksStats.FoundBlockCount,
			UploadOp.m_FindBlocksStats.FoundBlockChunkCount,
			NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount),
			UploadOp.m_FindBlocksStats.AcceptedBlockCount,
			UploadOp.m_FindBlocksStats.NewBlocksCount,
			UploadOp.m_FindBlocksStats.NewBlocksChunkCount,
			NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount));

		ZEN_CONSOLE_VERBOSE(
			"Reuse block stats:"
			"\n    AcceptedChunkCount:          {}"
			"\n    AcceptedByteCount:           {}"
			"\n    AcceptedRawByteCount:        {}"
			"\n    RejectedBlockCount:          {}"
			"\n    RejectedChunkCount:          {}"
			"\n    RejectedByteCount:           {}"
			"\n    AcceptedReduntantChunkCount: {}"
			"\n    AcceptedReduntantByteCount:  {}",
			UploadOp.m_ReuseBlocksStats.AcceptedChunkCount,
			NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount),
			NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount),
			UploadOp.m_ReuseBlocksStats.RejectedBlockCount,
			UploadOp.m_ReuseBlocksStats.RejectedChunkCount,
			NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount),
			UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount,
			NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount));

		ZEN_CONSOLE_VERBOSE(
			"Generate blocks stats:"
			"\n    GeneratedBlockByteCount:         {}"
			"\n    GeneratedBlockCount:             {}"
			"\n    GenerateBlocksElapsedWallTimeUS: {}",
			NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
			UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
			NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000));

		ZEN_CONSOLE_VERBOSE(
			"Generate blocks stats:"
			"\n    ChunkCount:                      {}"
			"\n    ChunkByteCount:                  {}"
			"\n    CompressedChunkCount:            {}"
			"\n    CompressChunksElapsedWallTimeUS: {}",
			UploadOp.m_LooseChunksStats.ChunkCount,
			NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
			UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
			NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
			NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000));

		ZEN_CONSOLE_VERBOSE(
			"Disk stats:"
			"\n    OpenReadCount:        {}"
			"\n    OpenWriteCount:       {}"
			"\n    ReadCount:            {}"
			"\n    ReadByteCount:        {}"
			"\n    WriteCount:           {} ({} cloned)"
			"\n    WriteByteCount:       {} ({} cloned)"
			"\n    CurrentOpenFileCount: {}",
			UploadOp.m_DiskStats.OpenReadCount.load(),
			UploadOp.m_DiskStats.OpenWriteCount.load(),
			UploadOp.m_DiskStats.ReadCount.load(),
			NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()),
			UploadOp.m_DiskStats.WriteCount.load(),
			UploadOp.m_DiskStats.CloneCount.load(),
			NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()),
			NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()),
			UploadOp.m_DiskStats.CurrentOpenFileCount.load());

		ZEN_CONSOLE_VERBOSE(
			"Upload stats:"
			"\n    BlockCount:               {}"
			"\n    BlocksBytes:              {}"
			"\n    ChunkCount:               {}"
			"\n    ChunksBytes:              {}"
			"\n    ReadFromDiskBytes:        {}"
			"\n    MultipartAttachmentCount: {}"
			"\n    ElapsedWallTimeUS:        {}",
			UploadOp.m_UploadStats.BlockCount.load(),
			NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
			UploadOp.m_UploadStats.ChunkCount.load(),
			NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
			NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()),
			UploadOp.m_UploadStats.MultipartAttachmentCount.load(),
			NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000));

		const double DeltaByteCountPercent =
			UploadOp.m_ChunkingStats.BytesHashed > 0
				? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) /
					  (UploadOp.m_ChunkingStats.BytesHashed)
				: 0.0;

		const std::string MultipartAttachmentStats =
			Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : "";

		if (!IsQuiet)
		{
			ZEN_CONSOLE(
				"Uploaded part {} ('{}') to build {}, {}\n"
				"    Scanned files: {:>8} ({}), {}B/sec, {}\n"
				"    New data:      {:>8} ({}) {:.1f}%\n"
				"    New blocks:    {:>8} ({} -> {}), {}B/sec, {}\n"
				"    New chunks:    {:>8} ({} -> {}), {}B/sec, {}\n"
				"    Uploaded:      {:>8} ({}), {}bits/sec, {}\n"
				"        Blocks:    {:>8} ({})\n"
				"        Chunks:    {:>8} ({}){}",
				BuildPartId,
				BuildPartName,
				BuildId,
				NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()),

				UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
				NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
				NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)),
				NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000),

				UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount,
				NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes),
				DeltaByteCountPercent,

				UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
				NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount),
				NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
				NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
										  UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)),
				NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),

				UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
				NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes),
				NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
				NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS,
										  UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)),
				NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),

				UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(),
				NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes),
				NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS,
										  (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)),
				NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000),

				UploadOp.m_UploadStats.BlockCount.load(),
				NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),

				UploadOp.m_UploadStats.ChunkCount.load(),
				NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
				MultipartAttachmentStats);
		}
		return UploadedParts;
	}

	struct VerifyFolderStatistics
	{
		std::atomic<uint64_t> FilesVerified			  = 0;
		std::atomic<uint64_t> FilesFailed			  = 0;
		std::atomic<uint64_t> ReadBytes				  = 0;
		uint64_t			  VerifyElapsedWallTimeUs = 0;
	};

	void VerifyFolder(TransferThreadWorkers&		  Workers,
					  const ChunkedFolderContent&	  Content,
					  const ChunkedContentLookup&	  Lookup,
					  const std::filesystem::path&	  Path,
					  const std::vector<std::string>& ExcludeFolders,
					  bool							  VerifyFileHash,
					  VerifyFolderStatistics&		  VerifyFolderStats)
	{
		ZEN_TRACE_CPU("VerifyFolder");

		Stopwatch Timer;

		ProgressBar ProgressBar(ProgressMode, "Verify Files");

		WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool();

		ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);

		const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size());

		RwLock					 ErrorLock;
		std::vector<std::string> Errors;

		auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool {
			for (const std::string& ExcludeFolder : ExcludeFolders)
			{
				if (RelativePath.starts_with(ExcludeFolder))
				{
					if (RelativePath.length() == ExcludeFolder.length())
					{
						return false;
					}
					else if (RelativePath[ExcludeFolder.length()] == '/')
					{
						return false;
					}
				}
			}
			return true;
		};

		for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
		{
			if (Work.IsAborted())
			{
				break;
			}

			Work.ScheduleWork(
				VerifyPool,
				[&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex](
					std::atomic<bool>&) {
					if (!AbortFlag)
					{
						ZEN_TRACE_CPU("VerifyFile_work");

						// TODO: Convert ScheduleWork body to function

						const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred();
						if (IsAcceptedFolder(TargetPath.parent_path().generic_string()))
						{
							const uint64_t ExpectedSize = Content.RawSizes[PathIndex];
							if (!IsFile(TargetPath))
							{
								ErrorLock.WithExclusiveLock([&]() {
									Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize));
								});
								VerifyFolderStats.FilesFailed++;
							}
							else
							{
								std::error_code Ec;
								uint64_t		SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec));
								if (Ec)
								{
									ErrorLock.WithExclusiveLock([&]() {
										Errors.push_back(
											fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value()));
									});
									VerifyFolderStats.FilesFailed++;
								}
								else if (SizeOnDisk < ExpectedSize)
								{
									ErrorLock.WithExclusiveLock([&]() {
										Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}",
																	 TargetPath,
																	 ExpectedSize,
																	 SizeOnDisk));
									});
									VerifyFolderStats.FilesFailed++;
								}
								else if (SizeOnDisk > ExpectedSize)
								{
									ErrorLock.WithExclusiveLock([&]() {
										Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}",
																	 TargetPath,
																	 ExpectedSize,
																	 SizeOnDisk));
									});
									VerifyFolderStats.FilesFailed++;
								}
								else if (SizeOnDisk > 0 && VerifyFileHash)
								{
									const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex];
									IoBuffer	  Buffer		  = IoBufferBuilder::MakeFromFile(TargetPath);
									IoHash		  RawHash		  = IoHash::HashBuffer(Buffer);
									if (RawHash != ExpectedRawHash)
									{
										uint64_t	   FileOffset	 = 0;
										const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash);
										const uint32_t OrderOffset	 = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex];
										for (uint32_t OrderIndex = OrderOffset;
											 OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex];
											 OrderIndex++)
										{
											uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex];
											uint64_t ChunkSize	= Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
											IoHash	 ChunkHash	= Content.ChunkedContent.ChunkHashes[ChunkIndex];
											IoBuffer FileChunk	= IoBuffer(Buffer, FileOffset, ChunkSize);
											if (IoHash::HashBuffer(FileChunk) != ChunkHash)
											{
												ErrorLock.WithExclusiveLock([&]() {
													Errors.push_back(fmt::format(
														"WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. "
														"Mismatch at chunk {}",
														TargetPath,
														ExpectedRawHash,
														RawHash,
														OrderIndex - OrderOffset));
												});
												break;
											}
											FileOffset += ChunkSize;
										}
										VerifyFolderStats.FilesFailed++;
									}
									VerifyFolderStats.ReadBytes += SizeOnDisk;
								}
							}
						}
						VerifyFolderStats.FilesVerified++;
					}
				},
				[&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) {
					std::string Description;
					try
					{
						std::rethrow_exception(Ex);
					}
					catch (const std::exception& Ex)
					{
						Description = Ex.what();
					}
					ErrorLock.WithExclusiveLock([&]() {
						Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}",
													 (Path / Content.Paths[PathIndex]).make_preferred(),
													 Description));
					});
					VerifyFolderStats.FilesFailed++;
				});
		}

		Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
			ZEN_UNUSED(PendingWork);
			std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}",
											  VerifyFolderStats.FilesVerified.load(),
											  PathCount,
											  NiceBytes(VerifyFolderStats.ReadBytes.load()),
											  VerifyFolderStats.FilesFailed.load());
			ProgressBar.UpdateState({.Task			 = "Verifying files  ",
									 .Details		 = Details,
									 .TotalCount	 = gsl::narrow<uint64_t>(PathCount),
									 .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()),
									 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
									false);
		});
		VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs();

		ProgressBar.Finish();
		if (AbortFlag)
		{
			return;
		}

		for (const std::string& Error : Errors)
		{
			ZEN_CONSOLE_ERROR("{}", Error);
		}
		if (!Errors.empty())
		{
			throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size()));
		}
	}

	CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId)
	{
		Stopwatch GetBuildTimer;
		CbObject  BuildObject = Storage.GetBuild(BuildId);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
						NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
						BuildObject["name"sv].AsString(),
						NiceBytes(BuildObject.GetSize()));

			ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, "    "sv, "\n"sv));
		}
		return BuildObject;
	}

	std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths,
												   const std::span<const std::filesystem::path> Paths)
	{
		tsl::robin_set<std::string> KnownPathsSet;
		KnownPathsSet.reserve(KnownPaths.size());
		for (const std::filesystem::path& LocalPath : KnownPaths)
		{
			KnownPathsSet.insert(LocalPath.generic_string());
		}

		std::vector<std::filesystem::path> NewPaths;
		for (const std::filesystem::path& UntrackedPath : Paths)
		{
			if (!KnownPathsSet.contains(UntrackedPath.generic_string()))
			{
				NewPaths.push_back(UntrackedPath);
			}
		}
		return NewPaths;
	}

	BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers&				 Workers,
										  GetFolderContentStatistics&			 LocalFolderScanStats,
										  ChunkingStatistics&					 ChunkingStats,
										  const std::filesystem::path&			 Path,
										  ChunkingController&					 ChunkController,
										  ChunkingCache&						 ChunkCache,
										  std::span<const std::filesystem::path> PathsToCheck)
	{
		FolderContent		 FolderState;
		ChunkedFolderContent ChunkedContent;
		{
			ProgressBar ProgressBar(ProgressMode, "Check Files");
			FolderState = GetValidFolderContent(
				Workers.GetIOWorkerPool(),
				LocalFolderScanStats,
				Path,
				PathsToCheck,
				[&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
					std::string Details =
						fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
					ProgressBar.UpdateState({.Task			 = "Checking files   ",
											 .Details		 = Details,
											 .TotalCount	 = PathCount,
											 .RemainingCount = PathCount - CompletedPathCount,
											 .Status		 = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
											false);
				},
				GetUpdateDelayMS(ProgressMode),
				AbortFlag,
				PauseFlag);
			ProgressBar.Finish();
		}

		if (FolderState.Paths.size() > 0)
		{
			uint64_t ByteCountToScan = 0;
			for (const uint64_t RawSize : FolderState.RawSizes)
			{
				ByteCountToScan += RawSize;
			}
			ProgressBar	 ProgressBar(ProgressMode, "Scan Files");
			FilteredRate FilteredBytesHashed;
			FilteredBytesHashed.Start();
			ChunkingStatistics LocalChunkingStats;
			ChunkedContent = ChunkFolderContent(
				LocalChunkingStats,
				Workers.GetIOWorkerPool(),
				Path,
				FolderState,
				ChunkController,
				ChunkCache,
				GetUpdateDelayMS(ProgressMode),
				[&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
					FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load());
					std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
													  LocalChunkingStats.FilesProcessed.load(),
													  FolderState.Paths.size(),
													  NiceBytes(LocalChunkingStats.BytesHashed.load()),
													  NiceBytes(ByteCountToScan),
													  NiceNum(FilteredBytesHashed.GetCurrent()),
													  LocalChunkingStats.UniqueChunksFound.load(),
													  NiceBytes(LocalChunkingStats.UniqueBytesFound.load()));
					ProgressBar.UpdateState({.Task			 = "Scanning files   ",
											 .Details		 = Details,
											 .TotalCount	 = ByteCountToScan,
											 .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(),
											 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
											false);
				},
				AbortFlag,
				PauseFlag);
			ChunkingStats += LocalChunkingStats;
			FilteredBytesHashed.Stop();
			ProgressBar.Finish();
		}

		return BuildSaveState{.State	   = BuildState{.ChunkedContent = std::move(ChunkedContent)},
							  .FolderState = FolderState,
							  .LocalPath   = Path};
	}

	BuildSaveState GetLocalContent(TransferThreadWorkers&		Workers,
								   GetFolderContentStatistics&	LocalFolderScanStats,
								   ChunkingStatistics&			ChunkingStats,
								   const std::filesystem::path& Path,
								   const std::filesystem::path& StateFilePath,
								   ChunkingController&			ChunkController,
								   ChunkingCache&				ChunkCache)
	{
		Stopwatch ReadStateTimer;
		bool	  FileExists = IsFile(StateFilePath);
		if (!FileExists)
		{
			ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path);
			return {};
		}

		BuildSaveState SavedLocalState;
		try
		{
			SavedLocalState = ReadBuildSaveStateFile(StateFilePath);
			if (!IsQuiet)
			{
				ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
			}
		}
		catch (const std::exception& Ex)
		{
			ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
			return {};
		}

		FolderContent CurrentLocalFolderState;
		{
			ProgressBar ProgressBar(ProgressMode, "Check Known Files");
			CurrentLocalFolderState = GetValidFolderContent(
				Workers.GetIOWorkerPool(),
				LocalFolderScanStats,
				Path,
				SavedLocalState.FolderState.Paths,
				[&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
					std::string Details =
						fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
					ProgressBar.UpdateState({.Task			 = "Checking files   ",
											 .Details		 = Details,
											 .TotalCount	 = PathCount,
											 .RemainingCount = PathCount - CompletedPathCount,
											 .Status		 = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
											false);
				},
				GetUpdateDelayMS(ProgressMode),
				AbortFlag,
				PauseFlag);
			ProgressBar.Finish();
		}
		if (AbortFlag)
		{
			return {};
		}

		if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState))
		{
			const size_t					   LocalStatePathCount = SavedLocalState.FolderState.Paths.size();
			std::vector<std::filesystem::path> DeletedPaths;
			FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths);
			if (!DeletedPaths.empty())
			{
				SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
			}

			if (!IsQuiet)
			{
				ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
							DeletedPaths.size(),
							UpdatedContent.Paths.size(),
							LocalStatePathCount);
			}
			if (UpdatedContent.Paths.size() > 0)
			{
				uint64_t ByteCountToScan = 0;
				for (const uint64_t RawSize : UpdatedContent.RawSizes)
				{
					ByteCountToScan += RawSize;
				}
				ProgressBar	 ProgressBar(ProgressMode, "Scan Known Files");
				FilteredRate FilteredBytesHashed;
				FilteredBytesHashed.Start();
				ChunkingStatistics	 LocalChunkingStats;
				ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
					LocalChunkingStats,
					Workers.GetIOWorkerPool(),
					Path,
					UpdatedContent,
					ChunkController,
					ChunkCache,
					GetUpdateDelayMS(ProgressMode),
					[&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
						FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load());
						std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
														  LocalChunkingStats.FilesProcessed.load(),
														  UpdatedContent.Paths.size(),
														  NiceBytes(LocalChunkingStats.BytesHashed.load()),
														  NiceBytes(ByteCountToScan),
														  NiceNum(FilteredBytesHashed.GetCurrent()),
														  LocalChunkingStats.UniqueChunksFound.load(),
														  NiceBytes(LocalChunkingStats.UniqueBytesFound.load()));
						ProgressBar.UpdateState({.Task			 = "Scanning files   ",
												 .Details		 = Details,
												 .TotalCount	 = ByteCountToScan,
												 .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(),
												 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
												false);
					},
					AbortFlag,
					PauseFlag);

				ChunkingStats += LocalChunkingStats;

				FilteredBytesHashed.Stop();
				ProgressBar.Finish();
				if (AbortFlag)
				{
					return {};
				}
				SavedLocalState.State.ChunkedContent =
					MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}});
			}
		}
		else
		{
			// Remove files from LocalContent no longer in LocalFolderState
			tsl::robin_set<std::string> LocalFolderPaths;
			LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size());
			for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths)
			{
				LocalFolderPaths.insert(LocalFolderPath.generic_string());
			}
			std::vector<std::filesystem::path> DeletedPaths;
			for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths)
			{
				if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
				{
					DeletedPaths.push_back(LocalContentPath);
				}
			}
			if (!DeletedPaths.empty())
			{
				SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
			}
		}

		SavedLocalState.FolderState = CurrentLocalFolderState;

		return SavedLocalState;
	}

	ChunkedFolderContent ScanAndChunkFolder(
		TransferThreadWorkers&																	 Workers,
		GetFolderContentStatistics&																 GetFolderContentStats,
		ChunkingStatistics&																		 ChunkingStats,
		const std::filesystem::path&															 Path,
		std::function<bool(const std::string_view& RelativePath)>&&								 IsAcceptedFolder,
		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
		ChunkingController&																		 ChunkController,
		ChunkingCache&																			 ChunkCache)
	{
		Stopwatch Timer;

		ZEN_TRACE_CPU("ScanAndChunkFolder");

		FolderContent Content = GetFolderContent(
			GetFolderContentStats,
			Path,
			std::move(IsAcceptedFolder),
			std::move(IsAcceptedFile),
			Workers.GetIOWorkerPool(),
			GetUpdateDelayMS(ProgressMode),
			[](bool, std::ptrdiff_t) {},
			AbortFlag);
		if (AbortFlag)
		{
			return {};
		}

		BuildState LocalContent = GetLocalContent(Workers,
												  GetFolderContentStats,
												  ChunkingStats,
												  Path,
												  ZenStateFilePath(Path / ZenFolderName),
												  ChunkController,
												  ChunkCache)
									  .State;

		std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths);

		BuildState UntrackedLocalContent =
			GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State;

		ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent,
																 std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent});

		const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
		const uint64_t ChunkedRawSize =
			std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));

		if (!IsQuiet)
		{
			ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
						Result.Paths.size(),
						NiceBytes(TotalRawSize),
						Result.ChunkedContent.ChunkHashes.size(),
						NiceBytes(ChunkedRawSize),
						Path,
						NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
						NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
		}
		return Result;
	};

	struct DownloadOptions
	{
		std::filesystem::path	 SystemRootDir;
		std::filesystem::path	 ZenFolderPath;
		bool					 AllowMultiparts				= true;
		EPartialBlockRequestMode PartialBlockRequestMode		= EPartialBlockRequestMode::Mixed;
		bool					 CleanTargetFolder				= false;
		bool					 PostDownloadVerify				= false;
		bool					 PrimeCacheOnly					= false;
		bool					 EnableOtherDownloadsScavenging = true;
		bool					 EnableTargetFolderScavenging	= true;
		bool					 AllowFileClone					= true;
		std::vector<std::string> IncludeWildcards;
		std::vector<std::string> ExcludeWildcards;
		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
		bool					 PopulateCache				= true;
		bool					 AppendNewContent			= false;
		std::vector<std::string> ExcludeFolders				= DefaultExcludeFolders;
	};

	void DownloadFolder(OperationLogOutput&					 Output,
						TransferThreadWorkers&				 Workers,
						StorageInstance&					 Storage,
						const BuildStorageCache::Statistics& StorageCacheStats,
						const Oid&							 BuildId,
						const std::vector<Oid>&				 BuildPartIds,
						std::span<const std::string>		 BuildPartNames,
						const std::filesystem::path&		 DownloadSpecPath,
						const std::filesystem::path&		 Path,
						const DownloadOptions&				 Options)
	{
		ZEN_TRACE_CPU("DownloadFolder");

		ProgressBar::SetLogOperationName(ProgressMode, "Download Folder");

		enum TaskSteps : uint32_t
		{
			CheckState,
			CompareState,
			Download,
			Verify,
			Cleanup,
			StepCount
		};

		auto EndProgress =
			MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });

		ZEN_ASSERT((!Options.PrimeCacheOnly) ||
				   (Options.PrimeCacheOnly && (Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off)));

		Stopwatch DownloadTimer;

		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount);

		const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath);
		CreateDirectories(ZenTempFolder);

		std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;

		CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);

		std::vector<std::pair<Oid, std::string>> AllBuildParts =
			ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);

		BuildManifest Manifest;
		if (!DownloadSpecPath.empty())
		{
			const std::filesystem::path AbsoluteDownloadSpecPath =
				DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath);
			Manifest = ParseBuildManifest(DownloadSpecPath);
		}

		std::vector<ChunkedFolderContent> PartContents;

		std::unique_ptr<ChunkingController> ChunkController;

		std::vector<ChunkBlockDescription> BlockDescriptions;
		std::vector<IoHash>				   LooseChunkHashes;

		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount);

		ChunkedFolderContent RemoteContent = GetRemoteContent(Output,
															  Storage,
															  BuildId,
															  AllBuildParts,
															  Manifest,
															  Options.IncludeWildcards,
															  Options.ExcludeWildcards,
															  ChunkController,
															  PartContents,
															  BlockDescriptions,
															  LooseChunkHashes,
															  IsQuiet,
															  IsVerbose,
															  DoExtraContentVerify);

		const std::uint64_t		   LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
		GetFolderContentStatistics LocalFolderScanStats;
		ChunkingStatistics		   ChunkingStats;

		BuildSaveState LocalState;

		if (!Options.PrimeCacheOnly)
		{
			if (IsDir(Path))
			{
				if (!ChunkController && !IsQuiet)
				{
					ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default");
					ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
				}
				std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache());

				LocalState = GetLocalContent(Workers,
											 LocalFolderScanStats,
											 ChunkingStats,
											 Path,
											 ZenStateFilePath(Path / ZenFolderName),
											 *ChunkController,
											 *ChunkCache);

				std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths);

				BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Workers,
																			  LocalFolderScanStats,
																			  ChunkingStats,
																			  Path,
																			  *ChunkController,
																			  *ChunkCache,
																			  UntrackedPaths);

				if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty())
				{
					LocalState.State.ChunkedContent =
						MergeChunkedFolderContents(LocalState.State.ChunkedContent,
												   std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent});

					// TODO: Helper
					LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(),
														UntrackedLocalContent.FolderState.Paths.begin(),
														UntrackedLocalContent.FolderState.Paths.end());
					LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(),
														   UntrackedLocalContent.FolderState.RawSizes.begin(),
														   UntrackedLocalContent.FolderState.RawSizes.end());
					LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(),
															 UntrackedLocalContent.FolderState.Attributes.begin(),
															 UntrackedLocalContent.FolderState.Attributes.end());
					LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(),
																	UntrackedLocalContent.FolderState.ModificationTicks.begin(),
																	UntrackedLocalContent.FolderState.ModificationTicks.end());
				}

				if (Options.AppendNewContent)
				{
					RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent,
															   RemoteContent,
															   Options.IncludeWildcards,
															   Options.ExcludeWildcards);
				}
#if ZEN_BUILD_DEBUG
				ValidateChunkedFolderContent(RemoteContent,
											 BlockDescriptions,
											 LooseChunkHashes,
											 Options.IncludeWildcards,
											 Options.ExcludeWildcards);
#endif	// ZEN_BUILD_DEBUG
			}
			else
			{
				CreateDirectories(Path);
			}
		}
		if (AbortFlag)
		{
			return;
		}

		LocalState.LocalPath = Path;

		{
			BuildsSelection::Build RemoteBuildState = {.Id				 = BuildId,
													   .IncludeWildcards = Options.IncludeWildcards,
													   .ExcludeWildcards = Options.ExcludeWildcards};
			RemoteBuildState.Parts.reserve(BuildPartIds.size());
			for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++)
			{
				RemoteBuildState.Parts.push_back(
					{BuildsSelection::BuildPart{.Id	  = BuildPartIds[PartIndex],
												.Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}});
			}

			if (Options.AppendNewContent)
			{
				LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState));
			}
			else
			{
				LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)};
			}
		}

		if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder &&
			CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent))
		{
			if (!IsQuiet)
			{
				ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
							NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
			}

			Stopwatch WriteStateTimer;

			CbObject StateObject = CreateBuildSaveStateObject(LocalState);
			CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
			TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
			if (!IsQuiet)
			{
				ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
			}

			AddDownloadedPath(Options.SystemRootDir,
							  BuildsDownloadInfo{.Selection		= LocalState.State.Selection,
												 .LocalPath		= Path,
												 .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
												 .Iso8601Date	= DateTime::Now().ToIso8601()});
		}
		else
		{
			ExtendableStringBuilder<128> BuildPartString;
			for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
			{
				BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first));
			}

			uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));

			if (!IsQuiet)
			{
				ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
			}

			Stopwatch IndexTimer;

			const ChunkedContentLookup LocalLookup	= BuildChunkedContentLookup(LocalState.State.ChunkedContent);
			const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);

			if (!IsQuiet)
			{
				ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
			}

			ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);

			BuildsOperationUpdateFolder Updater(
				Output,
				Storage,
				AbortFlag,
				PauseFlag,
				Workers.GetIOWorkerPool(),
				Workers.GetNetworkPool(),
				BuildId,
				Path,
				LocalState.State.ChunkedContent,
				LocalLookup,
				RemoteContent,
				RemoteLookup,
				BlockDescriptions,
				LooseChunkHashes,
				BuildsOperationUpdateFolder::Options{
					.IsQuiet						= IsQuiet,
					.IsVerbose						= IsVerbose,
					.AllowFileClone					= Options.AllowFileClone,
					.UseSparseFiles					= UseSparseFiles,
					.SystemRootDir					= Options.SystemRootDir,
					.ZenFolderPath					= Options.ZenFolderPath,
					.LargeAttachmentSize			= LargeAttachmentSize,
					.PreferredMultipartChunkSize	= PreferredMultipartChunkSize,
					.PartialBlockRequestMode		= Options.PartialBlockRequestMode,
					.WipeTargetFolder				= Options.CleanTargetFolder,
					.PrimeCacheOnly					= Options.PrimeCacheOnly,
					.EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
					.EnableTargetFolderScavenging	= Options.EnableTargetFolderScavenging || Options.AppendNewContent,
					.ValidateCompletedSequences		= Options.PostDownloadVerify,
					.ExcludeFolders					= Options.ExcludeFolders,
					.MaximumInMemoryPayloadSize		= Options.MaximumInMemoryPayloadSize,
					.PopulateCache					= Options.PopulateCache});
			{
				ProgressBar::PushLogOperation(ProgressMode, "Download");
				auto		  _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); });
				FolderContent UpdatedLocalFolderState;
				Updater.Execute(UpdatedLocalFolderState);

				LocalState.State.ChunkedContent = RemoteContent;
				LocalState.FolderState			= std::move(UpdatedLocalFolderState);
			}

			VerifyFolderStatistics VerifyFolderStats;
			if (!AbortFlag)
			{
				if (!Options.PrimeCacheOnly)
				{
					AddDownloadedPath(Options.SystemRootDir,
									  BuildsDownloadInfo{.Selection		= LocalState.State.Selection,
														 .LocalPath		= Path,
														 .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
														 .Iso8601Date	= DateTime::Now().ToIso8601()});

					ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount);

					VerifyFolder(Workers,
								 RemoteContent,
								 RemoteLookup,
								 Path,
								 Options.ExcludeFolders,
								 Options.PostDownloadVerify,
								 VerifyFolderStats);

					Stopwatch WriteStateTimer;
					CbObject  StateObject = CreateBuildSaveStateObject(LocalState);

					CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
					TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
					if (!IsQuiet)
					{
						ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
					}

#if 0
					ExtendableStringBuilder<1024> SB;
					CompactBinaryToJson(StateObject, SB);
					WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
#endif	// 0
				}
				const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() +
											   Updater.m_DownloadStats.DownloadedBlockCount.load() +
											   Updater.m_DownloadStats.DownloadedPartialBlockCount.load();
				const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() +
												   Updater.m_DownloadStats.DownloadedBlockByteCount.load() +
												   Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load();
				const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();

				if (!IsQuiet)
				{
					std::string CloneInfo;
					if (Updater.m_DiskStats.CloneByteCount > 0)
					{
						CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load()));
					}

					std::string DownloadDetails;
					{
						ExtendableStringBuilder<128>		 SB;
						BuildStorageBase::ExtendedStatistics ExtendedDownloadStats;
						if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats))
						{
							if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty())
							{
								for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource)
								{
									if (SB.Size() > 0)
									{
										SB.Append(", "sv);
									}
									SB.Append(It.first);
									SB.Append(": "sv);
									SB.Append(NiceBytes(It.second));
								}
							}
						}
						if (Storage.CacheStorage)
						{
							if (SB.Size() > 0)
							{
								SB.Append(", "sv);
							}
							SB.Append("Cache: ");
							SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load()));
						}
						if (SB.Size() > 0)
						{
							DownloadDetails = fmt::format(" ({})", SB.ToView());
						}
					}

					ZEN_CONSOLE(
						"Downloaded build {}, parts:{} in {}\n"
						"    Scavenge:   {} (Target: {}, Cache: {}, Others: {})\n"
						"    Download:   {} ({}) {}bits/s{}\n"
						"    Write:      {} ({}) {}B/s{}\n"
						"    Clean:      {}\n"
						"    Finalize:   {}\n"
						"    Verify:     {}",
						BuildId,
						BuildPartString.ToView(),
						NiceTimeSpanMs(DownloadTimeMs),

						NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs +
										Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs +
										Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) /
									   1000),
						NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000),
						NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000),
						NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000),

						DownloadCount,
						NiceBytes(DownloadByteCount),
						NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
						DownloadDetails,

						Updater.m_DiskStats.WriteCount.load(),
						NiceBytes(Updater.m_WrittenChunkByteCount.load()),
						NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())),
						CloneInfo,

						NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),

						NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),

						NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
				}
			}
		}
		if (Options.PrimeCacheOnly)
		{
			if (Storage.CacheStorage)
			{
				Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) {
					if (!IsQuiet)
					{
						if (Remaining == 0)
						{
							ZEN_CONSOLE("Build cache upload complete");
						}
						else
						{
							ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
						}
					}
					return !AbortFlag;
				});
			}
		}

		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);

		CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder);
	}

	void ListBuild(StorageInstance&				Storage,
				   const Oid&					BuildId,
				   const std::vector<Oid>&		BuildPartIds,
				   std::span<const std::string> BuildPartNames,
				   std::span<const std::string> IncludeWildcards,
				   std::span<const std::string> ExcludeWildcards,
				   CbObjectWriter*				OptionalStructuredOutput)
	{
		std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;

		CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);

		if (OptionalStructuredOutput != nullptr)
		{
			OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId);
			OptionalStructuredOutput->AddObject("build"sv, BuildObject);
		}

		std::vector<std::pair<Oid, std::string>> AllBuildParts =
			ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);

		if (!AllBuildParts.empty())
		{
			Stopwatch GetBuildPartTimer;

			if (OptionalStructuredOutput != nullptr)
			{
				OptionalStructuredOutput->BeginArray("parts"sv);
			}

			for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
			{
				const Oid			   BuildPartId		 = AllBuildParts[BuildPartIndex].first;
				const std::string_view BuildPartName	 = AllBuildParts[BuildPartIndex].second;
				CbObject			   BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);

				if (OptionalStructuredOutput != nullptr)
				{
					OptionalStructuredOutput->BeginObject();
					OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId);
					OptionalStructuredOutput->AddString("partName"sv, BuildPartName);
				}
				{
					if (OptionalStructuredOutput != nullptr)
					{
					}
					else if (!IsQuiet)
					{
						ZEN_CONSOLE("{}Part: {} ('{}'):\n",
									BuildPartIndex > 0 ? "\n" : "",
									BuildPartId,
									BuildPartName,
									NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
									NiceBytes(BuildPartManifest.GetSize()));
					}

					std::vector<std::filesystem::path> Paths;
					std::vector<IoHash>				   RawHashes;
					std::vector<uint64_t>			   RawSizes;
					std::vector<uint32_t>			   Attributes;

					SourcePlatform		  Platform;
					std::vector<IoHash>	  SequenceRawHashes;
					std::vector<uint32_t> ChunkCounts;
					std::vector<uint32_t> AbsoluteChunkOrders;
					std::vector<IoHash>	  LooseChunkHashes;
					std::vector<uint64_t> LooseChunkRawSizes;
					std::vector<IoHash>	  BlockRawHashes;

					ReadBuildContentFromCompactBinary(BuildPartManifest,
													  Platform,
													  Paths,
													  RawHashes,
													  RawSizes,
													  Attributes,
													  SequenceRawHashes,
													  ChunkCounts,
													  AbsoluteChunkOrders,
													  LooseChunkHashes,
													  LooseChunkRawSizes,
													  BlockRawHashes);

					std::vector<size_t> Order(Paths.size());
					std::iota(Order.begin(), Order.end(), 0);

					std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
						const std::filesystem::path& LhsPath = Paths[Lhs];
						const std::filesystem::path& RhsPath = Paths[Rhs];
						return LhsPath < RhsPath;
					});

					if (OptionalStructuredOutput != nullptr)
					{
						OptionalStructuredOutput->BeginArray("files"sv);
					}
					{
						for (size_t Index : Order)
						{
							const std::filesystem::path& Path = Paths[Index];
							if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true))
							{
								const IoHash&  RawHash	 = RawHashes[Index];
								const uint64_t RawSize	 = RawSizes[Index];
								const uint32_t Attribute = Attributes[Index];

								if (OptionalStructuredOutput != nullptr)
								{
									OptionalStructuredOutput->BeginObject();
									{
										OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path));
										OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize);
										switch (Platform)
										{
											case SourcePlatform::Windows:
												OptionalStructuredOutput->AddInteger("attributes"sv, Attribute);
												break;
											case SourcePlatform::MacOS:
											case SourcePlatform::Linux:
												OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute));
												break;
											default:
												throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform));
										}
									}
									OptionalStructuredOutput->EndObject();
								}
								else
								{
									ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
								}
							}
						}
					}
					if (OptionalStructuredOutput != nullptr)
					{
						OptionalStructuredOutput->EndArray();  // "files"
					}
				}
				if (OptionalStructuredOutput != nullptr)
				{
					OptionalStructuredOutput->EndObject();
				}
			}
			if (OptionalStructuredOutput != nullptr)
			{
				OptionalStructuredOutput->EndArray();  // parts
			}
		}
	}

	void DiffFolders(TransferThreadWorkers&			 Workers,
					 const std::filesystem::path&	 BasePath,
					 const std::filesystem::path&	 ComparePath,
					 ChunkingController&			 ChunkController,
					 ChunkingCache&					 ChunkCache,
					 const std::vector<std::string>& ExcludeFolders,
					 const std::vector<std::string>& ExcludeExtensions)
	{
		ZEN_TRACE_CPU("DiffFolders");

		ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders");

		enum TaskSteps : uint32_t
		{
			CheckBase,
			CheckCompare,
			Diff,
			Cleanup,
			StepCount
		};

		auto EndProgress =
			MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });

		ChunkedFolderContent BaseFolderContent;
		ChunkedFolderContent CompareFolderContent;

		{
			auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool {
				for (const std::string& ExcludeFolder : ExcludeFolders)
				{
					if (RelativePath.starts_with(ExcludeFolder))
					{
						if (RelativePath.length() == ExcludeFolder.length())
						{
							return false;
						}
						else if (RelativePath[ExcludeFolder.length()] == '/')
						{
							return false;
						}
					}
				}
				return true;
			};

			auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool {
				for (const std::string& ExcludeExtension : ExcludeExtensions)
				{
					if (RelativePath.ends_with(ExcludeExtension))
					{
						return false;
					}
				}
				return true;
			};

			ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount);

			GetFolderContentStatistics BaseGetFolderContentStats;
			ChunkingStatistics		   BaseChunkingStats;
			BaseFolderContent = ScanAndChunkFolder(Workers,
												   BaseGetFolderContentStats,
												   BaseChunkingStats,
												   BasePath,
												   IsAcceptedFolder,
												   IsAcceptedFile,
												   ChunkController,
												   ChunkCache);
			if (AbortFlag)
			{
				return;
			}

			ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount);

			GetFolderContentStatistics CompareGetFolderContentStats;
			ChunkingStatistics		   CompareChunkingStats;
			CompareFolderContent = ScanAndChunkFolder(Workers,
													  CompareGetFolderContentStats,
													  CompareChunkingStats,
													  ComparePath,
													  IsAcceptedFolder,
													  IsAcceptedFile,
													  ChunkController,
													  ChunkCache);

			if (AbortFlag)
			{
				return;
			}
		}

		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount);

		std::vector<IoHash> AddedHashes;
		std::vector<IoHash> RemovedHashes;
		uint64_t			RemovedSize = 0;
		uint64_t			AddedSize	= 0;

		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup;
		for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++)
		{
			const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
			BaseRawHashLookup.insert_or_assign(RawHash, PathIndex);
		}
		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup;
		for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++)
		{
			const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex];
			if (!BaseRawHashLookup.contains(RawHash))
			{
				AddedHashes.push_back(RawHash);
				AddedSize += CompareFolderContent.RawSizes[PathIndex];
			}
			CompareRawHashLookup.insert_or_assign(RawHash, PathIndex);
		}
		for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
		{
			const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
			if (!CompareRawHashLookup.contains(RawHash))
			{
				RemovedHashes.push_back(RawHash);
				RemovedSize += BaseFolderContent.RawSizes[PathIndex];
			}
		}

		uint64_t BaseTotalRawSize = 0;
		for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
		{
			BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex];
		}

		double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0;

		ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept",
					RemovedHashes.size(),
					NiceBytes(RemovedSize),
					AddedHashes.size(),
					NiceBytes(AddedSize),
					BaseFolderContent.Paths.size() - RemovedHashes.size(),
					NiceBytes(BaseTotalRawSize - RemovedSize),
					KeptPercent);

		uint64_t CompareTotalRawSize = 0;

		uint64_t				   FoundChunkCount	= 0;
		uint64_t				   FoundChunkSize	= 0;
		uint64_t				   NewChunkCount	= 0;
		uint64_t				   NewChunkSize		= 0;
		const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent);
		for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
		{
			const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex];
			if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash))
			{
				FoundChunkCount++;
				FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
			}
			else
			{
				NewChunkCount++;
				NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
			}
			CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
		}

		double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0;
		double NewPercent	= CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0;

		ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
					FoundChunkCount,
					NiceBytes(FoundChunkSize),
					FoundPercent,
					CompareFolderContent.ChunkedContent.ChunkHashes.size(),
					NiceBytes(CompareTotalRawSize),
					BaseFolderContent.ChunkedContent.ChunkHashes.size(),
					NiceBytes(BaseTotalRawSize),
					NewChunkCount,
					NiceBytes(NewChunkSize),
					NewPercent);

		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
	}

}  // namespace builds_impl

//////////////////////////////////////////////////////////////////////////////////////////////////////

BuildsCommand::BuildsCommand()
{
	using namespace builds_impl;
	m_Options.add_options()("h,help", "Print help");

	auto AddSystemOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
		Ops.add_option("",
					   "",
					   "use-sparse-files",
					   "Enable use of sparse files when writing large files. Defaults to true.",
					   cxxopts::value(m_UseSparseFiles),
					   "<usesparsefiles>");
	};

	auto AddCloudOptions = [this](cxxopts::Options& Ops) {
		m_AuthOptions.AddOptions(Ops);

		Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
		Ops.add_option("cloud build",
					   "",
					   "url",
					   "Cloud Builds host url (legacy - use --override-host)",
					   cxxopts::value(m_OverrideHost),
					   "<url>");
		Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>");
		Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
		Ops.add_option("cloud build",
					   "",
					   "assume-http2",
					   "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake",
					   cxxopts::value(m_AssumeHttp2),
					   "<assumehttp2>");
		Ops.add_option("cloud build",
					   "",
					   "verbose-http",
					   "Enable verbose option for http client",
					   cxxopts::value(m_VerboseHttp),
					   "<verbosehttp>");

		Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>");
		Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>");
		Ops.add_option("cloud build",
					   "",
					   "allow-redirect",
					   "Allow redirect of requests",
					   cxxopts::value(m_AllowRedirect),
					   "<allow-redirect>");
	};

	auto AddFileOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>");
		Ops.add_option("filestorage",
					   "",
					   "json-metadata",
					   "Write build, part and block metadata as .json files in addition to .cb files",
					   cxxopts::value(m_WriteMetadataAsJson),
					   "<jsonmetadata>");
	};

	auto AddCacheOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
	};

	auto AddOutputOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("output",
					   "",
					   "plain-progress",
					   "Show progress using plain output",
					   cxxopts::value(m_PlainProgress),
					   "<plainprogress>");
		Ops.add_option("output",
					   "",
					   "log-progress",
					   "Write @progress style progress to output",
					   cxxopts::value(m_LogProgress),
					   "<logprogress>");
		Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
		Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
	};

	auto AddWorkerOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "boost-worker-count",
					   "Increase the number of worker threads - may cause computer to be less responsive",
					   cxxopts::value(m_BoostWorkerCount),
					   "<boostworkercount>");

		Ops.add_option("",
					   "",
					   "boost-worker-memory",
					   "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to "
					   "be less responsive due to high memory usage",
					   cxxopts::value(m_BoostWorkerMemory),
					   "<boostworkermemory>");

		Ops.add_option("",
					   "",
					   "boost-workers",
					   "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
					   cxxopts::value(m_BoostWorkers),
					   "<boostworkermemory>");
	};

	auto AddZenFolderOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "zen-folder-path",
					   fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName),
					   cxxopts::value(m_ZenFolderPath),
					   "<boostworkers>");
	};

	auto AddChunkingCacheOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "chunking-cache-path",
					   "Path to cache for chunking information of scanned files. Default is empty resulting in no caching",
					   cxxopts::value(m_ChunkingCachePath),
					   "<chunkingcachepath>");
	};

	auto AddWildcardOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "wildcard",
					   "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;",
					   cxxopts::value(m_IncludeWildcard),
					   "<wildcard>");

		Ops.add_option("",
					   "",
					   "exclude-wildcard",
					   "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard "
					   "include filter",
					   cxxopts::value(m_ExcludeWildcard),
					   "<excludewildcard>");
	};

	auto AddExcludeFolderOption = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "exclude-folders",
					   "Names of folders to exclude, separated by ;",
					   cxxopts::value(m_ExcludeFolders),
					   "<excludefolders>");
	};

	auto AddExcludeExtensionsOption = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "exclude-extensions",
					   "Extensions to exclude, separated by ;"
					   "include filter",
					   cxxopts::value(m_ExcludeExtensions),
					   "<excludeextensions>");
	};

	auto AddMultipartOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "allow-multipart",
					   "Allow large attachments to be transfered using multipart protocol. Defaults to true.",
					   cxxopts::value(m_AllowMultiparts),
					   "<allowmultipart>");
	};

	auto AddPartialBlockRequestOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "allow-partial-block-requests",
					   "Allow request for partial chunk blocks.\n"
					   "  false        = only full block requests allowed\n"
					   "  mixed        = multiple partial block ranges requests per block allowed to zen cache, single partial block range "
					   "request per block to host\n"
					   "  zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests "
					   "allowed to host\n"
					   "  true         = multiple partial block ranges requests per block allowed to zen cache and host\n"
					   "Defaults to 'mixed'.",
					   cxxopts::value(m_AllowPartialBlockRequests),
					   "<allowpartialblockrequests>");
	};

	auto AddAppendNewContentOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("",
					   "",
					   "append",
					   "Decides if the remote data should replace or append to the current local data.\n"
					   "  false        = the local content will be replaced by the remote content\n"
					   "  true         = the remote data will be overlayed on top of local data\n"
					   "Defaults to false.",
					   cxxopts::value(m_AppendNewContent),
					   "<append>");
	};

	m_Options.add_option("",
						 "v",
						 "verb",
						 "Verb for build - list-namespaces, list, upload, download, diff, fetch-blob, validate-part",
						 cxxopts::value(m_Verb),
						 "<verb>");
	m_Options.parse_positional({"verb"});
	m_Options.positional_help("verb");

	// list-namespaces
	AddSystemOptions(m_ListNamespacesOptions);
	AddCloudOptions(m_ListNamespacesOptions);
	AddFileOptions(m_ListNamespacesOptions);
	AddOutputOptions(m_ListNamespacesOptions);
	AddZenFolderOptions(m_ListNamespacesOptions);
	m_ListNamespacesOptions.add_options()("h,help", "Print help");
	m_ListNamespacesOptions.add_option("",
									   "",
									   "recursive",
									   "Enable fetch of buckets within namespaces also",
									   cxxopts::value(m_ListNamespacesRecursive),
									   "<recursive>");
	m_ListNamespacesOptions.add_option(
		"",
		"",
		"result-path",
		"Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
		cxxopts::value(m_ListResultPath),
		"<result-path>");
	m_ListNamespacesOptions.parse_positional({"result-path"});
	m_ListNamespacesOptions.positional_help("result-path");

	// list
	AddSystemOptions(m_ListOptions);
	AddCloudOptions(m_ListOptions);
	AddFileOptions(m_ListOptions);
	AddOutputOptions(m_ListOptions);
	AddZenFolderOptions(m_ListOptions);
	m_ListOptions.add_options()("h,help", "Print help");
	m_ListOptions.add_option("",
							 "",
							 "query-path",
							 "Path to json or compactbinary file containing list query",
							 cxxopts::value(m_ListQueryPath),
							 "<query-path>");
	m_ListOptions.add_option("",
							 "",
							 "result-path",
							 "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
							 cxxopts::value(m_ListResultPath),
							 "<result-path>");
	m_ListOptions.parse_positional({"query-path", "result-path"});
	m_ListOptions.positional_help("query-path result-path");

	// list-blocks
	AddSystemOptions(m_ListBlocksOptions);
	AddCloudOptions(m_ListBlocksOptions);
	AddZenFolderOptions(m_ListBlocksOptions);
	m_ListBlocksOptions.add_options()("h,help", "Print help");
	m_ListBlocksOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
	m_ListBlocksOptions.add_option("",
								   "",
								   "result-path",
								   "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
								   cxxopts::value(m_ListResultPath),
								   "<result-path>");

	m_ListBlocksOptions
		.add_option("", "", "max-count", "Maximum number of blocks to list", cxxopts::value(m_ListBlocksMaxCount), "<maxcount>");

	m_ListBlocksOptions.parse_positional({"build-id"});
	m_ListBlocksOptions.positional_help("build-id");

	// upload
	AddSystemOptions(m_UploadOptions);
	AddCloudOptions(m_UploadOptions);
	AddFileOptions(m_UploadOptions);
	AddOutputOptions(m_UploadOptions);
	AddCacheOptions(m_UploadOptions);
	AddWorkerOptions(m_UploadOptions);
	AddZenFolderOptions(m_UploadOptions);
	AddExcludeFolderOption(m_UploadOptions);
	AddExcludeExtensionsOption(m_UploadOptions);
	AddChunkingCacheOptions(m_UploadOptions);
	m_UploadOptions.add_options()("h,help", "Print help");
	m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
	m_UploadOptions.add_option("",
							   "",
							   "create-build",
							   "Set to true to create the containing build, if unset a builds-id must be given and the build already exist",
							   cxxopts::value(m_CreateBuild),
							   "<id>");
	m_UploadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
	m_UploadOptions.add_option("",
							   "",
							   "build-part-id",
							   "Build part Id, if not given it will be auto generated",
							   cxxopts::value(m_BuildPartId),
							   "<id>");
	m_UploadOptions.add_option("",
							   "",
							   "build-part-name",
							   "Name of the build part, if not given it will be be named after the directory name at end of local-path",
							   cxxopts::value(m_BuildPartName),
							   "<name>");
	m_UploadOptions.add_option("",
							   "",
							   "metadata-path",
							   "Path to json file that holds the metadata for the build. Requires the create-build option to be set",
							   cxxopts::value(m_BuildMetadataPath),
							   "<metadata-path>");
	m_UploadOptions.add_option(
		"",
		"",
		"metadata",
		"Key-value pairs separated by ';' with build meta data. (key1=value1;key2=value2). Requires the create-build option to be set",
		cxxopts::value(m_BuildMetadata),
		"<metadata>");
	m_UploadOptions.add_option("", "", "clean", "Ignore existing blocks", cxxopts::value(m_Clean), "<clean>");
	m_UploadOptions.add_option("",
							   "",
							   "block-min-reuse",
							   "Percent of an existing block that must be relevant for it to be resused. Defaults to 85.",
							   cxxopts::value(m_BlockReuseMinPercentLimit),
							   "<minreuse>");
	m_UploadOptions.add_option("cache",
							   "",
							   "zen-cache-upload",
							   "Upload data downloaded from remote host to zen cache",
							   cxxopts::value(m_UploadToZenCache),
							   "<uploadtozencache>");

	AddMultipartOptions(m_UploadOptions);

	m_UploadOptions.add_option("",
							   "",
							   "manifest-path",
							   "Path to a text file with one line of <local path>[TAB]<modification date> per file to include or a "
							   "structured .json file describing the parts",
							   cxxopts::value(m_ManifestPath),
							   "<manifestpath>");

	m_UploadOptions
		.add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>");
	m_UploadOptions.add_option("",
							   "",
							   "find-max-block-count",
							   "The maximum number of blocks we search for in the build context",
							   cxxopts::value(m_FindBlockMaxCount),
							   "<maxblockcount>");

	m_UploadOptions.parse_positional({"local-path", "build-id"});
	m_UploadOptions.positional_help("local-path build-id");

	// download
	AddSystemOptions(m_DownloadOptions);
	AddCloudOptions(m_DownloadOptions);
	AddFileOptions(m_DownloadOptions);
	AddOutputOptions(m_DownloadOptions);
	AddCacheOptions(m_DownloadOptions);
	AddZenFolderOptions(m_DownloadOptions);
	AddWorkerOptions(m_DownloadOptions);
	AddWildcardOptions(m_DownloadOptions);
	AddAppendNewContentOptions(m_DownloadOptions);
	AddExcludeFolderOption(m_DownloadOptions);

	m_DownloadOptions.add_option("cache",
								 "",
								 "cache-prime-only",
								 "Only download blobs missing in cache and upload to cache",
								 cxxopts::value(m_PrimeCacheOnly),
								 "<cacheprimeonly>");

	m_DownloadOptions.add_options()("h,help", "Print help");
	m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
	m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
	m_DownloadOptions.add_option(
		"",
		"",
		"build-part-id",
		"Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
		cxxopts::value(m_BuildPartIds),
		"<id>");
	m_DownloadOptions.add_option("",
								 "",
								 "build-part-name",
								 "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
								 "all parts will be downloaded",
								 cxxopts::value(m_BuildPartNames),
								 "<name>");
	m_DownloadOptions.add_option("",
								 "",
								 "clean",
								 "Delete all data in target folder that is not part of the downloaded content",
								 cxxopts::value(m_Clean),
								 "<clean>");
	m_DownloadOptions.add_option("",
								 "",
								 "force",
								 "Force download of all content by ignoring any existing local content",
								 cxxopts::value(m_Force),
								 "<force>");
	m_DownloadOptions.add_option("cache",
								 "",
								 "zen-cache-upload",
								 "Upload data downloaded from remote host to zen cache",
								 cxxopts::value(m_UploadToZenCache),
								 "<uploadtozencache>");
	AddMultipartOptions(m_DownloadOptions);

	AddPartialBlockRequestOptions(m_DownloadOptions);

	m_DownloadOptions.add_option(
		"",
		"",
		"download-spec-path",
		"Path to a text file with one line of <local path> per file to include or a structured .json file describing what to download.",
		cxxopts::value(m_DownloadSpecPath),
		"<downloadspecpath>");

	m_DownloadOptions
		.add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>");
	m_DownloadOptions.add_option("",
								 "",
								 "enable-scavenge",
								 "Enable scavenging of data from previouse download locations",
								 cxxopts::value(m_EnableScavenging),
								 "<scavenge>");
	m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"});
	m_DownloadOptions.positional_help("local-path build-id build-part-name");

	m_DownloadOptions.add_option("",
								 "",
								 "allow-file-clone",
								 "Enable use of block reference counting when copying files",
								 cxxopts::value(m_AllowFileClone),
								 "<allowclone>");

	// ls
	AddSystemOptions(m_LsOptions);
	AddCloudOptions(m_LsOptions);
	AddFileOptions(m_LsOptions);
	AddOutputOptions(m_LsOptions);
	AddCacheOptions(m_LsOptions);
	AddZenFolderOptions(m_LsOptions);
	AddWorkerOptions(m_LsOptions);
	AddWildcardOptions(m_LsOptions);

	m_LsOptions.add_options()("h,help", "Print help");
	m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
	m_LsOptions.add_option(
		"",
		"",
		"build-part-id",
		"Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
		cxxopts::value(m_BuildPartIds),
		"<id>");
	m_LsOptions.add_option("",
						   "",
						   "build-part-name",
						   "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
						   "all parts will be downloaded",
						   cxxopts::value(m_BuildPartNames),
						   "<name>");

	m_LsOptions.add_option("",
						   "",
						   "result-path",
						   "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
						   cxxopts::value(m_LsResultPath),
						   "<result-path>");

	m_LsOptions.add_option("",
						   "o",
						   "output-path",
						   "Path to output, extension .json or .cb (compac binary). Default is output to console",
						   cxxopts::value(m_LsResultPath),
						   "<output-path>");

	m_LsOptions.parse_positional({"build-id", "wildcard"});
	m_LsOptions.positional_help("build-id wildcard");

	// diff
	AddOutputOptions(m_DiffOptions);
	AddWorkerOptions(m_DiffOptions);
	AddExcludeFolderOption(m_DiffOptions);
	AddExcludeExtensionsOption(m_DiffOptions);
	AddChunkingCacheOptions(m_DiffOptions);
	m_DiffOptions.add_options()("h,help", "Print help");
	m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
	m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>");
	m_DiffOptions.add_option("",
							 "",
							 "only-chunked",
							 "Skip files from diff summation that are not processed with chunking",
							 cxxopts::value(m_OnlyChunked),
							 "<only-chunked>");
	m_DiffOptions.parse_positional({"local-path", "compare-path"});
	m_DiffOptions.positional_help("local-path compare-path");

	// test
	AddSystemOptions(m_TestOptions);
	AddCloudOptions(m_TestOptions);
	AddFileOptions(m_TestOptions);
	AddOutputOptions(m_TestOptions);
	AddCacheOptions(m_TestOptions);
	AddWorkerOptions(m_TestOptions);
	m_TestOptions.add_options()("h,help", "Print help");
	m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
	AddMultipartOptions(m_TestOptions);
	AddPartialBlockRequestOptions(m_TestOptions);
	AddWildcardOptions(m_TestOptions);
	AddAppendNewContentOptions(m_TestOptions);
	AddChunkingCacheOptions(m_TestOptions);

	m_TestOptions.add_option("",
							 "",
							 "enable-scavenge",
							 "Enable scavenging of data from previouse download locations",
							 cxxopts::value(m_EnableScavenging),
							 "<scavenge>");
	m_TestOptions.add_option("",
							 "",
							 "allow-file-clone",
							 "Enable use of block reference counting when copying files",
							 cxxopts::value(m_AllowFileClone),
							 "<allowclone>");
	m_TestOptions.parse_positional({"local-path"});
	m_TestOptions.positional_help("local-path");

	// fetch-blob
	AddSystemOptions(m_FetchBlobOptions);
	AddCloudOptions(m_FetchBlobOptions);
	AddFileOptions(m_FetchBlobOptions);
	AddOutputOptions(m_FetchBlobOptions);
	AddCacheOptions(m_FetchBlobOptions);
	AddZenFolderOptions(m_FetchBlobOptions);
	m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
	m_FetchBlobOptions
		.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>");
	m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"});
	m_FetchBlobOptions.positional_help("build-id blob-hash");

	// prime-cache
	AddSystemOptions(m_PrimeCacheOptions);
	AddCloudOptions(m_PrimeCacheOptions);
	AddFileOptions(m_PrimeCacheOptions);
	AddOutputOptions(m_PrimeCacheOptions);
	AddCacheOptions(m_PrimeCacheOptions);
	AddWorkerOptions(m_PrimeCacheOptions);
	AddZenFolderOptions(m_PrimeCacheOptions);
	m_PrimeCacheOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
	m_PrimeCacheOptions.add_option(
		"",
		"",
		"build-part-id",
		"Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
		cxxopts::value(m_BuildPartIds),
		"<id>");
	m_PrimeCacheOptions.add_option("",
								   "",
								   "build-part-name",
								   "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
								   "all parts will be downloaded",
								   cxxopts::value(m_BuildPartNames),
								   "<name>");

	m_PrimeCacheOptions.add_option("",
								   "",
								   "force",
								   "Force download of all blobs by ignoring any existing blobs in cache",
								   cxxopts::value(m_Force),
								   "<force>");

	m_PrimeCacheOptions.parse_positional({"build-id"});
	m_PrimeCacheOptions.positional_help("build-id");

	auto AddZenProcessOptions = [this](cxxopts::Options& Ops) {
		Ops.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
	};
	AddZenProcessOptions(m_PauseOptions);
	m_PauseOptions.parse_positional({"process-id"});
	m_PauseOptions.positional_help("process-id");

	AddZenProcessOptions(m_ResumeOptions);
	m_ResumeOptions.parse_positional({"process-id"});
	m_ResumeOptions.positional_help("process-id");

	AddZenProcessOptions(m_AbortOptions);
	m_AbortOptions.parse_positional({"process-id"});
	m_AbortOptions.positional_help("process-id");

	// validate-part
	AddSystemOptions(m_ValidateBuildPartOptions);
	AddCloudOptions(m_ValidateBuildPartOptions);
	AddFileOptions(m_ValidateBuildPartOptions);
	AddOutputOptions(m_ValidateBuildPartOptions);
	AddWorkerOptions(m_ValidateBuildPartOptions);
	AddZenFolderOptions(m_ValidateBuildPartOptions);
	m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
	m_ValidateBuildPartOptions.add_option("",
										  "",
										  "build-part-id",
										  "Build part Id, if not given it will be auto generated",
										  cxxopts::value(m_BuildPartId),
										  "<id>");
	m_ValidateBuildPartOptions.add_option(
		"",
		"",
		"build-part-name",
		"Name of the build part, if not given it will be be named after the directory name at end of local-path",
		cxxopts::value(m_BuildPartName),
		"<name>");
	m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"});
	m_ValidateBuildPartOptions.positional_help("build-id build-part-id");

	// multi-test-download
	AddSystemOptions(m_MultiTestDownloadOptions);
	AddCloudOptions(m_MultiTestDownloadOptions);
	AddFileOptions(m_MultiTestDownloadOptions);
	AddOutputOptions(m_MultiTestDownloadOptions);
	AddCacheOptions(m_MultiTestDownloadOptions);
	AddWorkerOptions(m_MultiTestDownloadOptions);
	m_MultiTestDownloadOptions
		.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
	m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>");
	m_MultiTestDownloadOptions.add_option("",
										  "",
										  "enable-scavenge",
										  "Enable scavenging of data from previouse download locations",
										  cxxopts::value(m_EnableScavenging),
										  "<scavenge>");
	m_MultiTestDownloadOptions.add_option("",
										  "",
										  "allow-file-clone",
										  "Enable use of block reference counting when copying files",
										  cxxopts::value(m_AllowFileClone),
										  "<allowclone>");
	m_MultiTestDownloadOptions.parse_positional({"local-path"});
	m_MultiTestDownloadOptions.positional_help("local-path");
}

BuildsCommand::~BuildsCommand() = default;

void
BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
	using namespace builds_impl;
	ZEN_UNUSED(GlobalOptions);

	signal(SIGINT, SignalCallbackHandler);
#if ZEN_PLATFORM_WINDOWS
	signal(SIGBREAK, SignalCallbackHandler);
#endif	// ZEN_PLATFORM_WINDOWS

	using namespace std::literals;

	std::vector<char*> SubCommandArguments;
	cxxopts::Options*  SubOption			 = nullptr;
	int				   ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments);
	if (!ParseOptions(ParentCommandArgCount, argv))
	{
		return;
	}

	if (SubOption == nullptr)
	{
		throw OptionParseException("'verb' option is required", m_Options.help());
	}

	if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
	{
		return;
	}

	auto ParseSystemOptions = [&]() {
		if (m_SystemRootDir.empty())
		{
			m_SystemRootDir = PickDefaultSystemRootDirectory();
		}
		MakeSafeAbsolutePathInPlace(m_SystemRootDir);
	};
	ParseSystemOptions();

	auto ParseStorageOptions = [&](bool RequireNamespace, bool RequireBucket) {
		if (!m_Url.empty())
		{
			if (!m_Host.empty())
			{
				throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOption->help());
			}
			if (!m_Bucket.empty())
			{
				throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url),
										   SubOption->help());
			}
			if (!m_BuildId.empty())
			{
				throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", m_BuildId, m_Url),
										   SubOption->help());
			}
			if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId))
			{
				throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format",
										   SubOption->help());
			}
		}

		if (!m_OverrideHost.empty() || !m_Host.empty())
		{
			if (!m_StoragePath.empty())
			{
				throw OptionParseException(
					fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath),
					SubOption->help());
			}
			if (RequireNamespace && m_Namespace.empty())
			{
				throw OptionParseException("'--namespace' is required", SubOption->help());
			}
			if (RequireBucket && m_Bucket.empty())
			{
				throw OptionParseException("'--bucket' is required", SubOption->help());
			}
		}
		else if (m_StoragePath.empty())
		{
			throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help());
		}
		MakeSafeAbsolutePathInPlace(m_StoragePath);
	};

	auto ParseOutputOptions = [&]() {
		if (m_Verbose && m_Quiet)
		{
			throw OptionParseException("'--verbose' conflicts with '--quiet'", SubOption->help());
		}
		if (m_LogProgress && m_PlainProgress)
		{
			throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", SubOption->help());
		}
		if (m_LogProgress && m_Quiet)
		{
			throw OptionParseException("'--quiet' conflicts with '--log-progress'", SubOption->help());
		}
		if (m_PlainProgress && m_Quiet)
		{
			throw OptionParseException("'--quiet' conflicts with '--plain-progress'", SubOption->help());
		}
		IsVerbose = m_Verbose;
		IsQuiet	  = m_Quiet;
		if (m_LogProgress)
		{
			ProgressMode = ProgressBar::Mode::Log;
		}
		else if (m_PlainProgress)
		{
			ProgressMode = ProgressBar::Mode::Plain;
		}
		else if (m_Verbose)
		{
			ProgressMode = ProgressBar::Mode::Plain;
		}
		else if (IsQuiet)
		{
			ProgressMode = ProgressBar::Mode::Quiet;
		}
		else
		{
			ProgressMode = ProgressBar::Mode::Pretty;
		}
	};
	ParseOutputOptions();

	if (m_BoostWorkers)
	{
		m_BoostWorkerCount	= true;
		m_BoostWorkerMemory = true;
	}

	std::unique_ptr<AuthMgr> Auth;

	std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));

	auto CreateBuildStorage = [&](BuildStorageBase::Statistics&	 StorageStats,
								  BuildStorageCache::Statistics& StorageCacheStats,
								  const std::filesystem::path&	 TempPath,
								  bool							 RequireNamespace,
								  bool							 RequireBucket,
								  bool							 BoostCacheBackgroundWorkerPool) -> StorageInstance {
		ParseStorageOptions(RequireNamespace, RequireBucket);

		HttpClientSettings ClientSettings{
			.LogCategory				 = "httpbuildsclient",
			.AssumeHttp2				 = m_AssumeHttp2,
			.AllowResume				 = true,
			.RetryCount					 = 2,
			.Verbose					 = m_VerboseHttp,
			.MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)};

		std::string StorageDescription;
		std::string CacheDescription;

		StorageInstance Result;

		if (!m_Host.empty() || !m_OverrideHost.empty())
		{
			m_AuthOptions.ParseOptions(*SubOption,
									   m_SystemRootDir,
									   ClientSettings,
									   m_Host.empty() ? m_OverrideHost : m_Host,
									   Auth,
									   IsQuiet,
									   /*Hidden*/ false,
									   m_Verbose);

			BuildStorageResolveResult ResolveRes =
				ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose);
			if (!ResolveRes.Cloud.Address.empty())
			{
				ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
				Result.BuildStorageHttp =
					std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); });

				Result.BuildStorage		= CreateJupiterBuildStorage(Log(),
																*Result.BuildStorageHttp,
																StorageStats,
																m_Namespace,
																m_Bucket,
																m_AllowRedirect,
																TempPath / "storage");
				Result.BuildStorageHost = ResolveRes.Cloud;

				uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0;

				StorageDescription =
					fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}",
								ResolveRes.Cloud.Name,
								(ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
								Result.BuildStorageHttp->GetSessionId(),
								m_Namespace,
								m_Bucket,
								NiceLatencyNs(HostLatencyNs));

				if (!ResolveRes.Cache.Address.empty())
				{
					Result.CacheHttp = std::make_unique<HttpClient>(
						ResolveRes.Cache.Address,
						HttpClientSettings{
							.LogCategory				 = "httpcacheclient",
							.ConnectTimeout				 = std::chrono::milliseconds{3000},
							.Timeout					 = std::chrono::milliseconds{30000},
							.AssumeHttp2				 = ResolveRes.Cache.AssumeHttp2,
							.AllowResume				 = true,
							.RetryCount					 = 0,
							.Verbose					 = m_VerboseHttp,
							.MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
						[]() { return AbortFlag.load(); });
					Result.CacheStorage =
						CreateZenBuildStorageCache(*Result.CacheHttp,
												   StorageCacheStats,
												   m_Namespace,
												   m_Bucket,
												   TempPath / "zencache",
												   BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
																				  : GetTinyWorkerPool(EWorkloadType::Background));
					Result.CacheHost = ResolveRes.Cache;

					uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0;

					CacheDescription =
						fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}",
									ResolveRes.Cache.Name,
									(ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address),
									Result.CacheHttp->GetSessionId(),
									NiceLatencyNs(CacheLatencyNs));

					if (!m_Namespace.empty())
					{
						CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
					}
					if (!m_Bucket.empty())
					{
						CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
					}
				}
			}
		}
		else if (!m_StoragePath.empty())
		{
			StorageDescription	= fmt::format("folder {}", m_StoragePath);
			Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);

			Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address	  = m_StoragePath.generic_string(),
																	  .Name		  = "Disk",
																	  .LatencySec = 1.0 / 100000,  // 1 us
																	  .Caps		  = {.MaxRangeCountPerRequest = 2048u}};

			if (!m_ZenCacheHost.empty())
			{
				ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp);

				if (TestResult.Success)
				{
					Result.CacheHttp = std::make_unique<HttpClient>(
						m_ZenCacheHost,
						HttpClientSettings{
							.LogCategory				 = "httpcacheclient",
							.ConnectTimeout				 = std::chrono::milliseconds{3000},
							.Timeout					 = std::chrono::milliseconds{30000},
							.AssumeHttp2				 = m_AssumeHttp2,
							.AllowResume				 = true,
							.RetryCount					 = 0,
							.Verbose					 = m_VerboseHttp,
							.MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
						[]() { return AbortFlag.load(); });

					Result.CacheStorage =
						CreateZenBuildStorageCache(*Result.CacheHttp,
												   StorageCacheStats,
												   m_Namespace,
												   m_Bucket,
												   TempPath / "zencache",
												   BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
																				  : GetTinyWorkerPool(EWorkloadType::Background));
					Result.CacheHost =
						BuildStorageResolveResult::Host{.Address	 = m_ZenCacheHost,
														.Name		 = m_ZenCacheHost,
														.AssumeHttp2 = m_AssumeHttp2,
														.LatencySec	 = TestResult.LatencySeconds,
														.Caps		 = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}};

					CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId());

					if (!m_Namespace.empty())
					{
						CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
					}
					if (!m_Bucket.empty())
					{
						CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
					}
				}
			}
		}
		else
		{
			throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help());
		}

		if (!IsQuiet)
		{
			ZEN_CONSOLE("Remote: {}", StorageDescription);
			if (!Result.CacheHost.Name.empty())
			{
				ZEN_CONSOLE("Cache : {}", CacheDescription);
			}
		}
		return Result;
	};

	auto ParsePath = [&]() {
		if (m_Path.empty())
		{
			throw OptionParseException("'--local-path' is required", SubOption->help());
		}
		MakeSafeAbsolutePathInPlace(m_Path);
	};

	auto ParseFileFilters = [&](std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) {
		auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) {
			ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) {
				if (!Wildcard.empty())
				{
					std::string CleanWildcard(ToLower(Wildcard));
					for (auto It = begin(CleanWildcard); It != end(CleanWildcard); It++)
					{
						if (*It == '\\')
						{
							*It = '/';
						}
					}

					if (CleanWildcard.starts_with("./") || CleanWildcard.starts_with(".\\"))
					{
						CleanWildcard = CleanWildcard.substr(2);
					}

					Output.emplace_back(std::move(CleanWildcard));
				}
				return true;
			});
		};

		SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards);
		SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards);
	};

	auto ParseExcludeFolderAndExtension = [&](std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) {
		auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) {
			ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) {
				if (!Exclusion.empty())
				{
					std::string CleanExclusion(ToLower(Exclusion));
					if (CleanExclusion.length() > 2 && CleanExclusion.front() == '"' && CleanExclusion.back() == '"')
					{
						CleanExclusion = CleanExclusion.substr(1, CleanExclusion.length() - 2);
					}
					Output.emplace_back(std::move(CleanExclusion));
				}
				return true;
			});
		};

		SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders);
		SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions);
	};

	auto ParseDiffPath = [&]() {
		if (m_DiffPath.empty())
		{
			throw OptionParseException("'--compare-path' is required", SubOption->help());
		}
		MakeSafeAbsolutePathInPlace(m_DiffPath);
	};

	auto ParseBlobHash = [&]() -> IoHash {
		if (m_BlobHash.empty())
		{
			throw OptionParseException("'--blob-hash' is required", SubOption->help());
		}

		if (m_BlobHash.length() != IoHash::StringLength)
		{
			throw OptionParseException(
				fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", m_BlobHash, IoHash::StringLength),
				SubOption->help());
		}

		IoHash BlobHash;
		if (!IoHash::TryParse(m_BlobHash, BlobHash))
		{
			throw OptionParseException(fmt::format("'--blob-hash' ('{}') is malformed", m_BlobHash), SubOption->help());
		}

		return BlobHash;
	};

	auto ParseBuildId = [&]() -> Oid {
		if (m_BuildId.length() != Oid::StringLength)
		{
			throw OptionParseException(
				fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildId, Oid::StringLength),
				SubOption->help());
		}
		else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero)
		{
			throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", m_BuildId), SubOption->help());
		}
		else
		{
			return BuildId;
		}
	};

	auto ParseBuildPartId = [&]() -> Oid {
		if (m_BuildPartId.length() != Oid::StringLength)
		{
			throw OptionParseException(
				fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildPartId, Oid::StringLength),
				SubOption->help());
		}
		else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero)
		{
			throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", m_BuildPartId), SubOption->help());
		}
		else
		{
			return BuildPartId;
		}
	};

	auto ParseBuildPartIds = [&]() -> std::vector<Oid> {
		std::vector<Oid> BuildPartIds;
		for (const std::string& BuildPartId : m_BuildPartIds)
		{
			BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId)));
			if (BuildPartIds.back() == Oid::Zero)
			{
				throw OptionParseException(fmt::format("'--build-part-id' ('{}') is malformed", BuildPartId), SubOption->help());
			}
		}
		return BuildPartIds;
	};

	auto ParseBuildPartNames = [&]() -> std::vector<std::string> {
		std::vector<std::string> BuildPartNames;
		for (const std::string& BuildPartName : m_BuildPartNames)
		{
			BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName)));
			if (BuildPartNames.back().empty())
			{
				throw OptionParseException(fmt::format("'--build-part-names' ('{}') is invalid", BuildPartName), SubOption->help());
			}
		}
		return BuildPartNames;
	};

	auto ParseBuildMetadata = [&]() -> CbObject {
		if (m_CreateBuild)
		{
			if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty())
			{
				throw OptionParseException("'--metadata-path' or '--metadata' is required", SubOption->help());
			}
			if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty())
			{
				throw OptionParseException(
					fmt::format("'--metadata-path' ('{}') conflicts with '--metadata' ('{}')", m_BuildMetadataPath, m_BuildMetadata),
					SubOption->help());
			}

			if (!m_BuildMetadataPath.empty())
			{
				MakeSafeAbsolutePathInPlace(m_BuildMetadataPath);
				IoBuffer		 MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten();
				std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
				std::string		 JsonError;
				CbObject		 MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
				if (!JsonError.empty())
				{
					throw std::runtime_error(
						fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError));
				}
				return MetaData;
			}
			if (!m_BuildMetadata.empty())
			{
				CbObjectWriter MetaDataWriter(1024);
				ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) {
					size_t SplitPos = Pair.find('=');
					if (SplitPos == std::string::npos || SplitPos == 0)
					{
						throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair));
					}
					MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
					return true;
				});
				return MetaDataWriter.Save();
			}
		}
		else
		{
			if (!m_BuildMetadataPath.empty())
			{
				throw OptionParseException("'--metadata-path' requires '--create-build'", SubOption->help());
			}
			if (!m_BuildMetadata.empty())
			{
				throw OptionParseException("'--metadata' requires '--create-build'", SubOption->help());
			}
		}
		return {};
	};

	UseSparseFiles = m_UseSparseFiles;

	if (SubOption == &m_ListNamespacesOptions)
	{
		if (!m_ListResultPath.empty())
		{
			if (!IsQuiet)
			{
				LogExecutableVersionAndPid();
			}
		}

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
														? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
														: MakeSafeAbsolutePath(m_ZenFolderPath);
		CreateDirectories(ZenFolderPath);
		auto _ = MakeGuard([ZenFolderPath]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), ZenFolderPath); });

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(ZenFolderPath),
													 /*RequriesNamespace*/ false,
													 /*RequireBucket*/ false,
													 /*BoostCacheBackgroundWorkerPool */ false);

		CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive);
		ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
		if (m_ListResultPath.empty())
		{
			ExtendableStringBuilder<1024> SB;
			CompactBinaryToJson(Response.GetView(), SB);
			ZEN_CONSOLE("{}", SB.ToView());
		}
		else
		{
			std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath);
			if (ToLower(ListResultPath.extension().string()) == ".cbo")
			{
				MemoryView ResponseView = Response.GetView();
				WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
			}
			else
			{
				ExtendableStringBuilder<1024> SB;
				CompactBinaryToJson(Response.GetView(), SB);
				WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
			}
		}
		return;
	}

	if (SubOption == &m_ListOptions)
	{
		MakeSafeAbsolutePathInPlace(m_ListQueryPath);
		MakeSafeAbsolutePathInPlace(m_ListResultPath);

		if (!m_ListResultPath.empty())
		{
			if (!IsQuiet)
			{
				LogExecutableVersionAndPid();
			}
		}
		std::string JsonQuery;
		if (m_ListQueryPath.empty())
		{
			CbObjectWriter QueryWriter;
			QueryWriter.BeginObject("query");
			QueryWriter.EndObject();  // query
			CbObject					QueryObject = QueryWriter.Save();
			ExtendableStringBuilder<64> SB;
			CompactBinaryToJson(QueryObject, SB);
			JsonQuery = SB.ToString();
		}
		else
		{
			if (ToLower(m_ListQueryPath.extension().string()) == ".cbo")
			{
				CbObject					QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath));
				ExtendableStringBuilder<64> SB;
				CompactBinaryToJson(QueryObject, SB);
				JsonQuery = SB.ToString();
			}
			else
			{
				IoBuffer		 MetaDataJson = ReadFile(m_ListQueryPath).Flatten();
				std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
				std::string		 JsonError;
				CbObject		 QueryObject = LoadCompactBinaryFromJson(Json, JsonError)
										   .AsObject();	 // We try to convert it so it is at least reaonably verified in format
				if (!JsonError.empty())
				{
					throw std::runtime_error(
						fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError));
				}
				JsonQuery = std::string(Json);
			}
		}

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		CreateDirectories(m_ZenFolderPath);
		auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); });

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ false,
													 /*BoostCacheBackgroundWorkerPool */ false);

		CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery);
		ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
		if (m_ListResultPath.empty())
		{
			ExtendableStringBuilder<1024> SB;
			CompactBinaryToJson(Response.GetView(), SB);
			ZEN_CONSOLE("{}", SB.ToView());
		}
		else
		{
			if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
			{
				MemoryView ResponseView = Response.GetView();
				WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
			}
			else
			{
				ExtendableStringBuilder<1024> SB;
				CompactBinaryToJson(Response.GetView(), SB);
				WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
			}
		}
		return;
	}

	if (SubOption == &m_ListBlocksOptions)
	{
		MakeSafeAbsolutePathInPlace(m_ListResultPath);

		if (!m_ListResultPath.empty())
		{
			if (!IsQuiet)
			{
				LogExecutableVersionAndPid();
			}
		}

		if (m_ListBlocksMaxCount == 0)
		{
			throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_ListBlocksMaxCount), SubOption->help());
		}

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		CreateDirectories(m_ZenFolderPath);
		auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); });

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ false);

		const Oid BuildId = ParseBuildId();

		CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_ListBlocksMaxCount);
		ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);

		std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size());
		}
		if (m_ListResultPath.empty())
		{
			ExtendableStringBuilder<1024> SB;
			CompactBinaryToJson(Response.GetView(), SB);
			ForEachStrTok(SB.ToView(), '\n', [](std::string_view Row) {
				ZEN_CONSOLE("{}", Row);
				return true;
			});
		}
		else
		{
			if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
			{
				MemoryView ResponseView = Response.GetView();
				WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
			}
			else
			{
				ExtendableStringBuilder<1024> SB;
				CompactBinaryToJson(Response.GetView(), SB);
				WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
			}
		}
		return;
	}
	if (SubOption == &m_UploadOptions)
	{
		if (!IsQuiet)
		{
			LogExecutableVersionAndPid();
		}

		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		ZenState InstanceState;

		ParsePath();

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
		MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);

		CreateDirectories(m_ZenFolderPath);
		auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ false);

		if (m_BuildPartName.empty() && m_ManifestPath.empty())
		{
			m_BuildPartName = m_Path.filename().string();
		}

		const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
		if (m_BuildId.empty())
		{
			m_BuildId = BuildId.ToString();
		}

		Oid BuildPartId;
		if (!m_BuildPartId.empty())
		{
			BuildPartId = ParseBuildPartId();
		}

		CbObject MetaData = ParseBuildMetadata();

		const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath);

		std::vector<std::string> ExcludeFolders	   = DefaultExcludeFolders;
		std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
		ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);

		std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
		std::unique_ptr<ChunkingCache>		ChunkCache		= m_ChunkingCachePath.empty()
																  ? CreateNullChunkingCache()
																  : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u);

		std::vector<std::pair<Oid, std::string>> UploadedParts =
			UploadFolder(*Output,
						 Workers,
						 Storage,
						 BuildId,
						 BuildPartId,
						 m_BuildPartName,
						 m_Path,
						 m_ManifestPath,
						 MetaData,
						 *ChunkController,
						 *ChunkCache,
						 UploadFolderOptions{.TempDir					= TempDir,
											 .FindBlockMaxCount			= m_FindBlockMaxCount,
											 .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
											 .AllowMultiparts			= m_AllowMultiparts,
											 .CreateBuild				= m_CreateBuild,
											 .IgnoreExistingBlocks		= m_Clean,
											 .UploadToZenCache			= m_UploadToZenCache,
											 .ExcludeFolders			= ExcludeFolders,
											 .ExcludeExtensions			= ExcludeExtensions});

		if (!AbortFlag)
		{
			if (m_PostUploadVerify)
			{
				// TODO: Validate all parts
				for (const auto& Part : UploadedParts)
				{
					ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second);
				}
			}
		}

		if (true)
		{
			if (!IsQuiet)
			{
				ZEN_CONSOLE(
					"{}:\n"
					"Read:               {}\n"
					"Write:              {}\n"
					"Requests:           {}\n"
					"Avg Request Time:   {}\n"
					"Avg I/O Time:       {}",
					Storage.BuildStorageHost.Name,
					NiceBytes(StorageStats.TotalBytesRead.load()),
					NiceBytes(StorageStats.TotalBytesWritten.load()),
					StorageStats.TotalRequestCount.load(),
					StorageStats.TotalExecutionTimeUs.load() > 0
						? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
						: 0,
					StorageStats.TotalRequestCount.load() > 0
						? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
						: 0);
			}
		}
		if (AbortFlag)
		{
			throw std::runtime_error("Upload aborted");
		}
	}

	auto ParseAllowPartialBlockRequests = [&]() -> EPartialBlockRequestMode {
		if (m_PrimeCacheOnly)
		{
			return EPartialBlockRequestMode::Off;
		}
		EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
		if (Mode == EPartialBlockRequestMode::Invalid)
		{
			throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
									   SubOption->help());
		}
		return Mode;
	};

	if (SubOption == &m_DownloadOptions)
	{
		if (!IsQuiet)
		{
			LogExecutableVersionAndPid();
		}

		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		ZenState InstanceState;

		ParsePath();

		std::vector<std::string> IncludeWildcards;
		std::vector<std::string> ExcludeWildcards;
		ParseFileFilters(IncludeWildcards, ExcludeWildcards);

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = m_Path / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly);

		const Oid BuildId = ParseBuildId();

		if (m_PostDownloadVerify && m_PrimeCacheOnly)
		{
			throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", SubOption->help());
		}

		if (m_Clean && m_PrimeCacheOnly)
		{
			ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled");
		}

		if (m_Force && m_PrimeCacheOnly)
		{
			ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled");
		}

		if (m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly)
		{
			ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled");
		}

		std::vector<Oid>		 BuildPartIds	= ParseBuildPartIds();
		std::vector<std::string> BuildPartNames = ParseBuildPartNames();

		EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests();

		if (m_AppendNewContent && m_Clean)
		{
			throw OptionParseException("'--append' conflicts with '--clean'", SubOption->help());
		}

		std::vector<std::string> ExcludeFolders	   = DefaultExcludeFolders;
		std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
		ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);

		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId,
					   BuildPartIds,
					   BuildPartNames,
					   m_DownloadSpecPath,
					   m_Path,
					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = m_ZenFolderPath,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = m_Clean,
									   .PostDownloadVerify			   = m_PostDownloadVerify,
									   .PrimeCacheOnly				   = m_PrimeCacheOnly,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
									   .EnableTargetFolderScavenging   = !m_Force,
									   .AllowFileClone				   = m_AllowFileClone,
									   .IncludeWildcards			   = IncludeWildcards,
									   .ExcludeWildcards			   = ExcludeWildcards,
									   .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory),
									   .PopulateCache			   = m_UploadToZenCache,
									   .AppendNewContent		   = m_AppendNewContent,
									   .ExcludeFolders			   = ExcludeFolders});

		if (AbortFlag)
		{
			throw std::runtime_error("Download aborted");
		}
	}

	if (SubOption == &m_LsOptions)
	{
		if (!m_LsResultPath.empty())
		{
			if (!IsQuiet)
			{
				LogExecutableVersionAndPid();
			}
		}

		ZenState InstanceState;

		std::vector<std::string> IncludeWildcards;
		std::vector<std::string> ExcludeWildcards;
		ParseFileFilters(IncludeWildcards, ExcludeWildcards);

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = m_Path / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ false);

		const Oid BuildId = ParseBuildId();

		std::vector<Oid>		 BuildPartIds	= ParseBuildPartIds();
		std::vector<std::string> BuildPartNames = ParseBuildPartNames();

		std::unique_ptr<CbObjectWriter> StructuredOutput;
		if (!m_LsResultPath.empty())
		{
			MakeSafeAbsolutePathInPlace(m_LsResultPath);
			StructuredOutput = std::make_unique<CbObjectWriter>();
		}

		ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());

		if (StructuredOutput)
		{
			CbObject Response = StructuredOutput->Save();
			if (ToLower(m_LsResultPath.extension().string()) == ".cbo")
			{
				MemoryView ResponseView = Response.GetView();
				WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
			}
			else
			{
				ExtendableStringBuilder<1024> SB;
				CompactBinaryToJson(Response.GetView(), SB);
				WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
			}
		}

		if (AbortFlag)
		{
			throw std::runtime_error("List build aborted");
		}
	}

	if (SubOption == &m_DiffOptions)
	{
		if (!IsQuiet)
		{
			LogExecutableVersionAndPid();
		}

		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		ParsePath();
		ParseDiffPath();

		MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);

		std::vector<std::string> ExcludeFolders	   = DefaultExcludeFolders;
		std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
		ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);

		StandardChunkingControllerSettings	ChunkingSettings;
		std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings);
		std::unique_ptr<ChunkingCache>		ChunkCache		= m_ChunkingCachePath.empty()
																  ? CreateNullChunkingCache()
																  : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u);

		if (m_OnlyChunked)
		{
			ExcludeExtensions.insert(ExcludeExtensions.end(),
									 ChunkingSettings.SplitOnlyExtensions.begin(),
									 ChunkingSettings.SplitOnlyExtensions.end());
			ExcludeExtensions.insert(ExcludeExtensions.end(),
									 ChunkingSettings.SplitAndCompressExtensions.begin(),
									 ChunkingSettings.SplitAndCompressExtensions.end());
		}

		DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions);
		if (AbortFlag)
		{
			throw std::runtime_error("Diff folders aborted");
		}
	}

	if (SubOption == &m_PrimeCacheOptions)
	{
		if (!IsQuiet)
		{
			LogExecutableVersionAndPid();
		}

		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		CreateDirectories(m_ZenFolderPath);
		auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ true);

		const Oid BuildId = ParseBuildId();

		std::vector<Oid>		 BuildPartIds	= ParseBuildPartIds();
		std::vector<std::string> BuildPartNames = ParseBuildPartNames();

		std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;

		CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);

		std::vector<std::pair<Oid, std::string>> AllBuildParts =
			ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);

		std::vector<Oid> AllBuildPartIds;
		AllBuildPartIds.reserve(AllBuildParts.size());
		for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
		{
			AllBuildPartIds.push_back(BuildPart.first);
		}

		ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache");

		BuildsOperationPrimeCache PrimeOp(*Output,
										  Storage,
										  AbortFlag,
										  PauseFlag,
										  Workers.GetNetworkPool(),
										  BuildId,
										  AllBuildPartIds,
										  BuildsOperationPrimeCache::Options{.IsQuiet			  = IsQuiet,
																			 .IsVerbose			  = IsVerbose,
																			 .ZenFolderPath		  = m_ZenFolderPath,
																			 .LargeAttachmentSize = PreferredMultipartChunkSize * 4u,
																			 .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
																			 .ForceUpload				  = m_Force},
										  StorageCacheStats);
		PrimeOp.Execute();

		if (!IsQuiet)
		{
			if (Storage.CacheStorage)
			{
				ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
							StorageCacheStats.PutBlobCount.load(),
							NiceBytes(StorageCacheStats.PutBlobByteCount),
							Storage.CacheHost.Name);
			}
		}

		return;
	}

	if (SubOption == &m_FetchBlobOptions)
	{
		if (!IsQuiet)
		{
			LogExecutableVersionAndPid();
		}

		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		CreateDirectories(m_ZenFolderPath);
		auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ false);

		IoHash BlobHash = ParseBlobHash();

		const Oid BuildId = Oid::FromHexString(m_BuildId);

		uint64_t CompressedSize;
		uint64_t DecompressedSize;
		ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
		if (AbortFlag)
		{
			throw std::runtime_error("Fetch blob aborted");
		}
		if (!IsQuiet)
		{
			ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
						BlobHash,
						CompressedSize,
						DecompressedSize);
		}
		return;
	}

	if (SubOption == &m_ValidateBuildPartOptions)
	{
		if (!IsQuiet)
		{
			LogExecutableVersionAndPid();
		}

		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		ZenState InstanceState;

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		CreateDirectories(m_ZenFolderPath);
		auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ false);

		Oid BuildId = ParseBuildId();

		if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
		{
			throw OptionParseException(
				fmt::format("'--build-part-id' ('{}') conflicts with '--build-part-name' ('{}')", m_BuildPartId, m_BuildPartName),
				SubOption->help());
		}

		const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();

		ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);

		if (AbortFlag)
		{
			throw std::runtime_error("Validate build part failed");
		}
	}

	if (SubOption == &m_MultiTestDownloadOptions)
	{
		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
		CreateDirectories(m_SystemRootDir);
		CleanDirectory(m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
		auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });

		ParsePath();

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = m_Path / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);

		EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests();

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ false);

		Stopwatch Timer;
		for (const std::string& BuildIdString : m_BuildIds)
		{
			Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString));
			if (BuildId == Oid::Zero)
			{
				throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), SubOption->help());
			}
			DownloadFolder(*Output,
						   Workers,
						   Storage,
						   StorageCacheStats,
						   BuildId,
						   /*BuildPartIds,*/ {},
						   /*BuildPartNames*/ {},
						   /*ManifestPath*/ {},
						   m_Path,
						   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
										   .ZenFolderPath				   = m_ZenFolderPath,
										   .AllowMultiparts				   = m_AllowMultiparts,
										   .PartialBlockRequestMode		   = PartialBlockRequestMode,
										   .CleanTargetFolder			   = BuildIdString == m_BuildIds.front(),
										   .PostDownloadVerify			   = true,
										   .PrimeCacheOnly				   = false,
										   .EnableOtherDownloadsScavenging = m_EnableScavenging,
										   .EnableTargetFolderScavenging   = false,
										   .AllowFileClone				   = m_AllowFileClone});
			if (AbortFlag)
			{
				throw std::runtime_error("Multitest aborted");
			}
			if (!IsQuiet)
			{
				ZEN_CONSOLE("\n");
			}
		}
		if (!IsQuiet)
		{
			ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
		}
	}

	auto ParseZenProcessId = [&]() {
		if (m_ZenProcessId == -1)
		{
			const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath();
			ProcessHandle				RunningProcess;
			std::error_code				Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false);
			if (Ec)
			{
				throw std::runtime_error(
					fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message()));
			}
			if (!RunningProcess.IsValid())
			{
				throw std::runtime_error(
					fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath));
			}
			m_ZenProcessId = RunningProcess.Pid();
		}
	};

	if (SubOption == &m_PauseOptions)
	{
		ParseZenProcessId();
		ZenState RunningState(m_ZenProcessId);
		RunningState.StateData().Pause.store(true);
	}

	if (SubOption == &m_ResumeOptions)
	{
		ParseZenProcessId();
		ZenState RunningState(m_ZenProcessId);
		RunningState.StateData().Pause.store(false);
	}

	if (SubOption == &m_AbortOptions)
	{
		ParseZenProcessId();
		ZenState RunningState(m_ZenProcessId);
		RunningState.StateData().Abort.store(true);
	}

	if (SubOption == &m_TestOptions)
	{
		TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
		if (!IsQuiet)
		{
			ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
		}

		m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
		CreateDirectories(m_SystemRootDir);
		CleanDirectory(m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
		auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });

		ParsePath();

		if (m_OverrideHost.empty() && m_StoragePath.empty())
		{
			m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
			CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_StoragePath);
			CreateDirectories(m_StoragePath);
			m_StoragePath = m_StoragePath.generic_string();
		}

		auto __ = MakeGuard([&]() {
			if (m_OverrideHost.empty() && m_StoragePath.empty())
			{
				DeleteDirectories(m_StoragePath);
			}
		});

		EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests();

		BuildStorageBase::Statistics  StorageStats;
		BuildStorageCache::Statistics StorageCacheStats;

		const std::filesystem::path DownloadPath  = m_Path.parent_path() / (m_BuildPartName + "_test");
		const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
		const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3");

		CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
		CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
		CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);

		auto ___ = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() {
			CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
			CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
			CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
		});

		if (m_ZenFolderPath.empty())
		{
			m_ZenFolderPath = m_Path / ZenFolderName;
		}
		MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
		MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);

		StorageInstance Storage = CreateBuildStorage(StorageStats,
													 StorageCacheStats,
													 ZenTempFolderPath(m_ZenFolderPath),
													 /*RequriesNamespace*/ true,
													 /*RequireBucket*/ true,
													 /*BoostCacheBackgroundWorkerPool */ false);

		m_BuildId		= Oid::NewOid().ToString();
		m_BuildPartName = m_Path.filename().string();
		m_BuildPartId	= Oid::NewOid().ToString();
		m_CreateBuild	= true;

		const Oid BuildId	  = Oid::FromHexString(m_BuildId);
		const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);

		auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
			CbObjectWriter BuildMetaDataWriter;
			{
				const uint32_t CL = BuildId.OidBits[2];
				BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL));
				BuildMetaDataWriter.AddString("branch", "ZenTestBuild");
				BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild");
				BuildMetaDataWriter.AddString("platform", "Windows");
				BuildMetaDataWriter.AddString("project", "Test");
				BuildMetaDataWriter.AddInteger("changelist", CL);
				BuildMetaDataWriter.AddString("buildType", "test-folder");
			}
			return BuildMetaDataWriter.Save();
		};
		CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId));
		{
			ExtendableStringBuilder<256> SB;
			CompactBinaryToJson(MetaData, SB);
			ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
		}

		const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);

		std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
		std::unique_ptr<ChunkingCache>		ChunkCache		= m_ChunkingCachePath.empty()
																  ? CreateNullChunkingCache()
																  : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u);

		UploadFolder(*Output,
					 Workers,
					 Storage,
					 BuildId,
					 BuildPartId,
					 m_BuildPartName,
					 m_Path,
					 {},
					 MetaData,
					 *ChunkController,
					 *ChunkCache,
					 UploadFolderOptions{.TempDir					= UploadTempDir,
										 .FindBlockMaxCount			= m_FindBlockMaxCount,
										 .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
										 .AllowMultiparts			= m_AllowMultiparts,
										 .CreateBuild				= true,
										 .IgnoreExistingBlocks		= false,
										 .UploadToZenCache			= m_UploadToZenCache});

		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Upload build)");
		}

		{
			ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path);

			UploadFolder(*Output,
						 Workers,
						 Storage,
						 Oid::NewOid(),
						 Oid::NewOid(),
						 m_BuildPartName,
						 m_Path,
						 {},
						 MetaData,
						 *ChunkController,
						 *ChunkCache,
						 UploadFolderOptions{.TempDir					= UploadTempDir,
											 .FindBlockMaxCount			= m_FindBlockMaxCount,
											 .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
											 .AllowMultiparts			= m_AllowMultiparts,
											 .CreateBuild				= true,
											 .IgnoreExistingBlocks		= false,
											 .UploadToZenCache			= m_UploadToZenCache});

			if (AbortFlag)
			{
				throw std::runtime_error("Test aborted. (Upload again, chunking is cached)");
			}
		}

		ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);

		if (!m_IncludeWildcard.empty() || !m_ExcludeWildcard.empty())
		{
			auto __ = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); });

			ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);

			std::vector<std::string> IncludeWildcards;
			std::vector<std::string> ExcludeWildcards;
			ParseFileFilters(IncludeWildcards, ExcludeWildcards);

			DownloadFolder(*Output,
						   Workers,
						   Storage,
						   StorageCacheStats,
						   BuildId,
						   {BuildPartId},
						   /*BuildPartNames*/ {},
						   /*ManifestPath*/ {},
						   DownloadPath,
						   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
										   .ZenFolderPath				   = DownloadPath / ZenFolderName,
										   .AllowMultiparts				   = m_AllowMultiparts,
										   .PartialBlockRequestMode		   = PartialBlockRequestMode,
										   .CleanTargetFolder			   = true,
										   .PostDownloadVerify			   = true,
										   .PrimeCacheOnly				   = false,
										   .EnableOtherDownloadsScavenging = m_EnableScavenging,
										   .EnableTargetFolderScavenging   = false,
										   .AllowFileClone				   = m_AllowFileClone,
										   .IncludeWildcards			   = IncludeWildcards,
										   .ExcludeWildcards			   = ExcludeWildcards,
										   .AppendNewContent			   = false});
			if (AbortFlag)
			{
				throw std::runtime_error("Test aborted. (Download build)");
			}

			ZEN_CONSOLE("\nDownload Filtered Out Remaining of Build {}, Part {} ({}) to '{}'",
						BuildId,
						BuildPartId,
						m_BuildPartName,
						DownloadPath);
			DownloadFolder(*Output,
						   Workers,
						   Storage,
						   StorageCacheStats,
						   BuildId,
						   {BuildPartId},
						   /*BuildPartNames*/ {},
						   /*ManifestPath*/ {},
						   DownloadPath,
						   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
										   .ZenFolderPath				   = DownloadPath / ZenFolderName,
										   .AllowMultiparts				   = m_AllowMultiparts,
										   .PartialBlockRequestMode		   = PartialBlockRequestMode,
										   .CleanTargetFolder			   = true,
										   .PostDownloadVerify			   = true,
										   .PrimeCacheOnly				   = false,
										   .EnableOtherDownloadsScavenging = m_EnableScavenging,
										   .EnableTargetFolderScavenging   = true,
										   .AllowFileClone				   = m_AllowFileClone,
										   .IncludeWildcards			   = ExcludeWildcards,
										   .ExcludeWildcards			   = IncludeWildcards,
										   .AppendNewContent			   = true});
			if (AbortFlag)
			{
				throw std::runtime_error("Test aborted. (Download build)");
			}

			ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
			DownloadFolder(*Output,
						   Workers,
						   Storage,
						   StorageCacheStats,
						   BuildId,
						   {BuildPartId},
						   /*BuildPartNames*/ {},
						   /*ManifestPath*/ {},
						   DownloadPath,
						   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
										   .ZenFolderPath				   = DownloadPath / ZenFolderName,
										   .AllowMultiparts				   = m_AllowMultiparts,
										   .PartialBlockRequestMode		   = PartialBlockRequestMode,
										   .CleanTargetFolder			   = false,
										   .PostDownloadVerify			   = true,
										   .PrimeCacheOnly				   = false,
										   .EnableOtherDownloadsScavenging = m_EnableScavenging,
										   .EnableTargetFolderScavenging   = true,
										   .AllowFileClone				   = m_AllowFileClone,
										   .IncludeWildcards			   = {},
										   .ExcludeWildcards			   = {},
										   .AppendNewContent			   = false});
			if (AbortFlag)
			{
				throw std::runtime_error("Test aborted. (Download build)");
			}
		}

		ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId,
					   {BuildPartId},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath,

					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = true,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = false,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Download build)");
		}

		ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId,
					   {BuildPartId},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath,

					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = false,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = true,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Re-download identical target)");
		}

		auto ScrambleDir = [&Workers](const std::filesystem::path& Path) {
			ZEN_CONSOLE("\nScrambling '{}'", Path);
			Stopwatch		 Timer;
			DirectoryContent DownloadContent;
			GetDirectoryContent(
				Path,
				DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
				DownloadContent);
			auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool {
				std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string();
				for (const std::string& ExcludeFolder : ExcludeFolders)
				{
					if (RelativePath.starts_with(ExcludeFolder))
					{
						if (RelativePath.length() == ExcludeFolder.length())
						{
							return false;
						}
						else if (RelativePath[ExcludeFolder.length()] == '/')
						{
							return false;
						}
					}
				}
				return true;
			};

			ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);

			uint32_t Randomizer = 0;
			auto	 FileSizeIt = DownloadContent.FileSizes.begin();
			for (const std::filesystem::path& FilePath : DownloadContent.Files)
			{
				if (IsAcceptedFolder(FilePath))
				{
					uint32_t Case = (Randomizer++) % 7;
					switch (Case)
					{
						case 0:
							{
								uint64_t SourceSize = *FileSizeIt;
								if (SourceSize > 256)
								{
									Work.ScheduleWork(
										Workers.GetIOWorkerPool(),
										[SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
											if (!AbortFlag)
											{
												bool WasReadOnly = SetFileReadOnly(FilePath, false);
												{
													BasicFile Source(FilePath, BasicFile::Mode::kWrite);
													uint64_t  RangeSize = Min(SourceSize / 3, 512u * 1024u);
													IoBuffer  TempBuffer1(RangeSize);
													IoBuffer  TempBuffer2(RangeSize);
													IoBuffer  TempBuffer3(RangeSize);
													Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0);
													Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2);
													Source.Read(TempBuffer3.GetMutableView().GetData(), RangeSize, SourceSize - RangeSize);
													Source.Write(TempBuffer1, SourceSize / 2);
													Source.Write(TempBuffer2, SourceSize - RangeSize);
													Source.Write(TempBuffer3, SourceSize - 0);
												}
												if (WasReadOnly)
												{
													SetFileReadOnly(FilePath, true);
												}
											}
										});
								}
							}
							break;
						case 1:
							{
								(void)SetFileReadOnly(FilePath, false);
								(void)RemoveFile(FilePath);
							}
							break;
						default:
							break;
					}
				}
				FileSizeIt++;
			}
			Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
				ZEN_UNUSED(IsAborted, IsPaused);
				ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
			});
			ZEN_ASSERT(!AbortFlag.load());
			ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
		};

		ScrambleDir(DownloadPath);
		ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId,
					   {BuildPartId},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath,

					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = false,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = true,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Re-download scrambled target)");
		}

		ScrambleDir(DownloadPath);

		Oid BuildId2	 = Oid::NewOid();
		Oid BuildPartId2 = Oid::NewOid();

		CbObject MetaData2 = MakeMetaData(BuildId2);
		{
			ExtendableStringBuilder<256> SB;
			CompactBinaryToJson(MetaData, SB);
			ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
		}

		UploadFolder(*Output,
					 Workers,
					 Storage,
					 BuildId2,
					 BuildPartId2,
					 m_BuildPartName,
					 DownloadPath,
					 {},
					 MetaData2,
					 *ChunkController,
					 *ChunkCache,
					 UploadFolderOptions{.TempDir					= UploadTempDir,
										 .FindBlockMaxCount			= m_FindBlockMaxCount,
										 .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
										 .AllowMultiparts			= m_AllowMultiparts,
										 .CreateBuild				= true,
										 .IgnoreExistingBlocks		= false,
										 .UploadToZenCache			= m_UploadToZenCache});

		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Upload scrambled)");
		}

		ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);

		ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId,
					   {BuildPartId},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath,

					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = false,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = true,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Download original)");
		}

		ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId2,
					   {BuildPartId2},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath,
					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = false,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = true,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Download scrambled)");
		}

		ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId2,
					   {BuildPartId2},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath,
					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = false,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = true,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Re-download scrambled)");
		}

		ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId,
					   {BuildPartId},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath2,
					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath2 / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = false,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = true,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Download original)");
		}

		ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3);
		DownloadFolder(*Output,
					   Workers,
					   Storage,
					   StorageCacheStats,
					   BuildId,
					   {BuildPartId},
					   /*BuildPartNames*/ {},
					   /*ManifestPath*/ {},
					   DownloadPath3,
					   DownloadOptions{.SystemRootDir				   = m_SystemRootDir,
									   .ZenFolderPath				   = DownloadPath3 / ZenFolderName,
									   .AllowMultiparts				   = m_AllowMultiparts,
									   .PartialBlockRequestMode		   = PartialBlockRequestMode,
									   .CleanTargetFolder			   = false,
									   .PostDownloadVerify			   = true,
									   .PrimeCacheOnly				   = false,
									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
									   .EnableTargetFolderScavenging   = true,
									   .AllowFileClone				   = m_AllowFileClone});
		if (AbortFlag)
		{
			throw std::runtime_error("Test aborted. (Download original)");
		}
	}
}

}  // namespace zen