Merge branch 'main' into de/zen-service-command

164 files changed, 18379 insertions, 5470 deletions

diff --git a/src/transports/transport-sdk/include/transportplugin.h b/src/transports/transport-sdk/include/transportplugin.h
index 4347868e6..a78a758bc 100644
--- a/src/transports/transport-sdk/include/transportplugin.h
+++ b/src/transports/transport-sdk/include/transportplugin.h
@@ -17,10 +17,14 @@
 
 namespace zen {
 
+// Current API version, value will be incremented to represent breaking changes
+static const uint32_t kTransportApiVersion = 1;
+
 class TransportConnection;
 class TransportPlugin;
 class TransportServerConnection;
 class TransportServer;
+class TransportLogger;
 
 /*************************************************************************
 
@@ -60,6 +64,29 @@ public:
 	virtual TransportServerConnection* CreateConnectionHandler(TransportConnection* Connection) = 0;
 };
 
+/** Logger interface
+
+ There will be one instance of this provided by the system to the transport plugin
+
+ The plugin can use this to log messages back to zen server
+
+ */
+class TransportLogger
+{
+public:
+	enum class LogLevel : uint32_t
+	{
+		Trace	 = 0,
+		Debug	 = 1,
+		Info	 = 2,
+		Warn	 = 3,
+		Err		 = 4,
+		Critical = 5,
+	};
+
+	virtual void LogMessage(LogLevel Level, const char* Message) = 0;
+};
+
 /*************************************************************************
 
    The following interfaces are to be implemented by transport plugins.
@@ -116,7 +143,18 @@ public:
 
 extern "C"
 {
-	DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin();
+	/** Provide information about plugin version
+
+	   Fills out API version (kTransportApiVersion) plugin was built against.
+	   Fills out plugin own version ever increasing version number,
+	   a copy of plugin with higher version will be used.
+	*/
+	DLL_TRANSPORT_API void GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion);
+
+	// Return nullptr if requested api version mismatches api version plugin was built against
+	DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin(zen::TransportLogger* Logger);
 }
 
-typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)();
+typedef void (*PfnGetTransportPluginVersion)(uint32_t* OutApiVersion, uint32_t* OutPluginVersion);
+
+typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)(zen::TransportLogger* Logger);
diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp
index 1c3ee909a..f98984726 100644
--- a/src/transports/winsock/winsock.cpp
+++ b/src/transports/winsock/winsock.cpp
@@ -364,8 +364,22 @@ WinsockTransportPlugin::IsAvailable()
 
 //////////////////////////////////////////////////////////////////////////
 
+void
+GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion)
+{
+	if (OutApiVersion != nullptr)
+	{
+		*OutApiVersion = kTransportApiVersion;
+	}
+
+	if (OutPluginVersion != nullptr)
+	{
+		*OutPluginVersion = 1;
+	}
+}
+
 TransportPlugin*
-CreateTransportPlugin()
+CreateTransportPlugin([[maybe_unused]] TransportLogger* Logger)
 {
 	return new WinsockTransportPlugin;
 }
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 835e01151..fe2bbbdc7 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -57,10 +57,7 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	}
 	else if (int StatusCode = (int)Response.StatusCode)
 	{
-		ZEN_ERROR("scrub start failed: {}: {} ({})",
-				  (int)Response.StatusCode,
-				  ReasonStringForHttpResultCode((int)Response.StatusCode),
-				  Response.ToText());
+		ZEN_ERROR("scrub start failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText());
 	}
 	else
 	{
@@ -645,10 +642,7 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	}
 	else if (int StatusCode = (int)Response.StatusCode)
 	{
-		ZEN_ERROR("flush failed: {}: {} ({})",
-				  (int)Response.StatusCode,
-				  ReasonStringForHttpResultCode((int)Response.StatusCode),
-				  Response.ToText());
+		ZEN_ERROR("flush failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText());
 	}
 	else
 	{
@@ -688,7 +682,7 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target)
 static bool
 TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target)
 {
-	if (!std::filesystem::is_regular_file(Source))
+	if (!IsFile(Source))
 	{
 		return false;
 	}
@@ -714,29 +708,26 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		throw OptionParseException("data path must be given");
 	}
 
-	std::filesystem::path DataPath	 = StringToPath(m_DataPath);
-	std::filesystem::path TargetPath = StringToPath(m_TargetPath);
-
-	if (!std::filesystem::is_directory(DataPath))
+	if (!IsDir(m_DataPath))
 	{
 		throw OptionParseException("data path must exist");
 	}
 
-	if (TargetPath.empty())
+	if (m_TargetPath.empty())
 	{
 		throw OptionParseException("target path must be given");
 	}
 
-	std::filesystem::path RootManifestPath		 = DataPath / "root_manifest";
-	std::filesystem::path TargetRootManifestPath = TargetPath / "root_manifest";
+	std::filesystem::path RootManifestPath		 = m_DataPath / "root_manifest";
+	std::filesystem::path TargetRootManifestPath = m_TargetPath / "root_manifest";
 
 	if (!TryCopy(RootManifestPath, TargetRootManifestPath))
 	{
 		throw OptionParseException("data path is invalid, missing root_manifest");
 	}
 
-	std::filesystem::path CachePath		  = DataPath / "cache";
-	std::filesystem::path TargetCachePath = TargetPath / "cache";
+	std::filesystem::path CachePath		  = m_DataPath / "cache";
+	std::filesystem::path TargetCachePath = m_TargetPath / "cache";
 
 	// Copy cache state
 	DirectoryContent CacheDirectoryContent;
@@ -781,8 +772,8 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 	}
 
-	std::filesystem::path CasPath		= DataPath / "cas";
-	std::filesystem::path TargetCasPath = TargetPath / "cas";
+	std::filesystem::path CasPath		= m_DataPath / "cas";
+	std::filesystem::path TargetCasPath = m_TargetPath / "cas";
 
 	{
 		std::filesystem::path UCasRootPath		 = CasPath / ".ucas_root";
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 8b6d3e258..c593b2cac 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -155,10 +155,10 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"};
-	std::string		 m_DataPath;
-	std::string		 m_TargetPath;
-	bool			 m_SkipLogs = false;
+	cxxopts::Options	  m_Options{"copy-state", "Copy zen server disk state"};
+	std::filesystem::path m_DataPath;
+	std::filesystem::path m_TargetPath;
+	bool				  m_SkipLogs = false;
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index f0ee4904e..cd27daa9e 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -6,12 +6,16 @@
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinaryfile.h>
 #include <zencore/compactbinaryfmt.h>
+#include <zencore/compactbinaryvalue.h>
 #include <zencore/compress.h>
+#include <zencore/config.h>
 #include <zencore/except.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
+#include <zencore/session.h>
+#include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/trace.h>
 #include <zencore/uid.h>
@@ -19,6 +23,8 @@
 #include <zenhttp/httpclient.h>
 #include <zenhttp/httpclientauth.h>
 #include <zenhttp/httpcommon.h>
+#include <zenutil/bufferedwritefilecache.h>
+#include <zenutil/buildstoragecache.h>
 #include <zenutil/chunkblock.h>
 #include <zenutil/chunkedcontent.h>
 #include <zenutil/chunkedfile.h>
@@ -26,12 +32,13 @@
 #include <zenutil/filebuildstorage.h>
 #include <zenutil/jupiter/jupiterbuildstorage.h>
 #include <zenutil/jupiter/jupitersession.h>
-#include <zenutil/parallellwork.h>
+#include <zenutil/parallelwork.h>
 #include <zenutil/workerpools.h>
 #include <zenutil/zenserverprocess.h>
 
 #include <signal.h>
 #include <memory>
+#include <regex>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
@@ -50,24 +57,208 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 #define EXTRA_VERIFY 0
 
+#define ZEN_CLOUD_STORAGE "Cloud Storage"
+
 namespace zen {
+
+using namespace std::literals;
+
 namespace {
+	namespace zenutil {
+#if ZEN_PLATFORM_WINDOWS
+		class SecurityAttributes
+		{
+		public:
+			inline SECURITY_ATTRIBUTES* Attributes() { return &m_Attributes; }
+
+		protected:
+			SECURITY_ATTRIBUTES m_Attributes{};
+			SECURITY_DESCRIPTOR m_Sd{};
+		};
+
+		// Security attributes which allows any user access
+
+		class AnyUserSecurityAttributes : public SecurityAttributes
+		{
+		public:
+			AnyUserSecurityAttributes()
+			{
+				m_Attributes.nLength		= sizeof m_Attributes;
+				m_Attributes.bInheritHandle = false;  // Disable inheritance
+
+				const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION);
+
+				if (Success)
+				{
+					if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE))
+					{
+						ThrowLastError("SetSecurityDescriptorDacl failed");
+					}
+
+					m_Attributes.lpSecurityDescriptor = &m_Sd;
+				}
+			}
+		};
+#endif	// ZEN_PLATFORM_WINDOWS
+
+	}  // namespace zenutil
+
 	static std::atomic<bool> AbortFlag = false;
-	static void				 SignalCallbackHandler(int SigNum)
+	static std::atomic<bool> PauseFlag = false;
+
+	static void SignalCallbackHandler(int SigNum)
 	{
 		if (SigNum == SIGINT)
 		{
+			PauseFlag = false;
 			AbortFlag = true;
 		}
 #if ZEN_PLATFORM_WINDOWS
 		if (SigNum == SIGBREAK)
 		{
+			PauseFlag = false;
 			AbortFlag = true;
 		}
 #endif	// ZEN_PLATFORM_WINDOWS
 	}
 
-	using namespace std::literals;
+	struct ZenStateSharedData
+	{
+		static constexpr uint64_t kMagicV1 = 0x3176646d636e657a;  // zencmdv1
+
+		uint64_t Magic = 0;	 // Implies the size and layout of this struct - changes to the data requires change to Magic constant
+		std::atomic<uint32_t> Pid;
+		uint8_t				  SessionId[12];
+		uint8_t				  Padding1[4];
+		std::atomic<uint8_t>  Abort;
+		std::atomic<uint8_t>  Pause;
+		uint8_t				  Padding2[2];
+	};
+
+	struct MemMap
+	{
+		void*		Handle = nullptr;
+		void*		Data   = nullptr;
+		size_t		Size   = 0;
+		std::string Name;
+	};
+
+	class ZenState
+	{
+	public:
+		ZenState(const ZenState&) = delete;
+		ZenState& operator=(const ZenState&) = delete;
+
+		ZenState();
+		explicit ZenState(uint32_t Pid);
+		~ZenState();
+
+		const ZenStateSharedData& StateData() const
+		{
+			ZEN_ASSERT(m_Data);
+			return *m_Data;
+		}
+		ZenStateSharedData& StateData()
+		{
+			ZEN_ASSERT(m_Data);
+			return *m_Data;
+		}
+
+	private:
+		static constexpr std::string_view MapBaseName = "UnrealEngineZenCmd_"sv;
+		static constexpr size_t			  MapSize	  = sizeof(ZenStateSharedData);
+
+		bool						  m_Created = false;
+		std::unique_ptr<SharedMemory> m_MemMap;
+		ZenStateSharedData*			  m_Data = nullptr;
+
+		std::thread m_StateMonitor;
+		Event		m_ExitStateMonitorEvent;
+	};
+
+	ZenState::ZenState(uint32_t Pid)
+	{
+		const std::string ZenStateMapName = fmt::format("{}{}", MapBaseName, Pid);
+
+		if (!IsProcessRunning(Pid))
+		{
+			throw std::runtime_error(fmt::format("The process {} is not running", Pid));
+		}
+		std::unique_ptr<SharedMemory> MemMap = OpenSharedMemory(ZenStateMapName, MapSize, false);
+		if (!MemMap)
+		{
+			throw std::runtime_error(fmt::format("The process {} is not a running zen process", Pid));
+		}
+		ZenStateSharedData* data = (ZenStateSharedData*)MemMap->GetData();
+		if (uint64_t MemMagic = data->Magic; MemMagic != ZenStateSharedData::kMagicV1)
+		{
+			throw std::runtime_error(fmt::format("The mem map for process {} has an unsupported magic {:x}, expected {:x}",
+												 Pid,
+												 MemMagic,
+												 ZenStateSharedData::kMagicV1));
+		}
+		if (uint32_t MemPid = data->Pid.load(); MemPid != Pid)
+		{
+			throw std::runtime_error(
+				fmt::format("The mem map for process {} has an missmatching pid of {}, expected {}", Pid, MemPid, Pid));
+		}
+		m_MemMap = std::move(MemMap);
+		m_Data	 = data;
+	}
+
+	ZenState::ZenState()
+	{
+		int Pid = GetCurrentProcessId();
+
+		const std::string ZenStateMapName = fmt::format("{}{}", MapBaseName, Pid);
+
+		std::unique_ptr<SharedMemory> MemMap = CreateSharedMemory(ZenStateMapName, MapSize, false);
+		if (!MemMap)
+		{
+			throw std::runtime_error(fmt::format("The mem map for process {} could not be created", Pid));
+		}
+
+		ZenStateSharedData* data = (ZenStateSharedData*)MemMap->GetData();
+		memset(data, 0, sizeof(ZenStateSharedData));
+		data->Magic = ZenStateSharedData::kMagicV1;
+		data->Pid.store(gsl::narrow<uint32_t>(Pid));
+		data->SessionId;
+		data->Abort.store(false);
+		data->Pause.store(false);
+		const Oid SessionId = GetSessionId();
+		memcpy(data->SessionId, &SessionId, sizeof SessionId);
+
+		m_MemMap = std::move(MemMap);
+		m_Data	 = data;
+
+		m_StateMonitor = std::thread([this]() {
+			while (!m_ExitStateMonitorEvent.Wait(500))
+			{
+				if (m_Data->Abort.load())
+				{
+					AbortFlag.store(true);
+				}
+				PauseFlag.store(m_Data->Pause.load());
+			}
+		});
+	}
+
+	ZenState::~ZenState()
+	{
+		try
+		{
+			if (m_StateMonitor.joinable())
+			{
+				m_ExitStateMonitorEvent.Set();
+				m_StateMonitor.join();
+			}
+			m_MemMap.reset();
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_ERROR("ZenState::~ZenState threw exception: {}", Ex.what());
+		}
+	}
 
 	static const size_t DefaultMaxBlockSize		 = 64u * 1024u * 1024u;
 	static const size_t DefaultMaxChunkEmbedSize = 3u * 512u * 1024u;
@@ -78,30 +269,93 @@ namespace {
 		size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize;
 	};
 
-	const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize		= 32u * 1024u * 1024u,
-														 .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize};
-
+	const ChunksBlockParameters DefaultChunksBlockParams{
+		.MaxBlockSize	   = 32u * 1024u * 1024u,
+		.MaxChunkEmbedSize = 2u * 1024u * 1024u	 // DefaultChunkedParams.MaxSize
+	};
 	const uint64_t DefaultPreferredMultipartChunkSize = 32u * 1024u * 1024u;
 
 	const double DefaultLatency		  = 0;	// .0010;
 	const double DefaultDelayPerKBSec = 0;	// 0.00005;
 
-	const std::string ZenFolderName		   = ".zen";
-	const std::string ZenStateFilePath	   = fmt::format("{}/current_state.cbo", ZenFolderName);
-	const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName);
-	const std::string ZenTempFolderName	   = fmt::format("{}/tmp", ZenFolderName);
+	const bool SingleThreaded	  = false;
+	bool	   BoostWorkerThreads = false;
+	bool	   UseSparseFiles	  = false;
 
-	const std::string ZenTempCacheFolderName =
-		fmt::format("{}/cache", ZenTempFolderName);	 // Decompressed and verified data - chunks & sequences
-	const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName);	 // Temp storage for whole and partial blocks
-	const std::string ZenTempChunkFolderName =
-		fmt::format("{}/chunks", ZenTempFolderName);  // Temp storage for decompressed and validated chunks
+	WorkerThreadPool& GetIOWorkerPool()
+	{
+		return SingleThreaded		? GetSyncWorkerPool()
+			   : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst)
+									: GetMediumWorkerPool(EWorkloadType::Burst);
+	}
+	WorkerThreadPool& GetNetworkPool() { return SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); }
+
+	static const std::vector<uint32_t> NonCompressableExtensions({HashStringDjb2(".mp4"sv),
+																  HashStringDjb2(".zip"sv),
+																  HashStringDjb2(".7z"sv),
+																  HashStringDjb2(".bzip"sv),
+																  HashStringDjb2(".rar"sv),
+																  HashStringDjb2(".gzip"sv),
+																  HashStringDjb2(".apk"sv),
+																  HashStringDjb2(".nsp"sv),
+																  HashStringDjb2(".xvc"sv),
+																  HashStringDjb2(".pkg"sv),
+																  HashStringDjb2(".dmg"sv),
+																  HashStringDjb2(".ipa"sv)});
+
+	static const tsl::robin_set<uint32_t> NonCompressableExtensionSet(NonCompressableExtensions.begin(), NonCompressableExtensions.end());
+
+	static bool IsExtensionHashCompressable(const uint32_t PathHash) { return !NonCompressableExtensionSet.contains(PathHash); }
+
+	static bool IsChunkCompressable(const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
+	{
+		ZEN_UNUSED(Content);
+		const uint32_t ChunkLocationCount = Lookup.ChunkSequenceLocationCounts[ChunkIndex];
+		if (ChunkLocationCount == 0)
+		{
+			return false;
+		}
+		const size_t   ChunkLocationOffset = Lookup.ChunkSequenceLocationOffset[ChunkIndex];
+		const uint32_t SequenceIndex	   = Lookup.ChunkSequenceLocations[ChunkLocationOffset].SequenceIndex;
+		const uint32_t PathIndex		   = Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
+		const uint32_t ExtensionHash	   = Lookup.PathExtensionHash[PathIndex];
+
+		const bool IsCompressable = IsExtensionHashCompressable(ExtensionHash);
+		return IsCompressable;
+	}
 
-	const std::string ZenTempDownloadFolderName =
-		fmt::format("{}/download", ZenTempFolderName);	// Temp storage for unverfied downloaded blobs
+	const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u;
 
-	const std::string ZenTempStorageFolderName =
-		fmt::format("{}/storage", ZenTempFolderName);  // Temp storage folder for BuildStorage implementations
+	const std::string	  ZenFolderName = ".zen";
+	std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; }
+	// std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json";
+	// }
+	std::filesystem::path ZenTempFolderPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "tmp"; }
+
+	std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath)
+	{
+		return ZenTempFolderPath(ZenFolderPath) / "cache";	// Decompressed and verified data - chunks & sequences
+	}
+	std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath)
+	{
+		return ZenTempFolderPath(ZenFolderPath) / "blocks";	 // Temp storage for whole and partial blocks
+	}
+	std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path)
+	{
+		const std::u8string LocalPathString = Path.generic_u8string();
+		IoHash				PathHash		= IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length());
+		return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash);
+	}
+
+	std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath)
+	{
+		return ZenTempFolderPath(ZenFolderPath) / "download";  // Temp storage for decompressed and validated chunks
+	}
+
+	//	std::filesystem::path ZenTempStorageFolderPath(const std::filesystem::path& ZenFolderPath)
+	//	{
+	//		return ZenTempFolderPath(ZenFolderPath) / "storage";  // Temp storage folder for BuildStorage implementations
+	//	}
 
 	const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt";
 
@@ -113,8 +367,24 @@ namespace {
 	const std::vector<std::string_view> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName});
 	const std::vector<std::string_view> DefaultExcludeExtensions({});
 
-	static bool IsVerbose		 = false;
-	static bool UsePlainProgress = false;
+	static bool				 IsVerbose	  = false;
+	static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
+
+	uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode)
+	{
+		switch (InMode)
+		{
+			case ProgressBar::Mode::Plain:
+				return 5000;
+			case ProgressBar::Mode::Pretty:
+				return 200;
+			case ProgressBar::Mode::Log:
+				return 2000;
+			default:
+				ZEN_ASSERT(false);
+				return 0;
+		}
+	}
 
 #define ZEN_CONSOLE_VERBOSE(fmtstr, ...)                                   \
 	if (IsVerbose)                                                         \
@@ -132,6 +402,152 @@ namespace {
 
 	);
 
+	bool IsFileWithRetry(const std::filesystem::path& Path)
+	{
+		std::error_code Ec;
+		bool			Result = IsFile(Path, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			Ec.clear();
+			Result = IsFile(Path, Ec);
+		}
+		if (Ec)
+		{
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Check path '{}' is file failed with: {} ({})", Path, Ec.message(), Ec.value()));
+		}
+		return Result;
+	}
+
+	bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
+	{
+		std::error_code Ec;
+		bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			if (!IsFileWithRetry(Path))
+			{
+				return false;
+			}
+			Ec.clear();
+			Result = SetFileReadOnly(Path, ReadOnly, Ec);
+		}
+		if (Ec)
+		{
+			throw std::system_error(
+				std::error_code(Ec.value(), std::system_category()),
+				fmt::format("Failed {} read only flag for '{}' failed with: {}", ReadOnly ? "setting" : "clearing", Path, Ec.message()));
+		}
+		return Result;
+	}
+
+	void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
+	{
+		std::error_code Ec;
+		RenameFile(SourcePath, TargetPath, Ec);
+		for (size_t Retries = 0; Ec && Retries < 10; Retries++)
+		{
+			ZEN_ASSERT_SLOW(IsFile(SourcePath));
+			if (Retries > 5)
+			{
+				ZEN_CONSOLE("Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message());
+			}
+			Sleep(50 + int(Retries * 150));
+			Ec.clear();
+			RenameFile(SourcePath, TargetPath, Ec);
+		}
+		if (Ec)
+		{
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Rename from '{}' to '{}' failed with: {}", SourcePath, TargetPath, Ec.message()));
+		}
+	}
+
+	void TryRemoveFile(const std::filesystem::path& Path)
+	{
+		std::error_code Ec;
+		RemoveFile(Path, Ec);
+		if (Ec)
+		{
+			if (IsFile(Path, Ec))
+			{
+				Ec.clear();
+				RemoveFile(Path, Ec);
+				if (Ec)
+				{
+					ZEN_DEBUG("Failed removing file '{}', reason: {}", Path, Ec.message());
+				}
+			}
+		}
+	}
+
+	void RemoveFileWithRetry(const std::filesystem::path& Path)
+	{
+		std::error_code Ec;
+		RemoveFile(Path, Ec);
+		for (size_t Retries = 0; Ec && Retries < 6; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			if (!IsFileWithRetry(Path))
+			{
+				return;
+			}
+			Ec.clear();
+			RemoveFile(Path, Ec);
+		}
+		if (Ec)
+		{
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Removing file '{}' failed with: {}", Path, Ec.message()));
+		}
+	}
+
+	void RemoveDirWithRetry(const std::filesystem::path& Path)
+	{
+		std::error_code Ec;
+		RemoveDir(Path, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			if (!IsDir(Path))
+			{
+				return;
+			}
+			Ec.clear();
+			RemoveDir(Path, Ec);
+		}
+		if (Ec)
+		{
+			throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+									fmt::format("Removing directory '{}' failed with: {}", Path, Ec.message()));
+		}
+	}
+
+	void CopyFile(const std::filesystem::path& SourceFilePath,
+				  const std::filesystem::path& TargetFilePath,
+				  uint64_t					   RawSize,
+				  std::atomic<uint64_t>&	   WriteCount,
+				  std::atomic<uint64_t>&	   WriteByteCount)
+	{
+		BasicFile TargetFile(TargetFilePath, BasicFile::Mode::kTruncate);
+		if (UseSparseFiles)
+		{
+			PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize);
+		}
+		uint64_t Offset = 0;
+		if (!ScanFile(SourceFilePath, 512u * 1024u, [&](const void* Data, size_t Size) {
+				TargetFile.Write(Data, Size, Offset);
+				Offset += Size;
+				WriteCount++;
+				WriteByteCount += Size;
+			}))
+		{
+			throw std::runtime_error(fmt::format("Failed to copy scavenged file '{}' to '{}'", SourceFilePath, TargetFilePath));
+		}
+	}
+
 	uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes)
 	{
 #if ZEN_PLATFORM_WINDOWS
@@ -194,34 +610,109 @@ namespace {
 	bool CleanDirectory(const std::filesystem::path& Path, std::span<const std::string_view> ExcludeDirectories)
 	{
 		ZEN_TRACE_CPU("CleanDirectory");
+		Stopwatch Timer;
 
-		bool CleanWipe = true;
+		ProgressBar Progress(ProgressMode, "Clean Folder");
 
-		DirectoryContent LocalDirectoryContent;
-		GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
-		for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files)
+		std::atomic<bool>	  CleanWipe			  = true;
+		std::atomic<uint64_t> DiscoveredItemCount = 0;
+		std::atomic<uint64_t> DeletedItemCount	  = 0;
+		std::atomic<uint64_t> DeletedByteCount	  = 0;
+		ParallelWork		  Work(AbortFlag, PauseFlag);
+
+		struct AsyncVisitor : public GetDirectoryContentVisitor
 		{
-			try
+			AsyncVisitor(const std::filesystem::path&	   InPath,
+						 std::atomic<bool>&				   InCleanWipe,
+						 std::atomic<uint64_t>&			   InDiscoveredItemCount,
+						 std::atomic<uint64_t>&			   InDeletedItemCount,
+						 std::atomic<uint64_t>&			   InDeletedByteCount,
+						 std::span<const std::string_view> InExcludeDirectories)
+			: Path(InPath)
+			, CleanWipe(InCleanWipe)
+			, DiscoveredItemCount(InDiscoveredItemCount)
+			, DeletedItemCount(InDeletedItemCount)
+			, DeletedByteCount(InDeletedByteCount)
+			, ExcludeDirectories(InExcludeDirectories)
 			{
-				std::filesystem::remove(LocalFilePath);
 			}
-			catch (const std::exception&)
+			virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
 			{
-				// DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time
-				Sleep(200);
-				try
-				{
-					std::filesystem::remove(LocalFilePath);
-				}
-				catch (const std::exception& Ex)
+				ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
+				if (!AbortFlag)
 				{
-					ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what());
-					CleanWipe = false;
+					if (!Content.FileNames.empty())
+					{
+						DiscoveredItemCount += Content.FileNames.size();
+
+						const std::string RelativeRootString = RelativeRoot.generic_string();
+						bool			  RemoveContent		 = true;
+						for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+						{
+							if (RelativeRootString.starts_with(ExcludeDirectory))
+							{
+								if (RelativeRootString.length() > ExcludeDirectory.length())
+								{
+									const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
+									if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
+										MaybePathDelimiter == std::filesystem::path::preferred_separator)
+									{
+										RemoveContent = false;
+										break;
+									}
+								}
+								else
+								{
+									RemoveContent = false;
+									break;
+								}
+							}
+						}
+						if (RemoveContent)
+						{
+							ZEN_TRACE_CPU("DeleteFiles");
+							for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+							{
+								const std::filesystem::path& FileName = Content.FileNames[FileIndex];
+								const std::filesystem::path	 FilePath = (Path / RelativeRoot / FileName).make_preferred();
+								try
+								{
+									SetFileReadOnlyWithRetry(FilePath, false);
+									RemoveFileWithRetry(FilePath);
+									DeletedItemCount++;
+									DeletedByteCount += Content.FileSizes[FileIndex];
+								}
+								catch (const std::exception& Ex)
+								{
+									ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what());
+									CleanWipe = false;
+								}
+							}
+						}
+					}
 				}
 			}
-		}
+			const std::filesystem::path&	  Path;
+			std::atomic<bool>&				  CleanWipe;
+			std::atomic<uint64_t>&			  DiscoveredItemCount;
+			std::atomic<uint64_t>&			  DeletedItemCount;
+			std::atomic<uint64_t>&			  DeletedByteCount;
+			std::span<const std::string_view> ExcludeDirectories;
+		} Visitor(Path, CleanWipe, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories);
+
+		GetDirectoryContent(
+			Path,
+			DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes,
+			Visitor,
+			GetIOWorkerPool(),
+			Work.PendingWork());
 
-		for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
+		DirectoryContent LocalDirectoryContent;
+		GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
+		DiscoveredItemCount += LocalDirectoryContent.Directories.size();
+		std::vector<std::filesystem::path> DirectoriesToDelete;
+		DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size());
+		for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
 		{
 			bool Leave = false;
 			for (const std::string_view ExcludeDirectory : ExcludeDirectories)
@@ -234,33 +725,134 @@ namespace {
 			}
 			if (!Leave)
 			{
-				try
-				{
-					zen::CleanDirectory(LocalDirPath);
-					std::filesystem::remove(LocalDirPath);
-				}
-				catch (const std::exception&)
+				DirectoriesToDelete.emplace_back(std::move(LocalDirPath));
+				DiscoveredItemCount++;
+			}
+		}
+
+		uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+		Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) {
+			ZEN_UNUSED(PendingWork);
+			LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+			uint64_t Deleted	  = DeletedItemCount.load();
+			uint64_t DeletedBytes = DeletedByteCount.load();
+			uint64_t Discovered	  = DiscoveredItemCount.load();
+			Progress.UpdateState({.Task			  = "Cleaning folder  ",
+								  .Details		  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+								  .TotalCount	  = Discovered,
+								  .RemainingCount = Discovered - Deleted,
+								  .Status		  = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+								 false);
+		});
+
+		for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete)
+		{
+			ZEN_TRACE_CPU("DeleteDirs");
+			try
+			{
+				std::error_code Ec;
+				zen::CleanDirectory(DirectoryToDelete, true, Ec);
+				if (Ec)
 				{
 					Sleep(200);
-					try
-					{
-						zen::CleanDirectory(LocalDirPath);
-						std::filesystem::remove(LocalDirPath);
-					}
-					catch (const std::exception& Ex)
-					{
-						ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what());
-						CleanWipe = false;
-					}
+					zen::CleanDirectory(DirectoryToDelete, true);
+					Ec.clear();
 				}
+
+				RemoveDirWithRetry(DirectoryToDelete);
+
+				DeletedItemCount++;
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
+				CleanWipe = false;
+			}
+
+			uint64_t NowMs = Timer.GetElapsedTimeMs();
+			if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode))
+			{
+				LastUpdateTimeMs = NowMs;
+
+				uint64_t Deleted	  = DeletedItemCount.load();
+				uint64_t DeletedBytes = DeletedByteCount.load();
+				uint64_t Discovered	  = DiscoveredItemCount.load();
+				Progress.UpdateState({.Task		  = "Cleaning folder  ",
+									  .Details	  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+									  .TotalCount = Discovered,
+									  .RemainingCount = Discovered - Deleted,
+									  .Status		  = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
+									 false);
 			}
 		}
+
+		Progress.Finish();
+		if (AbortFlag)
+		{
+			return false;
+		}
+
+		uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
+		if (ElapsedTimeMs >= 200)
+		{
+			ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}",
+						Path,
+						DiscoveredItemCount.load(),
+						NiceBytes(DeletedByteCount.load()),
+						NiceTimeSpanMs(ElapsedTimeMs));
+		}
 		return CleanWipe;
 	}
 
+	bool ParseCloudUrl(std::string_view InUrl,
+					   std::string&		OutHost,
+					   std::string&		OutNamespace,
+					   std::string&		OutBucket,
+					   std::string&		OutBuildId)
+	{
+		std::string			   Url(RemoveQuotes(InUrl));
+		const std::string_view ExtendedApiString = "api/v2/builds/";
+		if (auto ApiString = ToLower(Url).find(ExtendedApiString); ApiString != std::string::npos)
+		{
+			Url.erase(ApiString, ExtendedApiString.length());
+		}
+
+		const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))";
+		const std::regex  ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase);
+		std::match_results<std::string_view::const_iterator> MatchResults;
+		std::string_view									 UrlToParse(Url);
+		if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5)
+		{
+			auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view {
+				ZEN_ASSERT(Index < MatchResults.size());
+
+				const auto& Match = MatchResults[Index];
+
+				return std::string_view(&*Match.first, Match.second - Match.first);
+			};
+
+			const std::string_view Host		 = GetMatch(1);
+			const std::string_view Namespace = GetMatch(2);
+			const std::string_view Bucket	 = GetMatch(3);
+			const std::string_view BuildId	 = GetMatch(4);
+
+			OutHost		 = Host;
+			OutNamespace = Namespace;
+			OutBucket	 = Bucket;
+			OutBuildId	 = BuildId;
+			return true;
+		}
+		else
+		{
+			return false;
+		}
+	}
+
 	std::string ReadAccessTokenFromFile(const std::filesystem::path& Path)
 	{
-		if (!std::filesystem::is_regular_file(Path))
+		if (!IsFile(Path))
 		{
 			throw std::runtime_error(fmt::format("the file '{}' does not exist", Path));
 		}
@@ -280,30 +872,12 @@ namespace {
 		return AuthToken;
 	}
 
-	bool IsBufferDiskBased(const IoBuffer& Buffer)
-	{
-		IoBufferFileReference FileRef;
-		if (Buffer.GetFileReference(FileRef))
-		{
-			return true;
-		}
-		return false;
-	}
-
-	bool IsBufferDiskBased(const CompositeBuffer& Buffer)
-	{
-		// If this is a file based buffer or a compressed buffer with a memory-based header, we don't need to rewrite to disk to save memory
-		std::span<const SharedBuffer> Segments = Buffer.GetSegments();
-		ZEN_ASSERT(Buffer.GetSegments().size() > 0);
-		return IsBufferDiskBased(Segments.back().AsIoBuffer());
-	}
-
 	IoBuffer WriteToTempFile(CompositeBuffer&&			  Buffer,
 							 const std::filesystem::path& TempFolderPath,
 							 const IoHash&				  Hash,
 							 const std::string&			  Suffix = {})
 	{
-		std::filesystem::path TempFilePath = (TempFolderPath / (Hash.ToHexString() + Suffix)).make_preferred();
+		std::filesystem::path TempFilePath = TempFolderPath / (Hash.ToHexString() + Suffix);
 		return WriteToTempFile(std::move(Buffer), TempFilePath);
 	}
 
@@ -401,83 +975,14 @@ namespace {
 
 	std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash)
 	{
-		return (CacheFolderPath / (RawHash.ToHexString() + ".tmp")).make_preferred();
+		return CacheFolderPath / (RawHash.ToHexString() + ".tmp");
 	}
 
 	std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash)
 	{
-		return (CacheFolderPath / RawHash.ToHexString()).make_preferred();
+		return CacheFolderPath / RawHash.ToHexString();
 	}
 
-	ChunkedFolderContent ScanAndChunkFolder(
-		GetFolderContentStatistics&																 GetFolderContentStats,
-		ChunkingStatistics&																		 ChunkingStats,
-		const std::filesystem::path&															 Path,
-		std::function<bool(const std::string_view& RelativePath)>&&								 IsAcceptedFolder,
-		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
-		ChunkingController&																		 ChunkController)
-	{
-		ZEN_TRACE_CPU("ScanAndChunkFolder");
-
-		FolderContent Content = GetFolderContent(
-			GetFolderContentStats,
-			Path,
-			std::move(IsAcceptedFolder),
-			std::move(IsAcceptedFile),
-			GetMediumWorkerPool(EWorkloadType::Burst),
-			UsePlainProgress ? 5000 : 200,
-			[](bool, std::ptrdiff_t) {},
-			AbortFlag);
-		if (AbortFlag)
-		{
-			return {};
-		}
-
-		ProgressBar	 ProgressBar(UsePlainProgress);
-		FilteredRate FilteredBytesHashed;
-		FilteredBytesHashed.Start();
-		ChunkedFolderContent FolderContent = ChunkFolderContent(
-			ChunkingStats,
-			GetMediumWorkerPool(EWorkloadType::Burst),
-			Path,
-			Content,
-			ChunkController,
-			UsePlainProgress ? 5000 : 200,
-			[&](bool, std::ptrdiff_t) {
-				FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
-				std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
-												  ChunkingStats.FilesProcessed.load(),
-												  GetFolderContentStats.AcceptedFileCount.load(),
-												  NiceBytes(ChunkingStats.BytesHashed.load()),
-												  NiceBytes(GetFolderContentStats.FoundFileByteCount),
-												  NiceNum(FilteredBytesHashed.GetCurrent()),
-												  ChunkingStats.UniqueChunksFound.load(),
-												  NiceBytes(ChunkingStats.UniqueBytesFound.load()));
-				ProgressBar.UpdateState({.Task			 = "Scanning files   ",
-										 .Details		 = Details,
-										 .TotalCount	 = GetFolderContentStats.AcceptedFileByteCount,
-										 .RemainingCount = GetFolderContentStats.AcceptedFileByteCount - ChunkingStats.BytesHashed.load()},
-										false);
-			},
-			AbortFlag);
-		if (AbortFlag)
-		{
-			return {};
-		}
-		FilteredBytesHashed.Stop();
-		ProgressBar.Finish();
-
-		ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
-					ChunkingStats.FilesProcessed.load(),
-					NiceBytes(ChunkingStats.BytesHashed.load()),
-					ChunkingStats.UniqueChunksFound.load(),
-					NiceBytes(ChunkingStats.UniqueBytesFound.load()),
-					Path,
-					NiceTimeSpanMs((GetFolderContentStats.ElapsedWallTimeUS + ChunkingStats.ElapsedWallTimeUS) / 1000),
-					NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
-		return FolderContent;
-	};
-
 	struct DiskStatistics
 	{
 		std::atomic<uint64_t> OpenReadCount		   = 0;
@@ -500,6 +1005,7 @@ namespace {
 		uint64_t AcceptedBlockCount			 = 0;
 		uint64_t AcceptedChunkCount			 = 0;
 		uint64_t AcceptedByteCount			 = 0;
+		uint64_t AcceptedRawByteCount		 = 0;
 		uint64_t RejectedBlockCount			 = 0;
 		uint64_t RejectedChunkCount			 = 0;
 		uint64_t RejectedByteCount			 = 0;
@@ -538,6 +1044,7 @@ namespace {
 		uint64_t			  ChunkCount					  = 0;
 		uint64_t			  ChunkByteCount				  = 0;
 		std::atomic<uint64_t> CompressedChunkCount			  = 0;
+		std::atomic<uint64_t> CompressedChunkRawBytes		  = 0;
 		std::atomic<uint64_t> CompressedChunkBytes			  = 0;
 		uint64_t			  CompressChunksElapsedWallTimeUS = 0;
 
@@ -546,6 +1053,7 @@ namespace {
 			ChunkCount += Rhs.ChunkCount;
 			ChunkByteCount += Rhs.ChunkByteCount;
 			CompressedChunkCount += Rhs.CompressedChunkCount;
+			CompressedChunkRawBytes += Rhs.CompressedChunkRawBytes;
 			CompressedChunkBytes += Rhs.CompressedChunkBytes;
 			CompressChunksElapsedWallTimeUS += Rhs.CompressChunksElapsedWallTimeUS;
 			return *this;
@@ -567,6 +1075,84 @@ namespace {
 		}
 	};
 
+	struct CacheMappingStatistics
+	{
+		uint64_t CacheChunkCount	 = 0;
+		uint64_t CacheChunkByteCount = 0;
+
+		uint64_t CacheBlockCount	  = 0;
+		uint64_t CacheBlocksByteCount = 0;
+
+		uint64_t CacheSequenceHashesCount	  = 0;
+		uint64_t CacheSequenceHashesByteCount = 0;
+
+		uint64_t CacheScanElapsedWallTimeUs = 0;
+
+		uint32_t LocalPathsMatchingSequencesCount	  = 0;
+		uint64_t LocalPathsMatchingSequencesByteCount = 0;
+
+		uint64_t LocalChunkMatchingRemoteCount	   = 0;
+		uint64_t LocalChunkMatchingRemoteByteCount = 0;
+
+		uint64_t LocalScanElapsedWallTimeUs = 0;
+
+		uint32_t ScavengedPathsMatchingSequencesCount	  = 0;
+		uint64_t ScavengedPathsMatchingSequencesByteCount = 0;
+
+		uint64_t ScavengedChunkMatchingRemoteCount	   = 0;
+		uint64_t ScavengedChunkMatchingRemoteByteCount = 0;
+
+		uint64_t ScavengeElapsedWallTimeUs = 0;
+	};
+
+	struct DownloadStatistics
+	{
+		std::atomic<uint64_t> RequestsCompleteCount = 0;
+
+		std::atomic<uint64_t> DownloadedChunkCount	   = 0;
+		std::atomic<uint64_t> DownloadedChunkByteCount = 0;
+		std::atomic<uint64_t> MultipartAttachmentCount = 0;
+
+		std::atomic<uint64_t> DownloadedBlockCount	   = 0;
+		std::atomic<uint64_t> DownloadedBlockByteCount = 0;
+
+		std::atomic<uint64_t> DownloadedPartialBlockCount	  = 0;
+		std::atomic<uint64_t> DownloadedPartialBlockByteCount = 0;
+	};
+
+	struct WriteChunkStatistics
+	{
+		uint64_t DownloadTimeUs				  = 0;
+		uint64_t WriteTimeUs				  = 0;
+		uint64_t WriteChunksElapsedWallTimeUs = 0;
+	};
+
+	struct RebuildFolderStateStatistics
+	{
+		uint64_t			  CleanFolderElapsedWallTimeUs	= 0;
+		std::atomic<uint32_t> FinalizeTreeFilesMovedCount	= 0;
+		std::atomic<uint32_t> FinalizeTreeFilesCopiedCount	= 0;
+		uint64_t			  FinalizeTreeElapsedWallTimeUs = 0;
+	};
+
+	struct VerifyFolderStatistics
+	{
+		std::atomic<uint64_t> FilesVerified			  = 0;
+		std::atomic<uint64_t> FilesFailed			  = 0;
+		std::atomic<uint64_t> ReadBytes				  = 0;
+		uint64_t			  VerifyElapsedWallTimeUs = 0;
+	};
+
+	struct StorageInstance
+	{
+		std::unique_ptr<HttpClient>		   BuildStorageHttp;
+		std::unique_ptr<BuildStorage>	   BuildStorage;
+		std::string						   StorageName;
+		std::unique_ptr<HttpClient>		   CacheHttp;
+		std::unique_ptr<BuildStorageCache> BuildCacheStorage;
+		std::string						   CacheName;
+	};
+
 	std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash>						   LocalChunkHashes,
 													   const std::span<const uint32_t>						   LocalChunkOrder,
 													   const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToLocalChunkIndex,
@@ -713,7 +1299,7 @@ namespace {
 										  std::span<const uint32_t>				 ChunkCounts,
 										  std::span<const IoHash>				 LocalChunkHashes,
 										  std::span<const uint64_t>				 LocalChunkRawSizes,
-										  std::vector<uint32_t>					 AbsoluteChunkOrders,
+										  const std::vector<uint32_t>&			 AbsoluteChunkOrders,
 										  const std::span<const uint32_t>		 LooseLocalChunkIndexes,
 										  const std::span<IoHash>				 BlockHashes)
 	{
@@ -971,12 +1557,14 @@ namespace {
 		}
 	}
 
-	CbObject CreateStateObject(const Oid&								BuildId,
-							   std::vector<std::pair<Oid, std::string>> AllBuildParts,
-							   std::span<const ChunkedFolderContent>	PartContents,
-							   const FolderContent&						LocalFolderState)
+	CbObject CreateStateObject(const Oid&									   BuildId,
+							   const std::vector<std::pair<Oid, std::string>>& AllBuildParts,
+							   std::span<const ChunkedFolderContent>		   PartContents,
+							   const FolderContent&							   LocalFolderState,
+							   const std::filesystem::path&					   LocalPath)
 	{
 		CbObjectWriter CurrentStateWriter;
+		CurrentStateWriter.AddString("path", (const char*)LocalPath.u8string().c_str());
 		CurrentStateWriter.BeginArray("builds"sv);
 		{
 			CurrentStateWriter.BeginObject();
@@ -1013,10 +1601,121 @@ namespace {
 		return CurrentStateWriter.Save();
 	}
 
+	void AddDownloadedPath(const std::filesystem::path&					   SystemRootDir,
+						   const Oid&									   BuildId,
+						   const std::vector<std::pair<Oid, std::string>>& BuildParts,
+						   const std::filesystem::path&					   StateFilePath,
+						   const std::filesystem::path&					   LocalPath)
+	{
+		ZEN_ASSERT(!SystemRootDir.empty());
+		ZEN_ASSERT(!StateFilePath.empty());
+		ZEN_ASSERT(!LocalPath.empty());
+		const std::u8string	  LocalPathString = LocalPath.generic_u8string();
+		IoHash				  PathHash		  = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length());
+		std::filesystem::path WritePath		  = SystemRootDir / "builds" / "downloads" / (PathHash.ToHexString() + ".json");
+		CreateDirectories(WritePath.parent_path());
+		CbObjectWriter Writer;
+		Writer.AddString("path", (const char*)LocalPath.u8string().c_str());
+		Writer.AddString("statePath", (const char*)StateFilePath.u8string().c_str());
+		Writer.AddDateTime("date", DateTime::Now());
+		Writer.BeginArray("builds"sv);
+		{
+			Writer.BeginObject();
+			{
+				Writer.AddObjectId("buildId", BuildId);
+				Writer.BeginArray("parts");
+				for (const auto& It : BuildParts)
+				{
+					Writer.BeginObject();
+					{
+						Writer.AddObjectId("partId", It.first);
+						Writer.AddString("partName", It.second);
+					}
+					Writer.EndObject();
+				}
+				Writer.EndArray();	// parts
+			}
+			Writer.EndObject();
+		}
+		Writer.EndArray();	// builds
+
+		CbObject					 Payload = Writer.Save();
+		ExtendableStringBuilder<512> SB;
+		CompactBinaryToJson(Payload.GetView(), SB);
+		MemoryView JsonPayload(SB.Data(), SB.Size());
+		TemporaryFile::SafeWriteFile(WritePath, JsonPayload);
+	}
+
+	struct ScavengeSource
+	{
+		std::filesystem::path StateFilePath;
+		std::filesystem::path Path;
+	};
+
+	std::vector<ScavengeSource> GetDownloadedStatePaths(const std::filesystem::path& SystemRootDir)
+	{
+		std::vector<ScavengeSource> Result;
+		DirectoryContent			Content;
+		GetDirectoryContent(SystemRootDir / "builds" / "downloads", DirectoryContentFlags::IncludeFiles, Content);
+		for (const std::filesystem::path& EntryPath : Content.Files)
+		{
+			bool   DeleteEntry = false;
+			IoHash EntryPathHash;
+			if (IoHash::TryParse(EntryPath.stem().string(), EntryPathHash))
+			{
+				// Read state and verify that it is valid
+				IoBuffer		 MetaDataJson = ReadFile(EntryPath).Flatten();
+				std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+				std::string		 JsonError;
+				CbObject		 DownloadInfo = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+				if (JsonError.empty())
+				{
+					std::filesystem::path StateFilePath = DownloadInfo["statePath"].AsU8String();
+					if (IsFile(StateFilePath))
+					{
+						std::filesystem::path Path = DownloadInfo["path"].AsU8String();
+						if (IsDir(Path))
+						{
+							Result.push_back({.StateFilePath = std::move(StateFilePath), .Path = std::move(Path)});
+						}
+						else
+						{
+							DeleteEntry = true;
+						}
+					}
+					else
+					{
+						DeleteEntry = true;
+					}
+				}
+				else
+				{
+					ZEN_WARN("Invalid download state file at {}. '{}'", EntryPath, JsonError);
+					DeleteEntry = true;
+				}
+			}
+
+			if (DeleteEntry)
+			{
+				std::error_code DummyEc;
+				std::filesystem::remove(EntryPath, DummyEc);
+			}
+		}
+		return Result;
+	}
+
 	class BufferedOpenFile
 	{
 	public:
-		BufferedOpenFile(const std::filesystem::path Path) : Source(Path, BasicFile::Mode::kRead), SourceSize(Source.FileSize()) {}
+		BufferedOpenFile(const std::filesystem::path Path, DiskStatistics& DiskStats)
+		: m_Source(Path, BasicFile::Mode::kRead)
+		, m_SourceSize(m_Source.FileSize())
+		, m_DiskStats(DiskStats)
+		{
+			m_DiskStats.OpenReadCount++;
+			m_DiskStats.CurrentOpenFileCount++;
+		}
+		~BufferedOpenFile() { m_DiskStats.CurrentOpenFileCount--; }
 		BufferedOpenFile()						  = delete;
 		BufferedOpenFile(const BufferedOpenFile&) = delete;
 		BufferedOpenFile(BufferedOpenFile&&)	  = delete;
@@ -1028,10 +1727,10 @@ namespace {
 		{
 			ZEN_TRACE_CPU("BufferedOpenFile::GetRange");
 
-			ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache);
-			auto _ = MakeGuard([&]() { ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); });
+			ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache);
+			auto _ = MakeGuard([&]() { ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); });
 
-			ZEN_ASSERT((Offset + Size) <= SourceSize);
+			ZEN_ASSERT((Offset + Size) <= m_SourceSize);
 			const uint64_t BlockIndexStart = Offset / BlockSize;
 			const uint64_t BlockIndexEnd   = (Offset + Size - 1) / BlockSize;
 
@@ -1042,21 +1741,23 @@ namespace {
 			for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++)
 			{
 				const uint64_t BlockStartOffset = BlockIndex * BlockSize;
-				if (CacheBlockIndex != BlockIndex)
+				if (m_CacheBlockIndex != BlockIndex)
 				{
-					uint64_t CacheSize = Min(BlockSize, SourceSize - BlockStartOffset);
+					uint64_t CacheSize = Min(BlockSize, m_SourceSize - BlockStartOffset);
 					ZEN_ASSERT(CacheSize > 0);
-					Cache = IoBuffer(CacheSize);
-					Source.Read(Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset);
-					CacheBlockIndex = BlockIndex;
+					m_Cache = IoBuffer(CacheSize);
+					m_Source.Read(m_Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset);
+					m_DiskStats.ReadCount++;
+					m_DiskStats.ReadByteCount += CacheSize;
+					m_CacheBlockIndex = BlockIndex;
 				}
 
 				const uint64_t BytesRead = ReadOffset - Offset;
 				ZEN_ASSERT(BlockStartOffset <= ReadOffset);
 				const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset;
-				ZEN_ASSERT(OffsetIntoBlock < Cache.GetSize());
-				const uint64_t BlockBytes = Min(Cache.GetSize() - OffsetIntoBlock, Size - BytesRead);
-				BufferRanges.emplace_back(SharedBuffer(IoBuffer(Cache, OffsetIntoBlock, BlockBytes)));
+				ZEN_ASSERT(OffsetIntoBlock < m_Cache.GetSize());
+				const uint64_t BlockBytes = Min(m_Cache.GetSize() - OffsetIntoBlock, Size - BytesRead);
+				BufferRanges.emplace_back(SharedBuffer(IoBuffer(m_Cache, OffsetIntoBlock, BlockBytes)));
 				ReadOffset += BlockBytes;
 			}
 			CompositeBuffer Result(std::move(BufferRanges));
@@ -1065,10 +1766,11 @@ namespace {
 		}
 
 	private:
-		BasicFile	   Source;
-		const uint64_t SourceSize;
-		uint64_t	   CacheBlockIndex = (uint64_t)-1;
-		IoBuffer	   Cache;
+		BasicFile		m_Source;
+		const uint64_t	m_SourceSize;
+		DiskStatistics& m_DiskStats;
+		uint64_t		m_CacheBlockIndex = (uint64_t)-1;
+		IoBuffer		m_Cache;
 	};
 
 	class ReadFileCache
@@ -1087,11 +1789,7 @@ namespace {
 		{
 			m_OpenFiles.reserve(MaxOpenFileCount);
 		}
-		~ReadFileCache()
-		{
-			m_DiskStats.CurrentOpenFileCount -= m_OpenFiles.size();
-			m_OpenFiles.clear();
-		}
+		~ReadFileCache() { m_OpenFiles.clear(); }
 
 		CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size)
 		{
@@ -1109,7 +1807,6 @@ namespace {
 					m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile)));
 				}
 				CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
-				m_DiskStats.ReadByteCount += Result.GetSize();
 				return Result;
 			}
 			const uint32_t				LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex];
@@ -1117,20 +1814,15 @@ namespace {
 			if (Size == m_LocalContent.RawSizes[LocalPathIndex])
 			{
 				IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath);
-				m_DiskStats.OpenReadCount++;
-				m_DiskStats.ReadByteCount += Result.GetSize();
 				return CompositeBuffer(SharedBuffer(Result));
 			}
 			if (m_OpenFiles.size() == m_OpenFiles.capacity())
 			{
 				m_OpenFiles.pop_back();
-				m_DiskStats.CurrentOpenFileCount--;
 			}
-			m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::make_unique<BufferedOpenFile>(LocalFilePath)));
+			m_OpenFiles.insert(m_OpenFiles.begin(),
+							   std::make_pair(SequenceIndex, std::make_unique<BufferedOpenFile>(LocalFilePath, m_DiskStats)));
 			CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
-			m_DiskStats.ReadByteCount += Result.GetSize();
-			m_DiskStats.OpenReadCount++;
-			m_DiskStats.CurrentOpenFileCount++;
 			return Result;
 		}
 
@@ -1167,17 +1859,21 @@ namespace {
 		}
 
 		IoHashStream Hash;
-		bool		 CouldDecompress = Compressed.DecompressToStream(0, RawSize, [&Hash](uint64_t, const CompositeBuffer& RangeBuffer) {
-			if (!AbortFlag)
-			{
-				for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+		bool		 CouldDecompress = Compressed.DecompressToStream(
+			0,
+			RawSize,
+			[&Hash](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+				ZEN_UNUSED(SourceOffset, SourceSize, Offset);
+				if (!AbortFlag)
 				{
-					Hash.Append(Segment.GetView());
+					for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+					{
+						Hash.Append(Segment.GetView());
+					}
+					return true;
 				}
-				return true;
-			}
-			return false;
-		});
+				return false;
+			});
 
 		if (AbortFlag)
 		{
@@ -1287,8 +1983,13 @@ namespace {
 					{
 						ZEN_ASSERT(false);
 					}
-					uint64_t RawSize = Chunk.GetSize();
-					return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)};
+					uint64_t   RawSize			   = Chunk.GetSize();
+					const bool ShouldCompressChunk = Lookup.RawHashToSequenceIndex.contains(ChunkHash) &&
+													 (RawSize >= MinimumSizeForCompressInBlock) &&
+													 IsChunkCompressable(Content, Lookup, ChunkIndex);
+					const OodleCompressionLevel CompressionLevel =
+						ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
+					return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, CompressionLevel)};
 				}));
 		}
 
@@ -1312,12 +2013,20 @@ namespace {
 		{
 			std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex);
 			ZEN_ASSERT(!ChunkLocations.empty());
-			CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex,
+			const IoHash&	ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex];
+			CompositeBuffer Chunk	  = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex,
 														   ChunkLocations[0].Offset,
 														   Content.ChunkedContent.ChunkRawSizes[ChunkIndex]);
-			ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]);
+			ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash);
+
+			const uint64_t RawSize			   = Chunk.GetSize();
+			const bool	   ShouldCompressChunk = Lookup.RawHashToSequenceIndex.contains(ChunkHash) &&
+											 (RawSize >= MinimumSizeForCompressInBlock) && IsChunkCompressable(Content, Lookup, ChunkIndex);
+			const OodleCompressionLevel CompressionLevel =
+				ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
+
 			CompositeBuffer CompressedChunk =
-				CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed();
+				CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, CompressionLevel).GetCompressed();
 			ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end());
 		}
 		return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers)));
@@ -1328,10 +2037,9 @@ namespace {
 						   const Oid&								 BuildId,
 						   const IoHash&							 ChunkHash,
 						   const std::uint64_t						 PreferredMultipartChunkSize,
-						   ParallellWork&							 Work,
+						   ParallelWork&							 Work,
 						   WorkerThreadPool&						 NetworkPool,
-						   std::atomic<uint64_t>&					 BytesDownloaded,
-						   std::atomic<uint64_t>&					 MultipartAttachmentCount,
+						   DownloadStatistics&						 DownloadStats,
 						   std::function<void(IoBuffer&& Payload)>&& OnDownloadComplete)
 	{
 		ZEN_TRACE_CPU("DownloadLargeBlob");
@@ -1353,56 +2061,89 @@ namespace {
 			BuildId,
 			ChunkHash,
 			PreferredMultipartChunkSize,
-			[Workload, &BytesDownloaded, OnDownloadComplete = std::move(OnDownloadComplete)](uint64_t		 Offset,
-																							 const IoBuffer& Chunk,
-																							 uint64_t		 BytesRemaining) {
-				BytesDownloaded += Chunk.GetSize();
+			[Workload, &DownloadStats](uint64_t Offset, const IoBuffer& Chunk) {
+				DownloadStats.DownloadedChunkByteCount += Chunk.GetSize();
 
 				if (!AbortFlag.load())
 				{
 					ZEN_TRACE_CPU("DownloadLargeBlob_Save");
 					Workload->TempFile.Write(Chunk.GetView(), Offset);
-					if (Chunk.GetSize() == BytesRemaining)
-					{
-						uint64_t PayloadSize = Workload->TempFile.FileSize();
-						void*	 FileHandle	 = Workload->TempFile.Detach();
-						ZEN_ASSERT(FileHandle != nullptr);
-						IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true);
-						Payload.SetDeleteOnClose(true);
-						OnDownloadComplete(std::move(Payload));
-					}
+				}
+			},
+			[Workload, &DownloadStats, OnDownloadComplete = std::move(OnDownloadComplete)]() {
+				DownloadStats.DownloadedChunkCount++;
+				if (!AbortFlag.load())
+				{
+					uint64_t PayloadSize = Workload->TempFile.FileSize();
+					void*	 FileHandle	 = Workload->TempFile.Detach();
+					ZEN_ASSERT(FileHandle != nullptr);
+					IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true);
+					Payload.SetDeleteOnClose(true);
+					OnDownloadComplete(std::move(Payload));
 				}
 			});
 		if (!WorkItems.empty())
 		{
-			MultipartAttachmentCount++;
+			DownloadStats.MultipartAttachmentCount++;
 		}
 		for (auto& WorkItem : WorkItems)
 		{
-			Work.ScheduleWork(
-				NetworkPool,  // GetSyncWorkerPool(),//
-				[WorkItem = std::move(WorkItem)](std::atomic<bool>&) {
-					ZEN_TRACE_CPU("DownloadLargeBlob_Work");
-					if (!AbortFlag)
-					{
-						WorkItem();
-					}
-				},
-				Work.DefaultErrorFunction());
+			Work.ScheduleWork(NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic<bool>&) {
+				ZEN_TRACE_CPU("DownloadLargeBlob_Work");
+				if (!AbortFlag)
+				{
+					WorkItem();
+				}
+			});
 		}
 	}
 
-	void ValidateBuildPart(BuildStorage& Storage, const Oid& BuildId, Oid BuildPartId, const std::string_view BuildPartName)
+	struct ValidateStatistics
+	{
+		uint64_t			  BuildBlobSize			  = 0;
+		uint64_t			  BuildPartSize			  = 0;
+		uint64_t			  ChunkAttachmentCount	  = 0;
+		uint64_t			  BlockAttachmentCount	  = 0;
+		std::atomic<uint64_t> VerifiedAttachmentCount = 0;
+		std::atomic<uint64_t> VerifiedByteCount		  = 0;
+		uint64_t			  ElapsedWallTimeUS		  = 0;
+	};
+
+	void ValidateBuildPart(BuildStorage&		  Storage,
+						   const Oid&			  BuildId,
+						   Oid					  BuildPartId,
+						   const std::string_view BuildPartName,
+						   ValidateStatistics&	  ValidateStats,
+						   DownloadStatistics&	  DownloadStats)
 	{
+		ZEN_TRACE_CPU("ValidateBuildPart");
+
+		ProgressBar::SetLogOperationName(ProgressMode, "Validate Part");
+
+		enum TaskSteps : uint32_t
+		{
+			FetchBuild,
+			FetchBuildPart,
+			ValidateBlobs,
+			Cleanup,
+			StepCount
+		};
+
+		auto EndProgress =
+			MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
+
 		Stopwatch Timer;
-		auto	  _		= MakeGuard([&]() {
+		auto	  _ = MakeGuard([&]() {
 			 ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
 						 BuildId,
 						 BuildPartId,
 						 BuildPartName,
 						 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		 });
-		CbObject  Build = Storage.GetBuild(BuildId);
+
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount);
+
+		CbObject Build = Storage.GetBuild(BuildId);
 		if (!BuildPartName.empty())
 		{
 			BuildPartId = Build["parts"sv].AsObjectView()[BuildPartName].AsObjectId();
@@ -1411,35 +2152,42 @@ namespace {
 				throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", BuildId, BuildPartName));
 			}
 		}
+		ValidateStats.BuildBlobSize			 = Build.GetSize();
 		uint64_t PreferredMultipartChunkSize = DefaultPreferredMultipartChunkSize;
 		if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
 		{
 			PreferredMultipartChunkSize = ChunkSize;
 		}
-		CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId);
+
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuildPart, TaskSteps::StepCount);
+
+		CbObject BuildPart			= Storage.GetBuildPart(BuildId, BuildPartId);
+		ValidateStats.BuildPartSize = BuildPart.GetSize();
 		ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize()));
 		std::vector<IoHash> ChunkAttachments;
 		for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv])
 		{
 			ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment());
 		}
+		ValidateStats.ChunkAttachmentCount = ChunkAttachments.size();
 		std::vector<IoHash> BlockAttachments;
 		for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv])
 		{
 			BlockAttachments.push_back(BlocksView.AsBinaryAttachment());
 		}
+		ValidateStats.BlockAttachmentCount = BlockAttachments.size();
 
-		std::vector<ChunkBlockDescription> VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments);
+		std::vector<ChunkBlockDescription> VerifyBlockDescriptions =
+			ParseChunkBlockDescriptionList(Storage.GetBlockMetadatas(BuildId, BlockAttachments));
 		if (VerifyBlockDescriptions.size() != BlockAttachments.size())
 		{
 			throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing",
 												 BlockAttachments.size() - VerifyBlockDescriptions.size()));
 		}
 
-		WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst);	//  GetSyncWorkerPool();  //
-		WorkerThreadPool& ReadPool	  = GetMediumWorkerPool(EWorkloadType::Burst);	// GetSyncWorkerPool();  //
-		WorkerThreadPool& VerifyPool  = GetMediumWorkerPool(EWorkloadType::Burst);	// GetSyncWorkerPool();  //
-		ParallellWork	  Work(AbortFlag);
+		WorkerThreadPool& NetworkPool = GetNetworkPool();
+		WorkerThreadPool& VerifyPool  = GetIOWorkerPool();
+		ParallelWork	  Work(AbortFlag, PauseFlag);
 
 		const std::filesystem::path TempFolder = ".zen-tmp";
 
@@ -1447,91 +2195,123 @@ namespace {
 		auto __ = MakeGuard([&TempFolder]() {
 			if (CleanDirectory(TempFolder, {}))
 			{
-				std::filesystem::remove(TempFolder);
+				std::error_code DummyEc;
+				RemoveDir(TempFolder, DummyEc);
 			}
 		});
 
-		ProgressBar ProgressBar(UsePlainProgress);
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::ValidateBlobs, TaskSteps::StepCount);
+
+		ProgressBar ProgressBar(ProgressMode, "Validate Blobs");
 
-		uint64_t			  AttachmentsToVerifyCount	= ChunkAttachments.size() + BlockAttachments.size();
-		std::atomic<uint64_t> DownloadedAttachmentCount = 0;
-		std::atomic<uint64_t> VerifiedAttachmentCount	= 0;
-		std::atomic<uint64_t> DownloadedByteCount		= 0;
-		std::atomic<uint64_t> VerifiedByteCount			= 0;
-		FilteredRate		  FilteredDownloadedBytesPerSecond;
-		FilteredRate		  FilteredVerifiedBytesPerSecond;
+		uint64_t	 AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size();
+		FilteredRate FilteredDownloadedBytesPerSecond;
+		FilteredRate FilteredVerifiedBytesPerSecond;
 
 		std::atomic<uint64_t> MultipartAttachmentCount = 0;
 
 		for (const IoHash& ChunkAttachment : ChunkAttachments)
 		{
-			Work.ScheduleWork(
-				ReadPool,
-				[&, ChunkAttachment](std::atomic<bool>&) {
-					if (!AbortFlag)
-					{
-						ZEN_TRACE_CPU("ValidateBuildPart_GetChunk");
-
-						FilteredDownloadedBytesPerSecond.Start();
-						DownloadLargeBlob(Storage,
+			Work.ScheduleWork(NetworkPool,
+							  [&Storage,
+							   &NetworkPool,
+							   &VerifyPool,
+							   &Work,
+							   &DownloadStats,
+							   &ValidateStats,
+							   AttachmentsToVerifyCount,
+							   &TempFolder,
+							   BuildId = Oid(BuildId),
+							   PreferredMultipartChunkSize,
+							   &FilteredDownloadedBytesPerSecond,
+							   &FilteredVerifiedBytesPerSecond,
+							   ChunkAttachment](std::atomic<bool>&) {
+								  if (!AbortFlag)
+								  {
+									  ZEN_TRACE_CPU("ValidateBuildPart_GetChunk");
+
+									  FilteredDownloadedBytesPerSecond.Start();
+									  DownloadLargeBlob(
+										  Storage,
 										  TempFolder,
 										  BuildId,
 										  ChunkAttachment,
 										  PreferredMultipartChunkSize,
 										  Work,
 										  NetworkPool,
-										  DownloadedByteCount,
-										  MultipartAttachmentCount,
-										  [&, ChunkHash = ChunkAttachment](IoBuffer&& Payload) {
+										  DownloadStats,
+										  [&Work,
+										   &VerifyPool,
+										   &DownloadStats,
+										   &ValidateStats,
+										   AttachmentsToVerifyCount,
+										   &FilteredDownloadedBytesPerSecond,
+										   &FilteredVerifiedBytesPerSecond,
+										   ChunkHash = ChunkAttachment](IoBuffer&& Payload) {
 											  Payload.SetContentType(ZenContentType::kCompressedBinary);
 											  if (!AbortFlag)
 											  {
 												  Work.ScheduleWork(
 													  VerifyPool,
-													  [&, Payload = std::move(Payload), ChunkHash](std::atomic<bool>&) mutable {
+													  [&DownloadStats,
+													   &ValidateStats,
+													   AttachmentsToVerifyCount,
+													   &FilteredDownloadedBytesPerSecond,
+													   &FilteredVerifiedBytesPerSecond,
+													   Payload = std::move(Payload),
+													   ChunkHash](std::atomic<bool>&) mutable {
 														  if (!AbortFlag)
 														  {
 															  ZEN_TRACE_CPU("ValidateBuildPart_Validate");
 
+															  if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount ==
+																  AttachmentsToVerifyCount)
+															  {
+																  FilteredDownloadedBytesPerSecond.Stop();
+															  }
+
 															  FilteredVerifiedBytesPerSecond.Start();
 
 															  uint64_t CompressedSize;
 															  uint64_t DecompressedSize;
 															  ValidateBlob(std::move(Payload), ChunkHash, CompressedSize, DecompressedSize);
-															  ZEN_CONSOLE_VERBOSE("Chunk attachment {} ({} -> {}) is valid",
-																				  ChunkHash,
-																				  NiceBytes(CompressedSize),
-																				  NiceBytes(DecompressedSize));
-															  VerifiedAttachmentCount++;
-															  VerifiedByteCount += DecompressedSize;
-															  if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount)
+															  ValidateStats.VerifiedAttachmentCount++;
+															  ValidateStats.VerifiedByteCount += DecompressedSize;
+															  if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount)
 															  {
 																  FilteredVerifiedBytesPerSecond.Stop();
 															  }
 														  }
-													  },
-													  Work.DefaultErrorFunction());
+													  });
 											  }
 										  });
-					}
-				},
-				Work.DefaultErrorFunction());
+								  }
+							  });
 		}
 
 		for (const IoHash& BlockAttachment : BlockAttachments)
 		{
 			Work.ScheduleWork(
 				NetworkPool,
-				[&, BlockAttachment](std::atomic<bool>&) {
+				[&Storage,
+				 &BuildId,
+				 &Work,
+				 &VerifyPool,
+				 &DownloadStats,
+				 &ValidateStats,
+				 AttachmentsToVerifyCount,
+				 &FilteredDownloadedBytesPerSecond,
+				 &FilteredVerifiedBytesPerSecond,
+				 BlockAttachment](std::atomic<bool>&) {
 					if (!AbortFlag)
 					{
 						ZEN_TRACE_CPU("ValidateBuildPart_GetBlock");
 
 						FilteredDownloadedBytesPerSecond.Start();
 						IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment);
-						DownloadedAttachmentCount++;
-						DownloadedByteCount += Payload.GetSize();
-						if (DownloadedAttachmentCount.load() == AttachmentsToVerifyCount)
+						DownloadStats.DownloadedBlockCount++;
+						DownloadStats.DownloadedBlockByteCount += Payload.GetSize();
+						if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == AttachmentsToVerifyCount)
 						{
 							FilteredDownloadedBytesPerSecond.Stop();
 						}
@@ -1543,7 +2323,11 @@ namespace {
 						{
 							Work.ScheduleWork(
 								VerifyPool,
-								[&, Payload = std::move(Payload), BlockAttachment](std::atomic<bool>&) mutable {
+								[&FilteredVerifiedBytesPerSecond,
+								 AttachmentsToVerifyCount,
+								 &ValidateStats,
+								 Payload = std::move(Payload),
+								 BlockAttachment](std::atomic<bool>&) mutable {
 									if (!AbortFlag)
 									{
 										ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock");
@@ -1553,39 +2337,36 @@ namespace {
 										uint64_t CompressedSize;
 										uint64_t DecompressedSize;
 										ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize);
-										ZEN_CONSOLE_VERBOSE("Chunk block {} ({} -> {}) is valid",
-															BlockAttachment,
-															NiceBytes(CompressedSize),
-															NiceBytes(DecompressedSize));
-										VerifiedAttachmentCount++;
-										VerifiedByteCount += DecompressedSize;
-										if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount)
+										ValidateStats.VerifiedAttachmentCount++;
+										ValidateStats.VerifiedByteCount += DecompressedSize;
+										if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount)
 										{
 											FilteredVerifiedBytesPerSecond.Stop();
 										}
 									}
-								},
-								Work.DefaultErrorFunction());
+								});
 						}
 					}
-				},
-				Work.DefaultErrorFunction());
+				});
 		}
 
-		Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-			ZEN_UNUSED(IsAborted, PendingWork);
+		Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+			ZEN_UNUSED(PendingWork);
+
+			const uint64_t DownloadedAttachmentCount = DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount;
+			const uint64_t DownloadedByteCount		 = DownloadStats.DownloadedChunkByteCount + DownloadStats.DownloadedBlockByteCount;
 
 			FilteredDownloadedBytesPerSecond.Update(DownloadedByteCount);
-			FilteredVerifiedBytesPerSecond.Update(VerifiedByteCount);
+			FilteredVerifiedBytesPerSecond.Update(ValidateStats.VerifiedByteCount);
 
 			std::string Details = fmt::format("Downloaded {}/{} ({}, {}bits/s). Verified {}/{} ({}, {}B/s)",
-											  DownloadedAttachmentCount.load(),
+											  DownloadedAttachmentCount,
 											  AttachmentsToVerifyCount,
-											  NiceBytes(DownloadedByteCount.load()),
+											  NiceBytes(DownloadedByteCount),
 											  NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8),
-											  VerifiedAttachmentCount.load(),
+											  ValidateStats.VerifiedAttachmentCount.load(),
 											  AttachmentsToVerifyCount,
-											  NiceBytes(VerifiedByteCount.load()),
+											  NiceBytes(ValidateStats.VerifiedByteCount.load()),
 											  NiceNum(FilteredVerifiedBytesPerSecond.GetCurrent()));
 
 			ProgressBar.UpdateState(
@@ -1593,11 +2374,15 @@ namespace {
 				 .Details		 = Details,
 				 .TotalCount	 = gsl::narrow<uint64_t>(AttachmentsToVerifyCount * 2),
 				 .RemainingCount = gsl::narrow<uint64_t>(AttachmentsToVerifyCount * 2 -
-														 (DownloadedAttachmentCount.load() + VerifiedAttachmentCount.load()))},
+														 (DownloadedAttachmentCount + ValidateStats.VerifiedAttachmentCount.load())),
+				 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
 				false);
 		});
 
 		ProgressBar.Finish();
+		ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs();
+
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
 	}
 
 	void ArrangeChunksIntoBlocks(const ChunkedFolderContent&		 Content,
@@ -1698,7 +2483,8 @@ namespace {
 								  const ChunkedFolderContent&  Content,
 								  const ChunkedContentLookup&  Lookup,
 								  uint32_t					   ChunkIndex,
-								  const std::filesystem::path& TempFolderPath)
+								  const std::filesystem::path& TempFolderPath,
+								  LooseChunksStatistics&	   LooseChunksStats)
 	{
 		ZEN_TRACE_CPU("CompressChunk");
 		ZEN_ASSERT(!TempFolderPath.empty());
@@ -1716,22 +2502,38 @@ namespace {
 		{
 			throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash));
 		}
-		ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash);
+
+		const bool					ShouldCompressChunk = IsChunkCompressable(Content, Lookup, ChunkIndex);
+		const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
+
+		if (ShouldCompressChunk)
 		{
-			std::filesystem::path TempFilePath = (TempFolderPath / ChunkHash.ToHexString()).make_preferred();
+			std::filesystem::path TempFilePath = TempFolderPath / ChunkHash.ToHexString();
 
 			BasicFile		CompressedFile;
 			std::error_code Ec;
-			CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncate, Ec);
+			CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec);
 			if (Ec)
 			{
 				throw std::runtime_error(
 					fmt::format("Failed creating temporary file for compressing blob {}. Reason: {}", ChunkHash, Ec.message()));
 			}
 
+			uint64_t StreamRawBytes		   = 0;
+			uint64_t StreamCompressedBytes = 0;
+
 			bool CouldCompress = CompressedBuffer::CompressToStream(
 				CompositeBuffer(SharedBuffer(RawSource)),
-				[&](uint64_t Offset, const CompositeBuffer& RangeBuffer) { CompressedFile.Write(RangeBuffer, Offset); });
+				[&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+					ZEN_UNUSED(SourceOffset);
+					LooseChunksStats.CompressedChunkRawBytes += SourceSize;
+					CompressedFile.Write(RangeBuffer, Offset);
+					LooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize();
+					StreamRawBytes += SourceSize;
+					StreamCompressedBytes += RangeBuffer.GetSize();
+				},
+				OodleCompressor::Mermaid,
+				CompressionLevel);
 			if (CouldCompress)
 			{
 				uint64_t CompressedSize = CompressedFile.FileSize();
@@ -1749,24 +2551,41 @@ namespace {
 				ZEN_ASSERT(Compressed);
 				ZEN_ASSERT(RawHash == ChunkHash);
 				ZEN_ASSERT(RawSize == ChunkSize);
+
+				LooseChunksStats.CompressedChunkCount++;
+
 				return Compressed.GetCompressed();
 			}
+			else
+			{
+				LooseChunksStats.CompressedChunkRawBytes -= StreamRawBytes;
+				LooseChunksStats.CompressedChunkBytes -= StreamCompressedBytes;
+			}
 			CompressedFile.Close();
-			std::filesystem::remove(TempFilePath, Ec);
+			RemoveFile(TempFilePath, Ec);
 			ZEN_UNUSED(Ec);
 		}
 
-		// Try regular compress - decompress may fail if compressed data is larger than non-compressed
-		CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(RawSource)));
+		CompressedBuffer CompressedBlob =
+			CompressedBuffer::Compress(SharedBuffer(std::move(RawSource)), OodleCompressor::Mermaid, CompressionLevel);
 		if (!CompressedBlob)
 		{
 			throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash));
 		}
-		if (!IsBufferDiskBased(CompressedBlob.GetCompressed()))
+		ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawHash() == ChunkHash);
+		ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawSize() == ChunkSize);
+
+		LooseChunksStats.CompressedChunkRawBytes += ChunkSize;
+		LooseChunksStats.CompressedChunkBytes += CompressedBlob.GetCompressedSize();
+
+		// If we use none-compression, the compressed blob references the data and has 64 kb in memory so we don't need to write it to disk
+		if (ShouldCompressChunk)
 		{
 			IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlob).GetCompressed(), TempFolderPath, ChunkHash);
 			CompressedBlob		 = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload));
 		}
+
+		LooseChunksStats.CompressedChunkCount++;
 		return std::move(CompressedBlob).GetCompressed();
 	}
 
@@ -1783,7 +2602,7 @@ namespace {
 	void GenerateBuildBlocks(const std::filesystem::path&			   Path,
 							 const ChunkedFolderContent&			   Content,
 							 const ChunkedContentLookup&			   Lookup,
-							 BuildStorage&							   Storage,
+							 StorageInstance&						   Storage,
 							 const Oid&								   BuildId,
 							 const std::vector<std::vector<uint32_t>>& NewBlockChunks,
 							 GeneratedBlocks&						   OutBlocks,
@@ -1795,7 +2614,7 @@ namespace {
 		const std::size_t NewBlockCount = NewBlockChunks.size();
 		if (NewBlockCount > 0)
 		{
-			ProgressBar ProgressBar(UsePlainProgress);
+			ProgressBar ProgressBar(ProgressMode, "Generate Blocks");
 
 			OutBlocks.BlockDescriptions.resize(NewBlockCount);
 			OutBlocks.BlockSizes.resize(NewBlockCount);
@@ -1806,14 +2625,13 @@ namespace {
 
 			RwLock Lock;
 
-			WorkerThreadPool& GenerateBlobsPool =
-				GetMediumWorkerPool(EWorkloadType::Burst);	// GetMediumWorkerPool(EWorkloadType::Burst);  // GetSyncWorkerPool();//
-			WorkerThreadPool& UploadBlocksPool = GetSmallWorkerPool(EWorkloadType::Burst);	// GetSyncWorkerPool();//
+			WorkerThreadPool& GenerateBlobsPool = GetIOWorkerPool();
+			WorkerThreadPool& UploadBlocksPool	= GetNetworkPool();
 
 			FilteredRate FilteredGeneratedBytesPerSecond;
 			FilteredRate FilteredUploadedBytesPerSecond;
 
-			ParallellWork Work(AbortFlag);
+			ParallelWork Work(AbortFlag, PauseFlag);
 
 			std::atomic<uint64_t> QueuedPendingBlocksForUpload = 0;
 
@@ -1826,7 +2644,24 @@ namespace {
 				const std::vector<uint32_t>& ChunksInBlock = NewBlockChunks[BlockIndex];
 				Work.ScheduleWork(
 					GenerateBlobsPool,
-					[&, BlockIndex](std::atomic<bool>&) {
+					[&Storage,
+					 BuildId,
+					 &Path,
+					 &Content,
+					 &Lookup,
+					 &Work,
+					 &UploadBlocksPool,
+					 NewBlockCount,
+					 ChunksInBlock,
+					 &Lock,
+					 &OutBlocks,
+					 &DiskStats,
+					 &GenerateBlocksStats,
+					 &UploadStats,
+					 &FilteredGeneratedBytesPerSecond,
+					 &QueuedPendingBlocksForUpload,
+					 &FilteredUploadedBytesPerSecond,
+					 BlockIndex](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("GenerateBuildBlocks_Generate");
@@ -1834,12 +2669,14 @@ namespace {
 							FilteredGeneratedBytesPerSecond.Start();
 							// TODO: Convert ScheduleWork body to function
 
+							Stopwatch		 GenerateTimer;
 							CompressedBuffer CompressedBlock =
 								GenerateBlock(Path, Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex], DiskStats);
-							ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks",
+							ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks in {}",
 												OutBlocks.BlockDescriptions[BlockIndex].BlockHash,
 												NiceBytes(CompressedBlock.GetCompressedSize()),
-												OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
+												OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(),
+												NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs()));
 
 							OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize();
 							{
@@ -1880,7 +2717,16 @@ namespace {
 
 									Work.ScheduleWork(
 										UploadBlocksPool,
-										[&, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic<bool>&) mutable {
+										[&Storage,
+										 BuildId,
+										 NewBlockCount,
+										 &FilteredUploadedBytesPerSecond,
+										 &QueuedPendingBlocksForUpload,
+										 &GenerateBlocksStats,
+										 &UploadStats,
+										 &OutBlocks,
+										 BlockIndex,
+										 Payload = std::move(CompressedBlock)](std::atomic<bool>&) mutable {
 											auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; });
 											if (!AbortFlag)
 											{
@@ -1907,26 +2753,44 @@ namespace {
 													const IoHash&  BlockHash		   = OutBlocks.BlockDescriptions[BlockIndex].BlockHash;
 													const uint64_t CompressedBlockSize = Payload.GetCompressedSize();
 
-													Storage.PutBuildBlob(BuildId,
-																		 BlockHash,
-																		 ZenContentType::kCompressedBinary,
-																		 std::move(Payload).GetCompressed());
+													if (Storage.BuildCacheStorage)
+													{
+														Storage.BuildCacheStorage->PutBuildBlob(BuildId,
+																								BlockHash,
+																								ZenContentType::kCompressedBinary,
+																								Payload.GetCompressed());
+													}
+
+													Storage.BuildStorage->PutBuildBlob(BuildId,
+																					   BlockHash,
+																					   ZenContentType::kCompressedBinary,
+																					   std::move(Payload).GetCompressed());
 													UploadStats.BlocksBytes += CompressedBlockSize;
+
 													ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks",
-																		OutBlocks.BlockDescriptions[BlockIndex].BlockHash,
+																		BlockHash,
 																		NiceBytes(CompressedBlockSize),
 																		OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
 
-													Storage.PutBlockMetadata(BuildId,
-																			 OutBlocks.BlockDescriptions[BlockIndex].BlockHash,
-																			 BlockMetaData);
-													ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})",
-																		OutBlocks.BlockDescriptions[BlockIndex].BlockHash,
-																		NiceBytes(BlockMetaData.GetSize()));
+													if (Storage.BuildCacheStorage)
+													{
+														Storage.BuildCacheStorage->PutBlobMetadatas(BuildId,
+																									std::vector<IoHash>({BlockHash}),
+																									std::vector<CbObject>({BlockMetaData}));
+													}
+
+													bool MetadataSucceeded =
+														Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
+													if (MetadataSucceeded)
+													{
+														ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})",
+																			BlockHash,
+																			NiceBytes(BlockMetaData.GetSize()));
 
-													OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+														OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+														UploadStats.BlocksBytes += BlockMetaData.GetSize();
+													}
 
-													UploadStats.BlocksBytes += BlockMetaData.GetSize();
 													UploadStats.BlockCount++;
 													if (UploadStats.BlockCount == NewBlockCount)
 													{
@@ -1934,17 +2798,15 @@ namespace {
 													}
 												}
 											}
-										},
-										Work.DefaultErrorFunction());
+										});
 								}
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			}
 
-			Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-				ZEN_UNUSED(IsAborted, PendingWork);
+			Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+				ZEN_UNUSED(PendingWork);
 
 				FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load());
 				FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load());
@@ -1963,7 +2825,8 @@ namespace {
 					{.Task			 = "Generating blocks",
 					 .Details		 = Details,
 					 .TotalCount	 = gsl::narrow<uint64_t>(NewBlockCount),
-					 .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load())},
+					 .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()),
+					 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
 					false);
 			});
 
@@ -1976,9 +2839,10 @@ namespace {
 		}
 	}
 
-	void UploadPartBlobs(BuildStorage&							   Storage,
+	void UploadPartBlobs(StorageInstance&						   Storage,
 						 const Oid&								   BuildId,
 						 const std::filesystem::path&			   Path,
+						 const std::filesystem::path&			   TempDir,
 						 const ChunkedFolderContent&			   Content,
 						 const ChunkedContentLookup&			   Lookup,
 						 std::span<IoHash>						   RawHashes,
@@ -1988,21 +2852,20 @@ namespace {
 						 const std::uint64_t					   LargeAttachmentSize,
 						 DiskStatistics&						   DiskStats,
 						 UploadStatistics&						   UploadStats,
-						 GenerateBlocksStatistics&				   GenerateBlocksStats,
 						 LooseChunksStatistics&					   LooseChunksStats)
 	{
 		ZEN_TRACE_CPU("UploadPartBlobs");
 		{
-			ProgressBar ProgressBar(UsePlainProgress);
+			ProgressBar ProgressBar(ProgressMode, "Upload Blobs");
 
-			WorkerThreadPool& ReadChunkPool	  = GetMediumWorkerPool(EWorkloadType::Burst);	// GetSyncWorkerPool();  //
-			WorkerThreadPool& UploadChunkPool = GetSmallWorkerPool(EWorkloadType::Burst);	// GetSyncWorkerPool();  //
+			WorkerThreadPool& ReadChunkPool	  = GetIOWorkerPool();
+			WorkerThreadPool& UploadChunkPool = GetNetworkPool();
 
 			FilteredRate FilteredGenerateBlockBytesPerSecond;
 			FilteredRate FilteredCompressedBytesPerSecond;
 			FilteredRate FilteredUploadedBytesPerSecond;
 
-			ParallellWork Work(AbortFlag);
+			ParallelWork Work(AbortFlag, PauseFlag);
 
 			std::atomic<size_t>	  UploadedBlockSize			  = 0;
 			std::atomic<size_t>	  UploadedBlockCount		  = 0;
@@ -2029,7 +2892,7 @@ namespace {
 					BlockIndexes.push_back(It->second);
 					TotalBlocksSize += NewBlocks.BlockSizes[It->second];
 				}
-				if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end())
+				else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end())
 				{
 					const uint32_t ChunkIndex = ChunkIndexIt->second;
 					if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex);
@@ -2039,21 +2902,38 @@ namespace {
 						TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
 					}
 				}
+				else
+				{
+					throw std::runtime_error(
+						fmt::format("Can not upload requested build blob {} as it was not generated by this upload", RawHash));
+				}
 			}
 			uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize;
 
 			const size_t   UploadBlockCount = BlockIndexes.size();
 			const uint32_t UploadChunkCount = gsl::narrow<uint32_t>(LooseChunkOrderIndexes.size());
 
-			auto AsyncUploadBlock = [&](const size_t		   BlockIndex,
-										const IoHash		   BlockHash,
-										CompositeBuffer&&	   Payload,
-										std::atomic<uint64_t>& QueuedPendingInMemoryBlocksForUpload) {
+			auto AsyncUploadBlock = [&Storage,
+									 &BuildId,
+									 &Work,
+									 &TempDir,
+									 &NewBlocks,
+									 UploadBlockCount,
+									 &UploadedBlockCount,
+									 UploadChunkCount,
+									 &UploadedChunkCount,
+									 &UploadedBlockSize,
+									 &UploadStats,
+									 &FilteredUploadedBytesPerSecond,
+									 &UploadChunkPool](const size_t			  BlockIndex,
+													   const IoHash			  BlockHash,
+													   CompositeBuffer&&	  Payload,
+													   std::atomic<uint64_t>& QueuedPendingInMemoryBlocksForUpload) {
 				bool IsInMemoryBlock = true;
 				if (QueuedPendingInMemoryBlocksForUpload.load() > 16)
 				{
 					ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock");
-					Payload			= CompositeBuffer(WriteToTempFile(std::move(Payload), Path / ZenTempBlockFolderName, BlockHash));
+					Payload			= CompositeBuffer(WriteToTempFile(std::move(Payload), TempDir, BlockHash));
 					IsInMemoryBlock = false;
 				}
 				else
@@ -2063,7 +2943,21 @@ namespace {
 
 				Work.ScheduleWork(
 					UploadChunkPool,
-					[&, IsInMemoryBlock, BlockIndex, BlockHash, Payload = std::move(Payload)](std::atomic<bool>&) mutable {
+					[&Storage,
+					 &BuildId,
+					 &QueuedPendingInMemoryBlocksForUpload,
+					 &NewBlocks,
+					 UploadBlockCount,
+					 &UploadedBlockCount,
+					 UploadChunkCount,
+					 &UploadedChunkCount,
+					 &UploadedBlockSize,
+					 &UploadStats,
+					 &FilteredUploadedBytesPerSecond,
+					 IsInMemoryBlock,
+					 BlockIndex,
+					 BlockHash,
+					 Payload = std::move(Payload)](std::atomic<bool>&) mutable {
 						auto _ = MakeGuard([IsInMemoryBlock, &QueuedPendingInMemoryBlocksForUpload] {
 							if (IsInMemoryBlock)
 							{
@@ -2080,23 +2974,34 @@ namespace {
 							const CbObject BlockMetaData =
 								BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
 
-							Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
+							if (Storage.BuildCacheStorage)
+							{
+								Storage.BuildCacheStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
+							}
+							Storage.BuildStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
 							ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks",
-												NewBlocks.BlockDescriptions[BlockIndex].BlockHash,
+												BlockHash,
 												NiceBytes(PayloadSize),
 												NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
 							UploadedBlockSize += PayloadSize;
 							UploadStats.BlocksBytes += PayloadSize;
 
-							Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
-							ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})",
-												NewBlocks.BlockDescriptions[BlockIndex].BlockHash,
-												NiceBytes(BlockMetaData.GetSize()));
+							if (Storage.BuildCacheStorage)
+							{
+								Storage.BuildCacheStorage->PutBlobMetadatas(BuildId,
+																			std::vector<IoHash>({BlockHash}),
+																			std::vector<CbObject>({BlockMetaData}));
+							}
+							bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
+							if (MetadataSucceeded)
+							{
+								ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize()));
 
-							NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+								NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+								UploadStats.BlocksBytes += BlockMetaData.GetSize();
+							}
 
 							UploadStats.BlockCount++;
-							UploadStats.BlocksBytes += BlockMetaData.GetSize();
 
 							UploadedBlockCount++;
 							if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount)
@@ -2104,25 +3009,56 @@ namespace {
 								FilteredUploadedBytesPerSecond.Stop();
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			};
 
-			auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) {
+			auto AsyncUploadLooseChunk = [&Storage,
+										  BuildId,
+										  LargeAttachmentSize,
+										  &Work,
+										  &UploadChunkPool,
+										  &FilteredUploadedBytesPerSecond,
+										  &UploadedBlockCount,
+										  &UploadedChunkCount,
+										  UploadBlockCount,
+										  UploadChunkCount,
+										  &UploadedCompressedChunkSize,
+										  &UploadedRawChunkSize,
+										  &UploadStats](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) {
 				Work.ScheduleWork(
 					UploadChunkPool,
-					[&, RawHash, RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic<bool>&) mutable {
+					[&Storage,
+					 BuildId,
+					 &Work,
+					 LargeAttachmentSize,
+					 &FilteredUploadedBytesPerSecond,
+					 &UploadChunkPool,
+					 &UploadedBlockCount,
+					 &UploadedChunkCount,
+					 UploadBlockCount,
+					 UploadChunkCount,
+					 &UploadedCompressedChunkSize,
+					 &UploadedRawChunkSize,
+					 &UploadStats,
+					 RawHash,
+					 RawSize,
+					 Payload = CompositeBuffer(std::move(Payload))](std::atomic<bool>&) mutable {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("AsyncUploadLooseChunk");
 
 							const uint64_t PayloadSize = Payload.GetSize();
-							;
+
+							if (Storage.BuildCacheStorage)
+							{
+								Storage.BuildCacheStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
+							}
+
 							if (PayloadSize >= LargeAttachmentSize)
 							{
 								ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart");
 								UploadStats.MultipartAttachmentCount++;
-								std::vector<std::function<void()>> MultipartWork = Storage.PutLargeBuildBlob(
+								std::vector<std::function<void()>> MultipartWork = Storage.BuildStorage->PutLargeBuildBlob(
 									BuildId,
 									RawHash,
 									ZenContentType::kCompressedBinary,
@@ -2135,7 +3071,16 @@ namespace {
 										PartPayload.SetContentType(ZenContentType::kBinary);
 										return PartPayload;
 									},
-									[&, RawSize](uint64_t SentBytes, bool IsComplete) {
+									[RawSize,
+									 &UploadStats,
+									 &UploadedCompressedChunkSize,
+									 &UploadChunkPool,
+									 &UploadedBlockCount,
+									 UploadBlockCount,
+									 &UploadedChunkCount,
+									 UploadChunkCount,
+									 &FilteredUploadedBytesPerSecond,
+									 &UploadedRawChunkSize](uint64_t SentBytes, bool IsComplete) {
 										UploadStats.ChunksBytes += SentBytes;
 										UploadedCompressedChunkSize += SentBytes;
 										if (IsComplete)
@@ -2151,23 +3096,20 @@ namespace {
 									});
 								for (auto& WorkPart : MultipartWork)
 								{
-									Work.ScheduleWork(
-										UploadChunkPool,
-										[Work = std::move(WorkPart)](std::atomic<bool>&) {
-											ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work");
-											if (!AbortFlag)
-											{
-												Work();
-											}
-										},
-										Work.DefaultErrorFunction());
+									Work.ScheduleWork(UploadChunkPool, [Work = std::move(WorkPart)](std::atomic<bool>&) {
+										ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work");
+										if (!AbortFlag)
+										{
+											Work();
+										}
+									});
 								}
 								ZEN_CONSOLE_VERBOSE("Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize));
 							}
 							else
 							{
 								ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart");
-								Storage.PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
+								Storage.BuildStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
 								ZEN_CONSOLE_VERBOSE("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize));
 								UploadStats.ChunksBytes += Payload.GetSize();
 								UploadStats.ChunkCount++;
@@ -2180,8 +3122,7 @@ namespace {
 								}
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			};
 
 			std::vector<size_t> GenerateBlockIndexes;
@@ -2189,16 +3130,8 @@ namespace {
 			std::atomic<uint64_t> GeneratedBlockCount	  = 0;
 			std::atomic<uint64_t> GeneratedBlockByteCount = 0;
 
-			std::vector<uint32_t> CompressLooseChunkOrderIndexes;
-
 			std::atomic<uint64_t> QueuedPendingInMemoryBlocksForUpload = 0;
 
-			// Start upload of any pre-compressed loose chunks
-			for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes)
-			{
-				CompressLooseChunkOrderIndexes.push_back(LooseChunkOrderIndex);
-			}
-
 			// Start generation of any non-prebuilt blocks and schedule upload
 			for (const size_t BlockIndex : BlockIndexes)
 			{
@@ -2206,14 +3139,28 @@ namespace {
 				if (!AbortFlag)
 				{
 					Work.ScheduleWork(
-						ReadChunkPool,	// GetSyncWorkerPool()
-						[&, BlockIndex](std::atomic<bool>&) {
+						ReadChunkPool,
+						[BlockHash = IoHash(BlockHash),
+						 BlockIndex,
+						 &FilteredGenerateBlockBytesPerSecond,
+						 Path,
+						 &Content,
+						 &Lookup,
+						 &NewBlocks,
+						 &NewBlockChunks,
+						 &GenerateBlockIndexes,
+						 &GeneratedBlockCount,
+						 &GeneratedBlockByteCount,
+						 &DiskStats,
+						 &AsyncUploadBlock,
+						 &QueuedPendingInMemoryBlocksForUpload](std::atomic<bool>&) {
 							if (!AbortFlag)
 							{
 								ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock");
 
 								FilteredGenerateBlockBytesPerSecond.Start();
 
+								Stopwatch		GenerateTimer;
 								CompositeBuffer Payload;
 								if (NewBlocks.BlockHeaders[BlockIndex])
 								{
@@ -2238,57 +3185,58 @@ namespace {
 									Payload = std::move(CompressedBlock).GetCompressed();
 								}
 
-								GenerateBlocksStats.GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex];
-								GenerateBlocksStats.GeneratedBlockCount++;
 								GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex];
 								GeneratedBlockCount++;
 								if (GeneratedBlockCount == GenerateBlockIndexes.size())
 								{
 									FilteredGenerateBlockBytesPerSecond.Stop();
 								}
+								ZEN_CONSOLE_VERBOSE("{} block {} ({}) containing {} chunks in {}",
+													NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated",
+													NewBlocks.BlockDescriptions[BlockIndex].BlockHash,
+													NiceBytes(NewBlocks.BlockSizes[BlockIndex]),
+													NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(),
+													NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs()));
 								if (!AbortFlag)
 								{
 									AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload);
 								}
-								ZEN_CONSOLE_VERBOSE("Regenerated block {} ({}) containing {} chunks",
-													NewBlocks.BlockDescriptions[BlockIndex].BlockHash,
-													NiceBytes(NewBlocks.BlockSizes[BlockIndex]),
-													NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
 							}
-						},
-						Work.DefaultErrorFunction());
+						});
 				}
 			}
 
-			std::atomic<uint64_t> CompressedLooseChunkCount		= 0;
-			std::atomic<uint64_t> CompressedLooseChunkByteCount = 0;
-			std::atomic<uint64_t> RawLooseChunkByteCount		= 0;
-
 			// Start compression of any non-precompressed loose chunks and schedule upload
-			for (const uint32_t CompressLooseChunkOrderIndex : CompressLooseChunkOrderIndexes)
+			for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes)
 			{
-				const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex];
+				const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex];
 				Work.ScheduleWork(
-					ReadChunkPool,	// GetSyncWorkerPool(),// ReadChunkPool,
-					[&, ChunkIndex](std::atomic<bool>&) {
+					ReadChunkPool,
+					[&Path,
+					 &Content,
+					 &Lookup,
+					 &TempDir,
+					 &LooseChunksStats,
+					 &LooseChunkOrderIndexes,
+					 &FilteredCompressedBytesPerSecond,
+					 &UploadStats,
+					 &AsyncUploadLooseChunk,
+					 ChunkIndex](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk");
 
 							FilteredCompressedBytesPerSecond.Start();
-							CompositeBuffer Payload = CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName);
-							ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})",
+							Stopwatch		CompressTimer;
+							CompositeBuffer Payload = CompressChunk(Path, Content, Lookup, ChunkIndex, TempDir, LooseChunksStats);
+							ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {}) in {}",
 												Content.ChunkedContent.ChunkHashes[ChunkIndex],
 												NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]),
-												NiceBytes(Payload.GetSize()));
+												NiceBytes(Payload.GetSize()),
+												NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs()));
 							const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
 							UploadStats.ReadFromDiskBytes += ChunkRawSize;
-							LooseChunksStats.CompressedChunkBytes += Payload.GetSize();
-							LooseChunksStats.CompressedChunkCount++;
-							CompressedLooseChunkByteCount += Payload.GetSize();
-							CompressedLooseChunkCount++;
-							RawLooseChunkByteCount += ChunkRawSize;
-							if (CompressedLooseChunkCount == CompressLooseChunkOrderIndexes.size())
+							if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size())
 							{
 								FilteredCompressedBytesPerSecond.Stop();
 							}
@@ -2297,26 +3245,26 @@ namespace {
 								AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkRawSize, std::move(Payload));
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			}
 
-			Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-				ZEN_UNUSED(IsAborted, PendingWork);
-				FilteredCompressedBytesPerSecond.Update(CompressedLooseChunkByteCount.load());
+			Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+				ZEN_UNUSED(PendingWork);
+				FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkRawBytes.load());
 				FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load());
 				FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load());
 				uint64_t UploadedRawSize		= UploadedRawChunkSize.load() + UploadedBlockSize.load();
 				uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load();
 
 				std::string Details = fmt::format(
-					"Compressed {}/{} ({}/{}) chunks. "
+					"Compressed {}/{} ({}/{} {}B/s) chunks. "
 					"Uploaded {}/{} ({}/{}) blobs "
 					"({} {}bits/s)",
-					CompressedLooseChunkCount.load(),
-					CompressLooseChunkOrderIndexes.size(),
-					NiceBytes(RawLooseChunkByteCount),
+					LooseChunksStats.CompressedChunkCount.load(),
+					LooseChunkOrderIndexes.size(),
+					NiceBytes(LooseChunksStats.CompressedChunkRawBytes),
 					NiceBytes(TotalLooseChunksSize),
+					NiceNum(FilteredCompressedBytesPerSecond.GetCurrent()),
 
 					UploadedBlockCount.load() + UploadedChunkCount.load(),
 					UploadBlockCount + UploadChunkCount,
@@ -2329,16 +3277,17 @@ namespace {
 				ProgressBar.UpdateState({.Task			 = "Uploading blobs  ",
 										 .Details		 = Details,
 										 .TotalCount	 = gsl::narrow<uint64_t>(TotalRawSize),
-										 .RemainingCount = gsl::narrow<uint64_t>(TotalRawSize - UploadedRawSize)},
+										 .RemainingCount = gsl::narrow<uint64_t>(TotalRawSize - UploadedRawSize),
+										 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
 										false);
 			});
 
 			ZEN_ASSERT(AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0);
 
 			ProgressBar.Finish();
-			UploadStats.ElapsedWallTimeUS						= FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
-			GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGenerateBlockBytesPerSecond.GetElapsedTimeUS();
-			LooseChunksStats.CompressChunksElapsedWallTimeUS	= FilteredCompressedBytesPerSecond.GetElapsedTimeUS();
+
+			UploadStats.ElapsedWallTimeUS					 = FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
+			LooseChunksStats.CompressChunksElapsedWallTimeUS = FilteredCompressedBytesPerSecond.GetElapsedTimeUS();
 		}
 	}
 
@@ -2424,10 +3373,8 @@ namespace {
 						}
 						else if (FoundAttachmentCount > 0)
 						{
-							ZEN_CONSOLE_VERBOSE("Skipping block {}. {} attachments found, usage level: {}%",
-												KnownBlock.BlockHash,
-												FoundAttachmentCount,
-												ReusePercent);
+							// ZEN_CONSOLE_VERBOSE("Skipping block {}. {} attachments found, usage level: {}%", KnownBlock.BlockHash,
+							// FoundAttachmentCount, ReusePercent);
 							FindBlocksStats.RejectedBlockCount++;
 							FindBlocksStats.RejectedChunkCount += FoundAttachmentCount;
 							FindBlocksStats.RejectedByteCount += ReuseSize;
@@ -2444,9 +3391,10 @@ namespace {
 					for (size_t KnownBlockIndex : ReuseBlockIndexes)
 					{
 						std::vector<uint32_t>		 FoundChunkIndexes;
-						size_t						 BlockSize		   = 0;
-						size_t						 AdjustedReuseSize = 0;
-						const ChunkBlockDescription& KnownBlock		   = KnownBlocks[KnownBlockIndex];
+						size_t						 BlockSize			  = 0;
+						size_t						 AdjustedReuseSize	  = 0;
+						size_t						 AdjustedRawReuseSize = 0;
+						const ChunkBlockDescription& KnownBlock			  = KnownBlocks[KnownBlockIndex];
 						for (size_t BlockChunkIndex = 0; BlockChunkIndex < KnownBlock.ChunkRawHashes.size(); BlockChunkIndex++)
 						{
 							const IoHash&  BlockChunkHash = KnownBlock.ChunkRawHashes[BlockChunkIndex];
@@ -2459,6 +3407,7 @@ namespace {
 								{
 									FoundChunkIndexes.push_back(ChunkIndex);
 									AdjustedReuseSize += KnownBlock.ChunkCompressedLengths[BlockChunkIndex];
+									AdjustedRawReuseSize += KnownBlock.ChunkRawLengths[BlockChunkIndex];
 								}
 							}
 						}
@@ -2479,12 +3428,13 @@ namespace {
 							}
 							FindBlocksStats.AcceptedChunkCount += FoundChunkIndexes.size();
 							FindBlocksStats.AcceptedByteCount += AdjustedReuseSize;
+							FindBlocksStats.AcceptedRawByteCount += AdjustedRawReuseSize;
 							FindBlocksStats.AcceptedReduntantChunkCount += KnownBlock.ChunkRawHashes.size() - FoundChunkIndexes.size();
 							FindBlocksStats.AcceptedReduntantByteCount += BlockSize - AdjustedReuseSize;
 						}
 						else
 						{
-							ZEN_CONSOLE_VERBOSE("Skipping block {}. filtered usage level: {}%", KnownBlock.BlockHash, ReusePercent);
+							// ZEN_CONSOLE_VERBOSE("Skipping block {}. filtered usage level: {}%", KnownBlock.BlockHash, ReusePercent);
 							FindBlocksStats.RejectedBlockCount++;
 							FindBlocksStats.RejectedChunkCount += FoundChunkIndexes.size();
 							FindBlocksStats.RejectedByteCount += AdjustedReuseSize;
@@ -2504,12 +3454,14 @@ namespace {
 		return FilteredReuseBlockIndexes;
 	};
 
-	void UploadFolder(BuildStorage&				   Storage,
+	void UploadFolder(StorageInstance&			   Storage,
 					  const Oid&				   BuildId,
 					  const Oid&				   BuildPartId,
 					  const std::string_view	   BuildPartName,
 					  const std::filesystem::path& Path,
+					  const std::filesystem::path& TempDir,
 					  const std::filesystem::path& ManifestPath,
+					  const uint64_t			   FindBlockMaxCount,
 					  const uint8_t				   BlockReuseMinPercentLimit,
 					  bool						   AllowMultiparts,
 					  const CbObject&			   MetaData,
@@ -2517,19 +3469,38 @@ namespace {
 					  bool						   IgnoreExistingBlocks,
 					  bool						   PostUploadVerify)
 	{
+		ZEN_TRACE_CPU("UploadFolder");
+
+		ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder");
+
+		enum TaskSteps : uint32_t
+		{
+			PrepareBuild,
+			CalculateDelta,
+			Upload,
+			Validate,
+			Cleanup,
+			StepCount
+		};
+
+		auto EndProgress =
+			MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
+
 		Stopwatch ProcessTimer;
 
-		const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName;
-		CreateDirectories(ZenTempFolder);
-		CleanDirectory(ZenTempFolder, {});
+		CreateDirectories(TempDir);
+		CleanDirectory(TempDir, {});
 		auto _ = MakeGuard([&]() {
-			if (CleanDirectory(ZenTempFolder, {}))
+			if (CleanDirectory(TempDir, {}))
 			{
-				std::filesystem::remove(ZenTempFolder);
+				std::error_code DummyEc;
+				RemoveDir(TempDir, DummyEc);
 			}
 		});
-		CreateDirectories(Path / ZenTempBlockFolderName);
-		CreateDirectories(Path / ZenTempChunkFolderName);
+
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::PrepareBuild, TaskSteps::StepCount);
+
+		std::uint64_t TotalRawSize = 0;
 
 		CbObject ChunkerParameters;
 
@@ -2545,54 +3516,56 @@ namespace {
 
 		FindBlocksStatistics FindBlocksStats;
 
-		std::future<PrepareBuildResult> PrepBuildResultFuture =
-			GetSmallWorkerPool(EWorkloadType::Burst)
-				.EnqueueTask(std::packaged_task<PrepareBuildResult()>{
-					[&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] {
-						ZEN_TRACE_CPU("PrepareBuild");
+		std::future<PrepareBuildResult> PrepBuildResultFuture = GetNetworkPool().EnqueueTask(std::packaged_task<PrepareBuildResult()>{
+			[&Storage, BuildId, FindBlockMaxCount, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] {
+				ZEN_TRACE_CPU("PrepareBuild");
 
-						PrepareBuildResult Result;
-						Stopwatch		   Timer;
-						if (CreateBuild)
-						{
-							ZEN_TRACE_CPU("CreateBuild");
+				PrepareBuildResult Result;
+				Stopwatch		   Timer;
+				if (CreateBuild)
+				{
+					ZEN_TRACE_CPU("CreateBuild");
 
-							Stopwatch PutBuildTimer;
-							CbObject  PutBuildResult		   = Storage.PutBuild(BuildId, MetaData);
-							Result.PrepareBuildTimeMs		   = PutBuildTimer.GetElapsedTimeMs();
-							Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize);
-							Result.PayloadSize				   = MetaData.GetSize();
-						}
-						else
-						{
-							ZEN_TRACE_CPU("PutBuild");
-							Stopwatch GetBuildTimer;
-							CbObject  Build			  = Storage.GetBuild(BuildId);
-							Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs();
-							Result.PayloadSize		  = Build.GetSize();
-							if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
-							{
-								Result.PreferredMultipartChunkSize = ChunkSize;
-							}
-							else if (AllowMultiparts)
-							{
-								ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
-										 NiceBytes(Result.PreferredMultipartChunkSize));
-							}
-						}
+					Stopwatch PutBuildTimer;
+					CbObject  PutBuildResult		   = Storage.BuildStorage->PutBuild(BuildId, MetaData);
+					Result.PrepareBuildTimeMs		   = PutBuildTimer.GetElapsedTimeMs();
+					Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize);
+					Result.PayloadSize				   = MetaData.GetSize();
+				}
+				else
+				{
+					ZEN_TRACE_CPU("PutBuild");
+					Stopwatch GetBuildTimer;
+					CbObject  Build			  = Storage.BuildStorage->GetBuild(BuildId);
+					Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs();
+					Result.PayloadSize		  = Build.GetSize();
+					if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
+					{
+						Result.PreferredMultipartChunkSize = ChunkSize;
+					}
+					else if (AllowMultiparts)
+					{
+						ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
+								 NiceBytes(Result.PreferredMultipartChunkSize));
+					}
+				}
 
-						if (!IgnoreExistingBlocks)
-						{
-							ZEN_TRACE_CPU("FindBlocks");
-							Stopwatch KnownBlocksTimer;
-							Result.KnownBlocks				= Storage.FindBlocks(BuildId);
-							FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs();
-							FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size();
-							Result.FindBlocksTimeMs			= KnownBlocksTimer.GetElapsedTimeMs();
-						}
-						Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
-						return Result;
-					}});
+				if (!IgnoreExistingBlocks)
+				{
+					ZEN_TRACE_CPU("FindBlocks");
+					Stopwatch KnownBlocksTimer;
+					CbObject  BlockDescriptionList = Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount);
+					if (BlockDescriptionList)
+					{
+						Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList);
+					}
+					FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs();
+					FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size();
+					Result.FindBlocksTimeMs			= KnownBlocksTimer.GetElapsedTimeMs();
+				}
+				Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
+				return Result;
+			}});
 
 		ChunkedFolderContent LocalContent;
 
@@ -2632,8 +3605,9 @@ namespace {
 			auto ParseManifest = [](const std::filesystem::path& Path,
 									const std::filesystem::path& ManifestPath) -> std::vector<std::filesystem::path> {
 				std::vector<std::filesystem::path> AssetPaths;
-				std::filesystem::path			   AbsoluteManifestPath = ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath;
-				IoBuffer						   ManifestContent		= ReadFile(AbsoluteManifestPath).Flatten();
+				std::filesystem::path			   AbsoluteManifestPath =
+					MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath);
+				IoBuffer					ManifestContent = ReadFile(AbsoluteManifestPath).Flatten();
 				std::string_view			ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize());
 				std::string_view::size_type Offset = 0;
 				while (Offset < ManifestContent.GetSize())
@@ -2671,7 +3645,7 @@ namespace {
 			{
 				std::filesystem::path		ExcludeManifestPath = Path / ZenExcludeManifestName;
 				tsl::robin_set<std::string> ExcludeAssetPaths;
-				if (std::filesystem::is_regular_file(ExcludeManifestPath))
+				if (IsFile(ExcludeManifestPath))
 				{
 					std::vector<std::filesystem::path> AssetPaths = ParseManifest(Path, ExcludeManifestPath);
 					ExcludeAssetPaths.reserve(AssetPaths.size());
@@ -2700,10 +3674,10 @@ namespace {
 						}
 						return true;
 					},
-					GetMediumWorkerPool(EWorkloadType::Burst),
-					UsePlainProgress ? 5000 : 200,
+					GetIOWorkerPool(),
+					GetUpdateDelayMS(ProgressMode),
 					[&](bool, std::ptrdiff_t) {
-						ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path);
+						ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path);
 					},
 					AbortFlag);
 			}
@@ -2714,12 +3688,13 @@ namespace {
 				for (const std::filesystem::path& AssetPath : AssetPaths)
 				{
 					Content.Paths.push_back(AssetPath);
-					Content.RawSizes.push_back(std::filesystem::file_size(Path / AssetPath));
+					const std::filesystem::path AssetFilePath = (Path / AssetPath).make_preferred();
+					Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath));
 #if ZEN_PLATFORM_WINDOWS
-					Content.Attributes.push_back(GetFileAttributes(Path / AssetPath));
+					Content.Attributes.push_back(GetFileAttributes(AssetFilePath));
 #endif	// ZEN_PLATFORM_WINDOWS
 #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
-					Content.Attributes.push_back(GetFileMode(Path / AssetPath));
+					Content.Attributes.push_back(GetFileMode(AssetFilePath));
 #endif	// ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
 					LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back();
 					LocalFolderScanStats.AcceptedFileCount++;
@@ -2727,12 +3702,13 @@ namespace {
 				if (ManifestPath.is_relative())
 				{
 					Content.Paths.push_back(ManifestPath);
-					Content.RawSizes.push_back(std::filesystem::file_size(ManifestPath));
+					const std::filesystem::path ManifestFilePath = (Path / ManifestPath).make_preferred();
+					Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath));
 #if ZEN_PLATFORM_WINDOWS
-					Content.Attributes.push_back(GetFileAttributes(ManifestPath));
+					Content.Attributes.push_back(GetFileAttributes(ManifestFilePath));
 #endif	// ZEN_PLATFORM_WINDOWS
 #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
-					Content.Attributes.push_back(GetFileMode(ManifestPath));
+					Content.Attributes.push_back(GetFileMode(ManifestFilePath));
 #endif	// ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
 
 					LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back();
@@ -2743,7 +3719,8 @@ namespace {
 				LocalFolderScanStats.ElapsedWallTimeUS = ManifestParseTimer.GetElapsedTimeUs();
 			}
 
-			std::unique_ptr<ChunkingController> ChunkController = CreateBasicChunkingController();
+			std::unique_ptr<ChunkingController> ChunkController =
+				CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
 			{
 				CbObjectWriter ChunkParametersWriter;
 				ChunkParametersWriter.AddString("name"sv, ChunkController->GetName());
@@ -2751,20 +3728,20 @@ namespace {
 				ChunkerParameters = ChunkParametersWriter.Save();
 			}
 
-			std::uint64_t TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0));
+			TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0));
 
 			{
-				ProgressBar	 ProgressBar(UsePlainProgress);
+				ProgressBar	 ProgressBar(ProgressMode, "Scan Files");
 				FilteredRate FilteredBytesHashed;
 				FilteredBytesHashed.Start();
 				LocalContent = ChunkFolderContent(
 					ChunkingStats,
-					GetMediumWorkerPool(EWorkloadType::Burst),
+					GetIOWorkerPool(),
 					Path,
 					Content,
 					*ChunkController,
-					UsePlainProgress ? 5000 : 200,
-					[&](bool, std::ptrdiff_t) {
+					GetUpdateDelayMS(ProgressMode),
+					[&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
 						FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
 						std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
 														  ChunkingStats.FilesProcessed.load(),
@@ -2777,16 +3754,18 @@ namespace {
 						ProgressBar.UpdateState({.Task			 = "Scanning files   ",
 												 .Details		 = Details,
 												 .TotalCount	 = TotalRawSize,
-												 .RemainingCount = TotalRawSize - ChunkingStats.BytesHashed.load()},
+												 .RemainingCount = TotalRawSize - ChunkingStats.BytesHashed.load(),
+												 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
 												false);
 					},
-					AbortFlag);
+					AbortFlag,
+					PauseFlag);
+				FilteredBytesHashed.Stop();
+				ProgressBar.Finish();
 				if (AbortFlag)
 				{
 					return;
 				}
-				FilteredBytesHashed.Stop();
-				ProgressBar.Finish();
 			}
 
 			ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
@@ -2819,6 +3798,8 @@ namespace {
 													   PrepBuildResult.KnownBlocks.size(),
 													   NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
 
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount);
+
 		const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
 
 		Stopwatch BlockArrangeTimer;
@@ -2880,16 +3861,17 @@ namespace {
 		}
 		FindBlocksStats.NewBlocksChunkCount = NewBlockChunkIndexes.size();
 
-		const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0
-													? (100.0 * FindBlocksStats.AcceptedByteCount / FindBlocksStats.PotentialChunkByteCount)
-													: 0.0;
+		const double AcceptedByteCountPercent =
+			FindBlocksStats.PotentialChunkByteCount > 0
+				? (100.0 * FindBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount)
+				: 0.0;
 
 		const double AcceptedReduntantByteCountPercent =
 			FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) /
 														(FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount)
 												  : 0.0;
 		ZEN_CONSOLE(
-			"Found {} chunks in {} ({}) blocks eligeble for reuse in {}\n"
+			"Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
 			"    Reusing   {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
 			"    Accepting {} ({}) redundant chunks ({:.1f}%)\n"
 			"    Rejected  {} ({}) chunks in {} blocks\n"
@@ -2902,7 +3884,7 @@ namespace {
 			NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
 
 			FindBlocksStats.AcceptedChunkCount,
-			NiceBytes(FindBlocksStats.AcceptedByteCount),
+			NiceBytes(FindBlocksStats.AcceptedRawByteCount),
 			FindBlocksStats.AcceptedBlockCount,
 			AcceptedByteCountPercent,
 
@@ -2927,6 +3909,8 @@ namespace {
 		UploadStatistics UploadStats;
 		GeneratedBlocks	 NewBlocks;
 
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Upload, TaskSteps::StepCount);
+
 		if (!NewBlockChunks.empty())
 		{
 			Stopwatch GenerateBuildBlocksTimer;
@@ -3108,40 +4092,46 @@ namespace {
 		}
 
 		Stopwatch							   PutBuildPartResultTimer;
-		std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest);
+		std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
+			Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest);
 		ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.",
 					NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
 					NiceBytes(PartManifest.GetSize()),
 					PutBuildPartResult.second.size());
 		IoHash PartHash = PutBuildPartResult.first;
 
-		auto UploadAttachments = [&](std::span<IoHash> RawHashes) {
+		auto UploadAttachments = [&Storage,
+								  &BuildId,
+								  &Path,
+								  &TempDir,
+								  &LocalContent,
+								  &LocalLookup,
+								  &NewBlockChunks,
+								  &NewBlocks,
+								  &LooseChunkIndexes,
+								  &LargeAttachmentSize,
+								  &DiskStats,
+								  &UploadStats,
+								  &LooseChunksStats](std::span<IoHash> RawHashes) {
 			if (!AbortFlag)
 			{
-				ZEN_CONSOLE_VERBOSE("Uploading attachments: {}", FormatArray<IoHash>(RawHashes, "\n    "sv));
-
-				UploadStatistics		 TempUploadStats;
-				GenerateBlocksStatistics TempGenerateBlocksStats;
-				LooseChunksStatistics	 TempLooseChunksStats;
+				UploadStatistics	  TempUploadStats;
+				LooseChunksStatistics TempLooseChunksStats;
 
 				Stopwatch TempUploadTimer;
 				auto	  __ = MakeGuard([&]() {
 					 uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
 					 ZEN_CONSOLE(
-						 "Generated {} ({} {}B/s) and uploaded {} ({}) blocks. "
+						 "Uploaded {} ({}) blocks. "
 							 "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
 							 "Transferred {} ({}bits/s) in {}",
-						 TempGenerateBlocksStats.GeneratedBlockCount.load(),
-						 NiceBytes(TempGenerateBlocksStats.GeneratedBlockByteCount.load()),
-						 NiceNum(GetBytesPerSecond(TempGenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
-												   TempGenerateBlocksStats.GeneratedBlockByteCount)),
 						 TempUploadStats.BlockCount.load(),
 						 NiceBytes(TempUploadStats.BlocksBytes),
 
 						 TempLooseChunksStats.CompressedChunkCount.load(),
 						 NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
-						 NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
-												   TempLooseChunksStats.CompressedChunkBytes)),
+						 NiceNum(
+							 GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)),
 						 TempUploadStats.ChunkCount.load(),
 						 NiceBytes(TempUploadStats.ChunksBytes),
 
@@ -3152,6 +4142,7 @@ namespace {
 				UploadPartBlobs(Storage,
 								BuildId,
 								Path,
+								TempDir,
 								LocalContent,
 								LocalLookup,
 								RawHashes,
@@ -3161,11 +4152,9 @@ namespace {
 								LargeAttachmentSize,
 								DiskStats,
 								TempUploadStats,
-								TempGenerateBlocksStats,
 								TempLooseChunksStats);
 				UploadStats += TempUploadStats;
 				LooseChunksStats += TempLooseChunksStats;
-				GenerateBlocksStats += TempGenerateBlocksStats;
 			}
 		};
 		if (IgnoreExistingBlocks)
@@ -3197,10 +4186,11 @@ namespace {
 			UploadAttachments(PutBuildPartResult.second);
 		}
 
-		while (!AbortFlag)
+		uint32_t FinalizeBuildPartRetryCount = 5;
+		while (!AbortFlag && (FinalizeBuildPartRetryCount--) > 0)
 		{
 			Stopwatch			FinalizeBuildPartTimer;
-			std::vector<IoHash> Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash);
+			std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash);
 			ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.",
 						NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
 						Needs.size());
@@ -3215,29 +4205,50 @@ namespace {
 		if (CreateBuild && !AbortFlag)
 		{
 			Stopwatch FinalizeBuildTimer;
-			Storage.FinalizeBuild(BuildId);
+			Storage.BuildStorage->FinalizeBuild(BuildId);
 			ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
 		}
 
 		if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag)
 		{
-			uint64_t			UploadBlockMetadataCount = 0;
-			std::vector<IoHash> BlockHashes;
-			BlockHashes.reserve(NewBlocks.BlockDescriptions.size());
+			uint64_t  UploadBlockMetadataCount = 0;
 			Stopwatch UploadBlockMetadataTimer;
-			for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
+
+			uint32_t FailedMetadataUploadCount = 1;
+			int32_t	 MetadataUploadRetryCount  = 3;
+			while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0))
 			{
-				const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
-				if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
+				FailedMetadataUploadCount = 0;
+				for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
 				{
-					const CbObject BlockMetaData =
-						BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
-					Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
-					UploadStats.BlocksBytes += BlockMetaData.GetSize();
-					NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
-					UploadBlockMetadataCount++;
+					if (AbortFlag)
+					{
+						break;
+					}
+					const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
+					if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
+					{
+						const CbObject BlockMetaData =
+							BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
+						if (Storage.BuildCacheStorage)
+						{
+							Storage.BuildCacheStorage->PutBlobMetadatas(BuildId,
+																		std::vector<IoHash>({BlockHash}),
+																		std::vector<CbObject>({BlockMetaData}));
+						}
+						bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
+						if (MetadataSucceeded)
+						{
+							UploadStats.BlocksBytes += BlockMetaData.GetSize();
+							NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+							UploadBlockMetadataCount++;
+						}
+						else
+						{
+							FailedMetadataUploadCount++;
+						}
+					}
 				}
-				BlockHashes.push_back(BlockHash);
 			}
 			if (UploadBlockMetadataCount > 0)
 			{
@@ -3247,19 +4258,14 @@ namespace {
 			}
 		}
 
+		ValidateStatistics ValidateStats;
+		DownloadStatistics ValidateDownloadStats;
 		if (PostUploadVerify && !AbortFlag)
 		{
-			ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName);
+			ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Validate, TaskSteps::StepCount);
+			ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats);
 		}
 
-		const double DeltaByteCountPercent =
-			ChunkingStats.BytesHashed > 0
-				? (100.0 * (FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes)) / (ChunkingStats.BytesHashed)
-				: 0.0;
-
-		const std::string LargeAttachmentStats =
-			(LargeAttachmentSize != (uint64_t)-1) ? fmt::format(" ({} as multipart)", UploadStats.MultipartAttachmentCount.load()) : "";
-
 		ZEN_CONSOLE_VERBOSE(
 			"Folder scanning stats:"
 			"\n    FoundFileCount:        {}"
@@ -3301,6 +4307,7 @@ namespace {
 			"\n    AcceptedBlockCount:          {}"
 			"\n    AcceptedChunkCount:          {}"
 			"\n    AcceptedByteCount:           {}"
+			"\n    AcceptedRawByteCount:        {}"
 			"\n    RejectedBlockCount:          {}"
 			"\n    RejectedChunkCount:          {}"
 			"\n    RejectedByteCount:           {}"
@@ -3318,6 +4325,7 @@ namespace {
 			FindBlocksStats.AcceptedBlockCount,
 			FindBlocksStats.AcceptedChunkCount,
 			NiceBytes(FindBlocksStats.AcceptedByteCount),
+			NiceBytes(FindBlocksStats.AcceptedRawByteCount),
 			FindBlocksStats.RejectedBlockCount,
 			FindBlocksStats.RejectedChunkCount,
 			NiceBytes(FindBlocksStats.RejectedByteCount),
@@ -3382,46 +4390,148 @@ namespace {
 			UploadStats.MultipartAttachmentCount.load(),
 			NiceLatencyNs(UploadStats.ElapsedWallTimeUS * 1000));
 
+		if (PostUploadVerify)
+		{
+			ZEN_CONSOLE_VERBOSE(
+				"Validate stats:"
+				"\n    BuildBlobSize:             {}"
+				"\n    BuildPartSize:             {}"
+				"\n    ChunkAttachmentCount:      {}"
+				"\n    BlockAttachmentCount:      {}"
+				"\n    VerifiedAttachmentCount:   {}"
+				"\n    VerifiedByteCount:         {}"
+				"\n    ElapsedWallTimeUS:         {}",
+				NiceBytes(ValidateStats.BuildBlobSize),
+				NiceBytes(ValidateStats.BuildPartSize),
+				ValidateStats.ChunkAttachmentCount,
+				ValidateStats.BlockAttachmentCount,
+				ValidateStats.VerifiedAttachmentCount.load(),
+				NiceBytes(ValidateStats.VerifiedByteCount.load()),
+				NiceLatencyNs(ValidateStats.ElapsedWallTimeUS * 1000));
+
+			ZEN_CONSOLE_VERBOSE(
+				"Validate download stats:"
+				"\n    RequestsCompleteCount:           {}"
+				"\n    DownloadedChunkCount:            {}"
+				"\n    DownloadedChunkByteCount:        {}"
+				"\n    MultipartAttachmentCount:        {}"
+				"\n    DownloadedBlockCount:            {}"
+				"\n    DownloadedBlockByteCount:        {}"
+				"\n    DownloadedPartialBlockCount:     {}"
+				"\n    DownloadedPartialBlockByteCount: {}",
+				ValidateDownloadStats.RequestsCompleteCount.load(),
+				ValidateDownloadStats.DownloadedChunkCount.load(),
+				NiceBytes(ValidateDownloadStats.DownloadedChunkByteCount.load()),
+				ValidateDownloadStats.MultipartAttachmentCount.load(),
+				ValidateDownloadStats.DownloadedBlockCount.load(),
+				NiceBytes(ValidateDownloadStats.DownloadedBlockByteCount.load()),
+				ValidateDownloadStats.DownloadedPartialBlockCount.load(),
+				NiceBytes(ValidateDownloadStats.DownloadedPartialBlockByteCount.load()));
+		}
+
+		const double DeltaByteCountPercent =
+			ChunkingStats.BytesHashed > 0
+				? (100.0 * (FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes)) / (ChunkingStats.BytesHashed)
+				: 0.0;
+
+		const std::string MultipartAttachmentStats =
+			(LargeAttachmentSize != (uint64_t)-1) ? fmt::format(" ({} as multipart)", UploadStats.MultipartAttachmentCount.load()) : "";
+
+		std::string ValidateInfo;
+		if (PostUploadVerify)
+		{
+			const uint64_t DownloadedCount = ValidateDownloadStats.DownloadedChunkCount + ValidateDownloadStats.DownloadedBlockCount;
+			const uint64_t DownloadedByteCount =
+				ValidateDownloadStats.DownloadedChunkByteCount + ValidateDownloadStats.DownloadedBlockByteCount;
+			ValidateInfo = fmt::format("\n    Verified:      {:>8} ({}), {}B/sec, {}",
+									   DownloadedCount,
+									   NiceBytes(DownloadedByteCount),
+									   NiceNum(GetBytesPerSecond(ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)),
+									   NiceTimeSpanMs(ValidateStats.ElapsedWallTimeUS / 1000));
+		}
+
 		ZEN_CONSOLE(
-			"Uploaded {}\n"
-			"    Delta:  {}/{} ({:.1f}%)\n"
-			"    Blocks: {} ({})\n"
-			"    Chunks: {} ({}){}\n"
-			"    Rate:   {}bits/sec",
-			NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
+			"Uploaded part {} ('{}') to build {}, {}\n"
+			"    Scanned files: {:>8} ({}), {}B/sec, {}\n"
+			"    New data:      {:>8} ({}) {:.1f}%\n"
+			"    New blocks:    {:>8} ({} -> {}), {}B/sec, {}\n"
+			"    New chunks:    {:>8} ({} -> {}), {}B/sec, {}\n"
+			"    Uploaded:      {:>8} ({}), {}bits/sec, {}\n"
+			"        Blocks:    {:>8} ({})\n"
+			"        Chunks:    {:>8} ({}){}"
+			"{}",
+			BuildPartId,
+			BuildPartName,
+			BuildId,
+			NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()),
 
+			LocalFolderScanStats.FoundFileCount.load(),
+			NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()),
+			NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)),
+			NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000),
+
+			FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount,
 			NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
-			NiceBytes(ChunkingStats.BytesHashed),
 			DeltaByteCountPercent,
 
+			GenerateBlocksStats.GeneratedBlockCount.load(),
+			NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+			NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
+			NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)),
+			NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
+
+			LooseChunksStats.CompressedChunkCount.load(),
+			NiceBytes(LooseChunksStats.ChunkByteCount),
+			NiceBytes(LooseChunksStats.CompressedChunkBytes.load()),
+			NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)),
+			NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
+
+			UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(),
+			NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
+			NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
+			NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000),
+
 			UploadStats.BlockCount.load(),
-			NiceBytes(UploadStats.BlocksBytes),
+			NiceBytes(UploadStats.BlocksBytes.load()),
+
 			UploadStats.ChunkCount.load(),
-			NiceBytes(UploadStats.ChunksBytes),
-			LargeAttachmentStats,
+			NiceBytes(UploadStats.ChunksBytes.load()),
+			MultipartAttachmentStats,
 
-			NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes * 8))));
+			ValidateInfo);
 
-		ZEN_CONSOLE("Uploaded ({}) build {} part {} ({}) in {}",
-					NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes),
-					BuildId,
-					BuildPartName,
-					BuildPartId,
-					NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()));
+		Storage.BuildStorage->PutBuildPartStats(
+			BuildId,
+			BuildPartId,
+			{{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())},
+			 {"reusedRatio", AcceptedByteCountPercent / 100.0},
+			 {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)},
+			 {"reusedBlockByteCount", double(FindBlocksStats.AcceptedRawByteCount)},
+			 {"newBlockCount", double(FindBlocksStats.NewBlocksCount)},
+			 {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)},
+			 {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())},
+			 {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())},
+			 {"uploadedBytesPerSec",
+			  double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))},
+			 {"elapsedTimeSec", double(ProcessTimer.GetElapsedTimeMs() / 1000.0)}});
+
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
 	}
 
-	void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path, bool VerifyFileHash)
+	void VerifyFolder(const ChunkedFolderContent&  Content,
+					  const std::filesystem::path& Path,
+					  bool						   VerifyFileHash,
+					  VerifyFolderStatistics&	   VerifyFolderStats)
 	{
 		ZEN_TRACE_CPU("VerifyFolder");
 
-		ProgressBar			  ProgressBar(UsePlainProgress);
-		std::atomic<uint64_t> FilesVerified(0);
-		std::atomic<uint64_t> FilesFailed(0);
-		std::atomic<uint64_t> ReadBytes(0);
+		Stopwatch Timer;
 
-		WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst);  //  GetSyncWorkerPool();  //
+		ProgressBar ProgressBar(ProgressMode, "Verify Files");
 
-		ParallellWork Work(AbortFlag);
+		WorkerThreadPool& VerifyPool = GetIOWorkerPool();
+
+		ParallelWork Work(AbortFlag, PauseFlag);
 
 		const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size());
 
@@ -3457,7 +4567,8 @@ namespace {
 
 			Work.ScheduleWork(
 				VerifyPool,
-				[&, PathIndex](std::atomic<bool>&) {
+				[&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex](
+					std::atomic<bool>&) {
 					if (!AbortFlag)
 					{
 						ZEN_TRACE_CPU("VerifyFile_work");
@@ -3468,24 +4579,24 @@ namespace {
 						if (IsAcceptedFolder(TargetPath.parent_path().generic_string()))
 						{
 							const uint64_t ExpectedSize = Content.RawSizes[PathIndex];
-							if (!std::filesystem::exists(TargetPath))
+							if (!IsFile(TargetPath))
 							{
 								ErrorLock.WithExclusiveLock([&]() {
 									Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize));
 								});
-								FilesFailed++;
+								VerifyFolderStats.FilesFailed++;
 							}
 							else
 							{
 								std::error_code Ec;
-								uint64_t		SizeOnDisk = gsl::narrow<uint64_t>(std::filesystem::file_size(TargetPath, Ec));
+								uint64_t		SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec));
 								if (Ec)
 								{
 									ErrorLock.WithExclusiveLock([&]() {
 										Errors.push_back(
 											fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value()));
 									});
-									FilesFailed++;
+									VerifyFolderStats.FilesFailed++;
 								}
 								else if (SizeOnDisk < ExpectedSize)
 								{
@@ -3495,7 +4606,7 @@ namespace {
 																	 ExpectedSize,
 																	 SizeOnDisk));
 									});
-									FilesFailed++;
+									VerifyFolderStats.FilesFailed++;
 								}
 								else if (SizeOnDisk > ExpectedSize)
 								{
@@ -3505,7 +4616,7 @@ namespace {
 																	 ExpectedSize,
 																	 SizeOnDisk));
 									});
-									FilesFailed++;
+									VerifyFolderStats.FilesFailed++;
 								}
 								else if (SizeOnDisk > 0 && VerifyFileHash)
 								{
@@ -3540,39 +4651,56 @@ namespace {
 											}
 											FileOffset += ChunkSize;
 										}
-										FilesFailed++;
+										VerifyFolderStats.FilesFailed++;
 									}
-									ReadBytes += SizeOnDisk;
+									VerifyFolderStats.ReadBytes += SizeOnDisk;
 								}
 							}
 						}
-						FilesVerified++;
+						VerifyFolderStats.FilesVerified++;
 					}
 				},
-				[&, PathIndex](const std::exception& Ex, std::atomic<bool>&) {
+				[&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) {
+					std::string Description;
+					try
+					{
+						std::rethrow_exception(Ex);
+					}
+					catch (const std::exception& Ex)
+					{
+						Description = Ex.what();
+					}
 					ErrorLock.WithExclusiveLock([&]() {
 						Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}",
 													 (Path / Content.Paths[PathIndex]).make_preferred(),
-													 Ex.what()));
+													 Description));
 					});
-					FilesFailed++;
+					VerifyFolderStats.FilesFailed++;
 				});
 		}
 
-		Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-			ZEN_UNUSED(IsAborted, PendingWork);
+		Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+			ZEN_UNUSED(PendingWork);
 			std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}",
-											  FilesVerified.load(),
+											  VerifyFolderStats.FilesVerified.load(),
 											  PathCount,
-											  NiceBytes(ReadBytes.load()),
-											  FilesFailed.load());
+											  NiceBytes(VerifyFolderStats.ReadBytes.load()),
+											  VerifyFolderStats.FilesFailed.load());
 			ProgressBar.UpdateState({.Task			 = "Verifying files  ",
 									 .Details		 = Details,
 									 .TotalCount	 = gsl::narrow<uint64_t>(PathCount),
-									 .RemainingCount = gsl::narrow<uint64_t>(PathCount - FilesVerified.load())},
+									 .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()),
+									 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
 									false);
 		});
+		VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+
 		ProgressBar.Finish();
+		if (AbortFlag)
+		{
+			return;
+		}
+
 		for (const std::string& Error : Errors)
 		{
 			ZEN_CONSOLE("{}", Error);
@@ -3583,99 +4711,228 @@ namespace {
 		}
 	}
 
-	class WriteFileCache
+	std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> GetRemainingChunkTargets(
+		std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+		const ChunkedContentLookup&			   Lookup,
+		uint32_t							   ChunkIndex)
 	{
-	public:
-		WriteFileCache() {}
-		~WriteFileCache() { Flush(); }
+		std::span<const ChunkedContentLookup::ChunkSequenceLocation>	ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex);
+		std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs;
+		if (!ChunkSources.empty())
+		{
+			ChunkTargetPtrs.reserve(ChunkSources.size());
+			for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources)
+			{
+				if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0)
+				{
+					ChunkTargetPtrs.push_back(&Source);
+				}
+			}
+		}
+		return ChunkTargetPtrs;
+	};
 
-		template<typename TBufferType>
-		void WriteToFile(uint32_t													  TargetIndex,
-						 std::function<std::filesystem::path(uint32_t TargetIndex)>&& GetTargetPath,
-						 const TBufferType&											  Buffer,
-						 uint64_t													  FileOffset,
-						 uint64_t													  TargetFinalSize)
+	uint64_t GetChunkWriteCount(std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+								const ChunkedContentLookup&			   Lookup,
+								uint32_t							   ChunkIndex)
+	{
+		uint64_t													 WriteCount	  = 0;
+		std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex);
+		for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources)
 		{
-			ZEN_TRACE_CPU("WriteFileCache_WriteToFile");
-			if (!SeenTargetIndexes.empty() && SeenTargetIndexes.back() == TargetIndex)
+			if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0)
 			{
-				ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite");
-				ZEN_ASSERT(OpenFileWriter);
-				OpenFileWriter->Write(Buffer, FileOffset);
+				WriteCount++;
 			}
-			else
+		}
+		return WriteCount;
+	};
+
+	void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash)
+	{
+		ZEN_TRACE_CPU("FinalizeChunkSequence");
+		ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)));
+		std::error_code Ec;
+		RenameFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash),
+				   GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash),
+				   Ec);
+		if (Ec)
+		{
+			throw std::system_error(Ec);
+		}
+	}
+
+	void FinalizeChunkSequences(const std::filesystem::path& TargetFolder,
+								const ChunkedFolderContent&	 RemoteContent,
+								std::span<const uint32_t>	 RemoteSequenceIndexes)
+	{
+		ZEN_TRACE_CPU("FinalizeChunkSequences");
+		for (uint32_t SequenceIndex : RemoteSequenceIndexes)
+		{
+			FinalizeChunkSequence(TargetFolder, RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+		}
+	}
+
+	void VerifySequence(const std::filesystem::path& TargetFolder,
+						const ChunkedFolderContent&	 RemoteContent,
+						const ChunkedContentLookup&	 Lookup,
+						uint32_t					 RemoteSequenceIndex)
+	{
+		ZEN_TRACE_CPU("VerifySequence");
+		const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+		{
+			ZEN_TRACE_CPU("HashSequence");
+			const std::uint32_t RemotePathIndex = Lookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
+			const uint64_t		ExpectedSize	= RemoteContent.RawSizes[RemotePathIndex];
+			IoBuffer			VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash));
+			const uint64_t		VerifySize	 = VerifyBuffer.GetSize();
+			if (VerifySize != ExpectedSize)
 			{
-				std::unique_ptr<BasicFile> NewOutputFile;
-				{
-					ZEN_TRACE_CPU("WriteFileCache_WriteToFile_Open");
-					Flush();
-					const std::filesystem::path& TargetPath = GetTargetPath(TargetIndex);
-					CreateDirectories(TargetPath.parent_path());
-					uint32_t Tries = 5;
-					NewOutputFile =
-						std::make_unique<BasicFile>(TargetPath, BasicFile::Mode::kWrite, [&Tries, TargetPath](std::error_code& Ec) {
-							if (Tries < 3)
-							{
-								ZEN_CONSOLE("Failed opening file '{}': {}{}", TargetPath, Ec.message(), Tries > 1 ? " Retrying"sv : ""sv);
-							}
-							if (Tries > 1)
-							{
-								Sleep(100);
-							}
-							return --Tries > 0;
-						});
-				}
+				throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}",
+													 SequenceRawHash,
+													 VerifySize,
+													 ExpectedSize));
+			}
 
-				const bool CacheWriter = TargetFinalSize > Buffer.GetSize();
-				if (CacheWriter)
-				{
-					ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite");
-					ZEN_ASSERT_SLOW(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end());
+			const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer));
+			if (VerifyChunkHash != SequenceRawHash)
+			{
+				throw std::runtime_error(
+					fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash));
+			}
+		}
+	}
 
-					OutputFile	   = std::move(NewOutputFile);
-					OpenFileWriter = std::make_unique<BasicFileWriter>(*OutputFile, Min(TargetFinalSize, 256u * 1024u));
-					OpenFileWriter->Write(Buffer, FileOffset);
-					SeenTargetIndexes.push_back(TargetIndex);
-				}
-				else
+	void VerifyAndCompleteChunkSequencesAsync(const std::filesystem::path& TargetFolder,
+											  const ChunkedFolderContent&  RemoteContent,
+											  const ChunkedContentLookup&  Lookup,
+											  std::span<const uint32_t>	   RemoteSequenceIndexes,
+											  ParallelWork&				   Work,
+											  WorkerThreadPool&			   VerifyPool)
+	{
+		if (RemoteSequenceIndexes.empty())
+		{
+			return;
+		}
+		ZEN_TRACE_CPU("VerifyAndCompleteChunkSequence");
+		for (uint32_t RemoteSequenceIndexOffset = 1; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++)
+		{
+			const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset];
+			Work.ScheduleWork(VerifyPool, [&RemoteContent, &Lookup, TargetFolder, RemoteSequenceIndex](std::atomic<bool>&) {
+				if (!AbortFlag)
 				{
-					ZEN_TRACE_CPU("WriteFileCache_WriteToFile_Write");
-					NewOutputFile->Write(Buffer, FileOffset);
+					ZEN_TRACE_CPU("VerifyAndCompleteChunkSequenceAsync");
+					VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex);
+					if (!AbortFlag)
+					{
+						const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+						FinalizeChunkSequence(TargetFolder, SequenceRawHash);
+					}
 				}
-			}
+			});
 		}
+		const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0];
+
+		VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex);
+		const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+		FinalizeChunkSequence(TargetFolder, SequenceRawHash);
+	}
+
+	bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters)
+	{
+		uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1);
+		ZEN_ASSERT(PreviousValue >= 1);
+		ZEN_ASSERT(PreviousValue != (uint32_t)-1);
+		return PreviousValue == 1;
+	}
 
-		void Flush()
+	std::vector<uint32_t> CompleteChunkTargets(const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
+											   std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters)
+	{
+		ZEN_TRACE_CPU("CompleteChunkTargets");
+
+		std::vector<uint32_t> CompletedSequenceIndexes;
+		for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs)
 		{
-			ZEN_TRACE_CPU("WriteFileCache_Flush");
-			OpenFileWriter = {};
-			OutputFile	   = {};
+			const uint32_t RemoteSequenceIndex = Location->SequenceIndex;
+			if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
+			{
+				CompletedSequenceIndexes.push_back(RemoteSequenceIndex);
+			}
 		}
-		std::vector<uint32_t>			 SeenTargetIndexes;
-		std::unique_ptr<BasicFile>		 OutputFile;
-		std::unique_ptr<BasicFileWriter> OpenFileWriter;
-	};
+		return CompletedSequenceIndexes;
+	}
 
-	std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> GetRemainingChunkTargets(
-		std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
-		const ChunkedContentLookup&			   Lookup,
-		uint32_t							   ChunkIndex)
+	void WriteSequenceChunk(const std::filesystem::path&   TargetFolderPath,
+							const ChunkedFolderContent&	   RemoteContent,
+							BufferedWriteFileCache::Local& LocalWriter,
+							const CompositeBuffer&		   Chunk,
+							const uint32_t				   SequenceIndex,
+							const uint64_t				   FileOffset,
+							const uint32_t				   PathIndex,
+							DiskStatistics&				   DiskStats)
 	{
-		std::span<const ChunkedContentLookup::ChunkSequenceLocation>	ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex);
-		std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs;
-		if (!ChunkSources.empty())
+		ZEN_TRACE_CPU("WriteSequenceChunk");
+
+		const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex];
+
+		auto OpenFile = [&](BasicFile& File) {
+			const std::filesystem::path FileName =
+				GetTempChunkedSequenceFileName(TargetFolderPath, RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+			File.Open(FileName, BasicFile::Mode::kWrite);
+			if (UseSparseFiles)
+			{
+				PrepareFileForScatteredWrite(File.Handle(), SequenceSize);
+			}
+		};
+
+		const uint64_t ChunkSize = Chunk.GetSize();
+		ZEN_ASSERT(FileOffset + ChunkSize <= SequenceSize);
+		if (ChunkSize == SequenceSize)
 		{
-			ChunkTargetPtrs.reserve(ChunkSources.size());
-			for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources)
+			BasicFile SingleChunkFile;
+			OpenFile(SingleChunkFile);
+
+			DiskStats.CurrentOpenFileCount++;
+			auto _ = MakeGuard([&DiskStats]() { DiskStats.CurrentOpenFileCount--; });
+			SingleChunkFile.Write(Chunk, FileOffset);
+
+			DiskStats.WriteCount++;
+			DiskStats.WriteByteCount += ChunkSize;
+		}
+		else
+		{
+			const uint64_t MaxWriterBufferSize = 256u * 1025u;
+
+			BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(SequenceIndex);
+			if (Writer)
 			{
-				if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0)
+				if ((!Writer->Writer) && (ChunkSize < MaxWriterBufferSize))
 				{
-					ChunkTargetPtrs.push_back(&Source);
+					Writer->Writer = std::make_unique<BasicFileWriter>(*Writer->File, Min(SequenceSize, MaxWriterBufferSize));
+				}
+				Writer->Write(Chunk, FileOffset);
+
+				DiskStats.WriteCount++;
+				DiskStats.WriteByteCount += ChunkSize;
+			}
+			else
+			{
+				Writer = LocalWriter.PutWriter(SequenceIndex, std::make_unique<BufferedWriteFileCache::Local::Writer>());
+
+				Writer->File = std::make_unique<BasicFile>();
+				OpenFile(*Writer->File);
+				if (ChunkSize < MaxWriterBufferSize)
+				{
+					Writer->Writer = std::make_unique<BasicFileWriter>(*Writer->File, Min(SequenceSize, MaxWriterBufferSize));
 				}
+				Writer->Write(Chunk, FileOffset);
+
+				DiskStats.WriteCount++;
+				DiskStats.WriteByteCount += ChunkSize;
 			}
 		}
-		return ChunkTargetPtrs;
-	};
+	}
 
 	struct BlockWriteOps
 	{
@@ -3693,12 +4950,15 @@ namespace {
 							const ChunkedContentLookup&		 Lookup,
 							std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
 							const BlockWriteOps&			 Ops,
-							std::atomic<uint32_t>&			 OutChunksComplete,
-							std::atomic<uint64_t>&			 OutBytesWritten)
+							BufferedWriteFileCache&			 WriteCache,
+							ParallelWork&					 Work,
+							WorkerThreadPool&				 VerifyPool,
+							DiskStatistics&					 DiskStats)
 	{
 		ZEN_TRACE_CPU("WriteBlockChunkOps");
+
 		{
-			WriteFileCache OpenFileCache;
+			BufferedWriteFileCache::Local LocalWriter(WriteCache);
 			for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps)
 			{
 				if (AbortFlag)
@@ -3710,50 +4970,75 @@ namespace {
 				ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <=
 						   RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]);
 				ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0);
-				const uint64_t ChunkSize  = Chunk.GetSize();
 				const uint64_t FileOffset = WriteOp.Target->Offset;
 				const uint32_t PathIndex  = Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
-				ZEN_ASSERT(FileOffset + ChunkSize <= RemoteContent.RawSizes[PathIndex]);
 
-				OpenFileCache.WriteToFile<CompositeBuffer>(
-					SequenceIndex,
-					[&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) {
-						return GetTempChunkedSequenceFileName(CacheFolderPath,
-															  RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
-					},
-					Chunk,
-					FileOffset,
-					RemoteContent.RawSizes[PathIndex]);
-				OutBytesWritten += ChunkSize;
+				WriteSequenceChunk(CacheFolderPath, RemoteContent, LocalWriter, Chunk, SequenceIndex, FileOffset, PathIndex, DiskStats);
 			}
 		}
 		if (!AbortFlag)
 		{
-			// Write tracking, updating this must be done without any files open (WriteFileCache)
+			// Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local)
+			std::vector<uint32_t> CompletedChunkSequences;
 			for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps)
 			{
 				const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex;
-				if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1)
+				if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
 				{
-					const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
-					{
-						ZEN_TRACE_CPU("VerifyChunkHash");
-						const IoHash VerifyChunkHash = IoHash::HashBuffer(
-							IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)));
-						if (VerifyChunkHash != SequenceRawHash)
-						{
-							throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}",
-																 VerifyChunkHash,
-																 SequenceRawHash));
-						}
-					}
-					ZEN_TRACE_CPU("VerifyChunkHashes_rename");
-					ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)));
-					std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash),
-											GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash));
+					CompletedChunkSequences.push_back(RemoteSequenceIndex);
 				}
 			}
-			OutChunksComplete += gsl::narrow<uint32_t>(Ops.ChunkBuffers.size());
+			WriteCache.Close(CompletedChunkSequences);
+			VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, Lookup, CompletedChunkSequences, Work, VerifyPool);
+		}
+	}
+
+	IoBuffer MakeBufferMemoryBased(const CompositeBuffer& PartialBlockBuffer)
+	{
+		ZEN_TRACE_CPU("MakeBufferMemoryBased");
+		IoBuffer					  BlockMemoryBuffer;
+		std::span<const SharedBuffer> Segments = PartialBlockBuffer.GetSegments();
+		if (Segments.size() == 1)
+		{
+			IoBufferFileReference FileRef = {};
+			if (PartialBlockBuffer.GetSegments().front().AsIoBuffer().GetFileReference(FileRef))
+			{
+				BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize).MoveToShared().AsIoBuffer();
+				BasicFile Reader;
+				Reader.Attach(FileRef.FileHandle);
+				auto			  _		  = MakeGuard([&Reader]() { Reader.Detach(); });
+				MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView();
+				Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset);
+				return BlockMemoryBuffer;
+			}
+			else
+			{
+				return PartialBlockBuffer.GetSegments().front().AsIoBuffer();
+			}
+		}
+		else
+		{
+			// Not a homogenous memory buffer, read all to memory
+
+			BlockMemoryBuffer		  = UniqueBuffer::Alloc(PartialBlockBuffer.GetSize()).MoveToShared().AsIoBuffer();
+			MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView();
+			for (const SharedBuffer& Segment : Segments)
+			{
+				IoBufferFileReference FileRef = {};
+				if (Segment.AsIoBuffer().GetFileReference(FileRef))
+				{
+					BasicFile Reader;
+					Reader.Attach(FileRef.FileHandle);
+					Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset);
+					Reader.Detach();
+					ReadMem = ReadMem.Mid(FileRef.FileChunkSize);
+				}
+				else
+				{
+					ReadMem = ReadMem.CopyFrom(Segment.AsIoBuffer().GetView());
+				}
+			}
+			return BlockMemoryBuffer;
 		}
 	}
 
@@ -3761,31 +5046,14 @@ namespace {
 						  const ChunkedContentLookup&	   Lookup,
 						  std::span<const IoHash>		   ChunkRawHashes,
 						  std::span<const uint32_t>		   ChunkCompressedLengths,
-						  std::span<const uint32_t>		   ChunkRawLengths,
 						  std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
 						  std::span<std::atomic<bool>>	   RemoteChunkIndexNeedsCopyFromSourceFlags,
-						  CompositeBuffer&&				   PartialBlockBuffer,
+						  const MemoryView				   BlockView,
 						  uint32_t						   FirstIncludedBlockChunkIndex,
 						  uint32_t						   LastIncludedBlockChunkIndex,
 						  BlockWriteOps&				   OutOps)
 	{
 		ZEN_TRACE_CPU("GetBlockWriteOps");
-		MemoryView			  BlockMemoryView;
-		UniqueBuffer		  BlockMemoryBuffer;
-		IoBufferFileReference FileRef = {};
-		if (PartialBlockBuffer.GetSegments().size() == 1 && PartialBlockBuffer.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
-		{
-			BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize);
-			BasicFile Reader;
-			Reader.Attach(FileRef.FileHandle);
-			Reader.Read(BlockMemoryBuffer.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset);
-			BlockMemoryView = BlockMemoryBuffer.GetView();
-			Reader.Detach();
-		}
-		else
-		{
-			BlockMemoryView = PartialBlockBuffer.ViewOrCopyRange(0, PartialBlockBuffer.GetSize(), BlockMemoryBuffer);
-		}
 		uint32_t OffsetInBlock = 0;
 		for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++)
 		{
@@ -3802,34 +5070,36 @@ namespace {
 					bool NeedsWrite = true;
 					if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false))
 					{
-						// CompositeBuffer	 Chunk = PartialBlockBuffer.Mid(OffsetInBlock, ChunkCompressedSize);
-						MemoryView		 ChunkMemory = BlockMemoryView.Mid(OffsetInBlock, ChunkCompressedSize);
-						CompositeBuffer	 Chunk = CompositeBuffer(IoBuffer(IoBuffer::Wrap, ChunkMemory.GetData(), ChunkMemory.GetSize()));
+						MemoryView		 ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize);
 						IoHash			 VerifyChunkHash;
-						uint64_t		 VerifyRawSize;
-						CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Chunk, VerifyChunkHash, VerifyRawSize);
-						if (!Compressed)
-						{
-							ZEN_ASSERT(false);
-						}
-						if (VerifyChunkHash != ChunkHash)
-						{
-							ZEN_ASSERT(false);
-						}
-						if (!ChunkRawLengths.empty())
+						uint64_t		 VerifyChunkSize;
+						CompressedBuffer CompressedChunk =
+							CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize);
+						ZEN_ASSERT(CompressedChunk);
+						ZEN_ASSERT(VerifyChunkHash == ChunkHash);
+						ZEN_ASSERT(VerifyChunkSize == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]);
+
+						OodleCompressor		  ChunkCompressor;
+						OodleCompressionLevel ChunkCompressionLevel;
+						uint64_t			  ChunkBlockSize;
+
+						bool GetCompressParametersSuccess =
+							CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize);
+						ZEN_ASSERT(GetCompressParametersSuccess);
+
+						IoBuffer Decompressed;
+						if (ChunkCompressionLevel == OodleCompressionLevel::None)
 						{
-							if (VerifyRawSize != ChunkRawLengths[ChunkBlockIndex])
-							{
-								ZEN_ASSERT(false);
-							}
+							MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder());
+							Decompressed =
+								IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize());
 						}
-						CompositeBuffer Decompressed = Compressed.DecompressToComposite();
-						if (!Decompressed)
+						else
 						{
-							throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash));
+							Decompressed = CompressedChunk.Decompress().AsIoBuffer();
 						}
-						ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed));
 						ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]);
+						ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed));
 						for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs)
 						{
 							OutOps.WriteOps.push_back(
@@ -3842,19 +5112,22 @@ namespace {
 
 			OffsetInBlock += ChunkCompressedSize;
 		}
-		std::sort(OutOps.WriteOps.begin(),
-				  OutOps.WriteOps.end(),
-				  [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) {
-					  if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex)
-					  {
-						  return true;
-					  }
-					  if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex)
-					  {
-						  return false;
-					  }
-					  return Lhs.Target->Offset < Rhs.Target->Offset;
-				  });
+		{
+			ZEN_TRACE_CPU("GetBlockWriteOps_sort");
+			std::sort(OutOps.WriteOps.begin(),
+					  OutOps.WriteOps.end(),
+					  [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) {
+						  if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex)
+						  {
+							  return true;
+						  }
+						  if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex)
+						  {
+							  return false;
+						  }
+						  return Lhs.Target->Offset < Rhs.Target->Offset;
+					  });
+		}
 		return true;
 	}
 
@@ -3862,34 +5135,35 @@ namespace {
 						  const ChunkedFolderContent&	   RemoteContent,
 						  const ChunkBlockDescription&	   BlockDescription,
 						  std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+						  ParallelWork&					   Work,
+						  WorkerThreadPool&				   VerifyPool,
 						  CompositeBuffer&&				   BlockBuffer,
 						  const ChunkedContentLookup&	   Lookup,
 						  std::span<std::atomic<bool>>	   RemoteChunkIndexNeedsCopyFromSourceFlags,
-						  std::atomic<uint32_t>&		   OutChunksComplete,
-						  std::atomic<uint64_t>&		   OutBytesWritten)
+						  BufferedWriteFileCache&		   WriteCache,
+						  DiskStatistics&				   DiskStats)
 	{
 		ZEN_TRACE_CPU("WriteBlockToDisk");
 
+		IoBuffer		 BlockMemoryBuffer = MakeBufferMemoryBased(BlockBuffer);
+		const MemoryView BlockView		   = BlockMemoryBuffer.GetView();
+
 		BlockWriteOps Ops;
 		if ((BlockDescription.HeaderSize == 0) || BlockDescription.ChunkCompressedLengths.empty())
 		{
 			ZEN_TRACE_CPU("WriteBlockToDisk_Legacy");
 
-			UniqueBuffer				CopyBuffer;
-			const MemoryView			BlockView = BlockBuffer.ViewOrCopyRange(0, BlockBuffer.GetSize(), CopyBuffer);
 			uint64_t					HeaderSize;
-			const std::vector<uint32_t> ChunkCompressedLengths = ReadChunkBlockHeader(BlockView, HeaderSize);
-
-			CompositeBuffer PartialBlockBuffer = std::move(BlockBuffer).Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize);
+			const std::vector<uint32_t> ChunkCompressedLengths =
+				ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize);
 
 			if (GetBlockWriteOps(RemoteContent,
 								 Lookup,
 								 BlockDescription.ChunkRawHashes,
 								 ChunkCompressedLengths,
-								 BlockDescription.ChunkRawLengths,
 								 SequenceIndexChunksLeftToWriteCounters,
 								 RemoteChunkIndexNeedsCopyFromSourceFlags,
-								 std::move(PartialBlockBuffer),
+								 BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize),
 								 0,
 								 gsl::narrow<uint32_t>(BlockDescription.ChunkRawHashes.size() - 1),
 								 Ops))
@@ -3899,23 +5173,22 @@ namespace {
 								   Lookup,
 								   SequenceIndexChunksLeftToWriteCounters,
 								   Ops,
-								   OutChunksComplete,
-								   OutBytesWritten);
+								   WriteCache,
+								   Work,
+								   VerifyPool,
+								   DiskStats);
 				return true;
 			}
 			return false;
 		}
 
-		CompositeBuffer PartialBlockBuffer =
-			std::move(BlockBuffer).Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize);
 		if (GetBlockWriteOps(RemoteContent,
 							 Lookup,
 							 BlockDescription.ChunkRawHashes,
 							 BlockDescription.ChunkCompressedLengths,
-							 BlockDescription.ChunkRawLengths,
 							 SequenceIndexChunksLeftToWriteCounters,
 							 RemoteChunkIndexNeedsCopyFromSourceFlags,
-							 std::move(PartialBlockBuffer),
+							 BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize),
 							 0,
 							 gsl::narrow<uint32_t>(BlockDescription.ChunkRawHashes.size() - 1),
 							 Ops))
@@ -3925,8 +5198,10 @@ namespace {
 							   Lookup,
 							   SequenceIndexChunksLeftToWriteCounters,
 							   Ops,
-							   OutChunksComplete,
-							   OutBytesWritten);
+							   WriteCache,
+							   Work,
+							   VerifyPool,
+							   DiskStats);
 			return true;
 		}
 		return false;
@@ -3936,24 +5211,29 @@ namespace {
 								 const ChunkedFolderContent&	  RemoteContent,
 								 const ChunkBlockDescription&	  BlockDescription,
 								 std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+								 ParallelWork&					  Work,
+								 WorkerThreadPool&				  VerifyPool,
 								 CompositeBuffer&&				  PartialBlockBuffer,
 								 uint32_t						  FirstIncludedBlockChunkIndex,
 								 uint32_t						  LastIncludedBlockChunkIndex,
 								 const ChunkedContentLookup&	  Lookup,
 								 std::span<std::atomic<bool>>	  RemoteChunkIndexNeedsCopyFromSourceFlags,
-								 std::atomic<uint32_t>&			  OutChunksComplete,
-								 std::atomic<uint64_t>&			  OutBytesWritten)
+								 BufferedWriteFileCache&		  WriteCache,
+								 DiskStatistics&				  DiskStats)
 	{
 		ZEN_TRACE_CPU("WritePartialBlockToDisk");
+
+		IoBuffer		 BlockMemoryBuffer = MakeBufferMemoryBased(PartialBlockBuffer);
+		const MemoryView BlockView		   = BlockMemoryBuffer.GetView();
+
 		BlockWriteOps Ops;
 		if (GetBlockWriteOps(RemoteContent,
 							 Lookup,
 							 BlockDescription.ChunkRawHashes,
 							 BlockDescription.ChunkCompressedLengths,
-							 BlockDescription.ChunkRawLengths,
 							 SequenceIndexChunksLeftToWriteCounters,
 							 RemoteChunkIndexNeedsCopyFromSourceFlags,
-							 std::move(PartialBlockBuffer),
+							 BlockView,
 							 FirstIncludedBlockChunkIndex,
 							 LastIncludedBlockChunkIndex,
 							 Ops))
@@ -3963,8 +5243,10 @@ namespace {
 							   Lookup,
 							   SequenceIndexChunksLeftToWriteCounters,
 							   Ops,
-							   OutChunksComplete,
-							   OutBytesWritten);
+							   WriteCache,
+							   Work,
+							   VerifyPool,
+							   DiskStats);
 			return true;
 		}
 		else
@@ -3973,77 +5255,15 @@ namespace {
 		}
 	}
 
-	SharedBuffer Decompress(CompositeBuffer&& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize)
-	{
-		ZEN_TRACE_CPU("Decompress");
-
-		IoHash			 RawHash;
-		uint64_t		 RawSize;
-		CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedChunk, RawHash, RawSize);
-		if (!Compressed)
-		{
-			throw std::runtime_error(fmt::format("Invalid build blob format for chunk {}", ChunkHash));
-		}
-		if (RawHash != ChunkHash)
-		{
-			throw std::runtime_error(fmt::format("Mismatching build blob {}, but compressed header rawhash is {}", ChunkHash, RawHash));
-		}
-		if (RawSize != ChunkRawSize)
-		{
-			throw std::runtime_error(
-				fmt::format("Mismatching build blob {}, expected raw size {} but recevied raw size {}", ChunkHash, ChunkRawSize, RawSize));
-		}
-		if (!Compressed)
-		{
-			throw std::runtime_error(fmt::format("Invalid build blob {}, not a compressed buffer", ChunkHash));
-		}
-
-		SharedBuffer Decompressed = Compressed.Decompress();
-
-		if (!Decompressed)
-		{
-			throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash));
-		}
-		return Decompressed;
-	}
-
-	void WriteChunkToDisk(const std::filesystem::path&										  CacheFolderPath,
-						  const ChunkedFolderContent&										  Content,
-						  const ChunkedContentLookup&										  Lookup,
-						  std::span<const ChunkedContentLookup::ChunkSequenceLocation* const> ChunkTargets,
-						  CompositeBuffer&&													  ChunkData,
-						  WriteFileCache&													  OpenFileCache,
-						  std::atomic<uint64_t>&											  OutBytesWritten)
-	{
-		ZEN_TRACE_CPU("WriteChunkToDisk");
-
-		for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargets)
-		{
-			const auto&	   Target		 = *TargetPtr;
-			const uint64_t FileOffset	 = Target.Offset;
-			const uint32_t SequenceIndex = Target.SequenceIndex;
-			const uint32_t PathIndex	 = Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
-
-			OpenFileCache.WriteToFile(
-				SequenceIndex,
-				[&CacheFolderPath, &Content](uint32_t SequenceIndex) {
-					return GetTempChunkedSequenceFileName(CacheFolderPath, Content.ChunkedContent.SequenceRawHashes[SequenceIndex]);
-				},
-				ChunkData,
-				FileOffset,
-				Content.RawSizes[PathIndex]);
-			OutBytesWritten += ChunkData.GetSize();
-		}
-	}
-
-	bool CanDecompressDirectToSequence(const ChunkedFolderContent&											 RemoteContent,
-									   const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> Locations)
+	bool IsSingleFileChunk(const ChunkedFolderContent&											 RemoteContent,
+						   const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> Locations)
 	{
 		if (Locations.size() == 1)
 		{
 			const uint32_t FirstSequenceIndex = Locations[0]->SequenceIndex;
-			if (Locations[0]->Offset == 0 && RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1)
+			if (RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1)
 			{
+				ZEN_ASSERT_SLOW(Locations[0]->Offset == 0);
 				return true;
 			}
 		}
@@ -4053,7 +5273,7 @@ namespace {
 	void StreamDecompress(const std::filesystem::path& CacheFolderPath,
 						  const IoHash&				   SequenceRawHash,
 						  CompositeBuffer&&			   CompressedPart,
-						  std::atomic<uint64_t>&	   WriteToDiskBytes)
+						  DiskStatistics&			   DiskStats)
 	{
 		ZEN_TRACE_CPU("StreamDecompress");
 		const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash);
@@ -4076,21 +5296,30 @@ namespace {
 		{
 			throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash));
 		}
+		PrepareFileForScatteredWrite(DecompressedTemp.Handle(), RawSize);
+
 		IoHashStream Hash;
-		bool CouldDecompress = Compressed.DecompressToStream(0, (uint64_t)-1, [&](uint64_t Offset, const CompositeBuffer& RangeBuffer) {
-			ZEN_TRACE_CPU("StreamDecompress_Write");
-			if (!AbortFlag)
-			{
-				DecompressedTemp.Write(RangeBuffer, Offset);
-				for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+		bool		 CouldDecompress = Compressed.DecompressToStream(
+			0,
+			(uint64_t)-1,
+			[&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+				ZEN_UNUSED(SourceOffset);
+				ZEN_TRACE_CPU("StreamDecompress_Write");
+				DiskStats.ReadByteCount += SourceSize;
+				if (!AbortFlag)
 				{
-					Hash.Append(Segment.GetView());
+					for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+					{
+						Hash.Append(Segment.GetView());
+						DecompressedTemp.Write(Segment, Offset);
+						Offset += Segment.GetSize();
+						DiskStats.WriteByteCount += Segment.GetSize();
+						DiskStats.WriteCount++;
+					}
+					return true;
 				}
-				WriteToDiskBytes += RangeBuffer.GetSize();
-				return true;
-			}
-			return false;
-		});
+				return false;
+			});
 
 		if (AbortFlag)
 		{
@@ -4113,6 +5342,7 @@ namespace {
 			throw std::runtime_error(
 				fmt::format("Failed moving temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message()));
 		}
+		//		WriteChunkStats.ChunkCountWritten++;
 	}
 
 	bool WriteCompressedChunk(const std::filesystem::path&											 TargetFolder,
@@ -4120,97 +5350,102 @@ namespace {
 							  const ChunkedContentLookup&											 RemoteLookup,
 							  const IoHash&															 ChunkHash,
 							  const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
+							  BufferedWriteFileCache&												 WriteCache,
 							  IoBuffer&&															 CompressedPart,
-							  std::atomic<uint64_t>&												 WriteToDiskBytes)
+							  DiskStatistics&														 DiskStats)
 	{
+		ZEN_TRACE_CPU("WriteCompressedChunk");
 		auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
 		ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end());
-		if (CanDecompressDirectToSequence(RemoteContent, ChunkTargetPtrs))
+		if (IsSingleFileChunk(RemoteContent, ChunkTargetPtrs))
 		{
-			const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[ChunkTargetPtrs.front()->SequenceIndex];
-			StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), WriteToDiskBytes);
+			const std::uint32_t SequenceIndex	= ChunkTargetPtrs.front()->SequenceIndex;
+			const IoHash&		SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex];
+			StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats);
+			return false;
 		}
 		else
 		{
-			const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second;
-			SharedBuffer   Chunk =
-				Decompress(CompositeBuffer(std::move(CompressedPart)), ChunkHash, RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]);
-
-			if (!AbortFlag)
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
+			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompositeBuffer(std::move(CompressedPart)), RawHash, RawSize);
+			if (!Compressed)
 			{
-				WriteFileCache OpenFileCache;
-
-				WriteChunkToDisk(TargetFolder,
-								 RemoteContent,
-								 RemoteLookup,
-								 ChunkTargetPtrs,
-								 CompositeBuffer(std::move(Chunk)),
-								 OpenFileCache,
-								 WriteToDiskBytes);
-				return true;
+				throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", ChunkHash));
 			}
-		}
-		return false;
-	}
-
-	void CompleteChunkTargets(const std::filesystem::path&											 TargetFolder,
-							  const ChunkedFolderContent&											 RemoteContent,
-							  const IoHash&															 ChunkHash,
-							  const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
-							  std::span<std::atomic<uint32_t>>										 SequenceIndexChunksLeftToWriteCounters,
-							  const bool															 NeedHashVerify)
-	{
-		ZEN_TRACE_CPU("CompleteChunkTargets");
-
-		for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs)
-		{
-			const uint32_t RemoteSequenceIndex = Location->SequenceIndex;
-			if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1)
+			if (RawHash != ChunkHash)
 			{
-				const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
-				if (NeedHashVerify)
-				{
-					ZEN_TRACE_CPU("VerifyChunkHash");
+				throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, ChunkHash));
+			}
+
+			BufferedWriteFileCache::Local LocalWriter(WriteCache);
 
-					const IoHash VerifyChunkHash =
-						IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash)));
-					if (VerifyChunkHash != ChunkHash)
+			IoHashStream Hash;
+			bool		 CouldDecompress = Compressed.DecompressToStream(
+				0,
+				(uint64_t)-1,
+				[&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+					ZEN_UNUSED(SourceOffset);
+					ZEN_TRACE_CPU("StreamDecompress_Write");
+					DiskStats.ReadByteCount += SourceSize;
+					if (!AbortFlag)
 					{
-						throw std::runtime_error(
-							fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, ChunkHash));
+						for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargetPtrs)
+						{
+							const auto&	   Target		 = *TargetPtr;
+							const uint64_t FileOffset	 = Target.Offset + Offset;
+							const uint32_t SequenceIndex = Target.SequenceIndex;
+							const uint32_t PathIndex	 = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+
+							WriteSequenceChunk(TargetFolder,
+											   RemoteContent,
+											   LocalWriter,
+											   RangeBuffer,
+											   SequenceIndex,
+											   FileOffset,
+											   PathIndex,
+											   DiskStats);
+						}
+
+						return true;
 					}
-				}
+					return false;
+				});
+
+			if (AbortFlag)
+			{
+				return false;
+			}
 
-				ZEN_TRACE_CPU("RenameToFinal");
-				ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)));
-				std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash),
-										GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash));
+			if (!CouldDecompress)
+			{
+				throw std::runtime_error(fmt::format("Failed to decompress large chunk {}", ChunkHash));
 			}
+
+			return true;
 		}
 	}
 
-	void AsyncWriteDownloadedChunk(const std::filesystem::path&										 Path,
+	void AsyncWriteDownloadedChunk(const std::filesystem::path&										 ZenFolderPath,
 								   const ChunkedFolderContent&										 RemoteContent,
 								   const ChunkedContentLookup&										 RemoteLookup,
 								   uint32_t															 RemoteChunkIndex,
 								   std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>&& ChunkTargetPtrs,
-								   ParallellWork&													 Work,
+								   BufferedWriteFileCache&											 WriteCache,
+								   ParallelWork&													 Work,
 								   WorkerThreadPool&												 WritePool,
 								   IoBuffer&&														 Payload,
 								   std::span<std::atomic<uint32_t>>									 SequenceIndexChunksLeftToWriteCounters,
-								   std::atomic<uint64_t>&											 WriteToDiskBytes,
-								   std::atomic<uint32_t>&											 ChunkCountWritten,
 								   std::atomic<uint64_t>&											 WritePartsComplete,
-								   std::atomic<uint64_t>&											 TotalPartWriteCount,
-								   std::atomic<uint64_t>&											 LooseChunksBytes,
-								   FilteredRate&													 FilteredWrittenBytesPerSecond)
+								   const uint64_t													 TotalPartWriteCount,
+								   FilteredRate&													 FilteredWrittenBytesPerSecond,
+								   DiskStatistics&													 DiskStats)
 	{
 		ZEN_TRACE_CPU("AsyncWriteDownloadedChunk");
 
 		const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
 
-		uint64_t Size = Payload.GetSize();
-		LooseChunksBytes += Size;
+		const uint64_t Size = Payload.GetSize();
 
 		std::filesystem::path CompressedChunkPath;
 
@@ -4226,8 +5461,8 @@ namespace {
 				{
 					Payload.SetDeleteOnClose(false);
 					Payload				= {};
-					CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString();
-					std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec);
+					CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString();
+					RenameFile(TempBlobPath, CompressedChunkPath, Ec);
 					if (Ec)
 					{
 						CompressedChunkPath = std::filesystem::path{};
@@ -4245,17 +5480,26 @@ namespace {
 		{
 			ZEN_TRACE_CPU("WriteTempChunk");
 			// Could not be moved and rather large, lets store it on disk
-			CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString();
+			CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString();
 			TemporaryFile::SafeWriteFile(CompressedChunkPath, Payload);
 			Payload = {};
 		}
 
 		Work.ScheduleWork(
-			WritePool,	// GetSyncWorkerPool(),//
-			[&,
+			WritePool,
+			[&ZenFolderPath,
+			 &RemoteContent,
+			 &RemoteLookup,
 			 SequenceIndexChunksLeftToWriteCounters,
+			 &Work,
+			 &WritePool,
 			 CompressedChunkPath,
 			 RemoteChunkIndex,
+			 TotalPartWriteCount,
+			 &WriteCache,
+			 &DiskStats,
+			 &WritePartsComplete,
+			 &FilteredWrittenBytesPerSecond,
 			 ChunkTargetPtrs = std::move(ChunkTargetPtrs),
 			 CompressedPart	 = std::move(Payload)](std::atomic<bool>&) mutable {
 				ZEN_TRACE_CPU("UpdateFolder_WriteChunk");
@@ -4280,42 +5524,181 @@ namespace {
 						}
 					}
 
-					std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName;
+					std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath);
 
 					bool NeedHashVerify = WriteCompressedChunk(TargetFolder,
 															   RemoteContent,
 															   RemoteLookup,
 															   ChunkHash,
 															   ChunkTargetPtrs,
+															   WriteCache,
 															   std::move(CompressedPart),
-															   WriteToDiskBytes);
-
+															   DiskStats);
 					if (!AbortFlag)
 					{
-						ChunkCountWritten++;
 						WritePartsComplete++;
 						if (WritePartsComplete == TotalPartWriteCount)
 						{
 							FilteredWrittenBytesPerSecond.Stop();
 						}
 
-						std::filesystem::remove(CompressedChunkPath);
+						if (!CompressedChunkPath.empty())
+						{
+							TryRemoveFile(CompressedChunkPath);
+						}
 
-						CompleteChunkTargets(TargetFolder,
-											 RemoteContent,
-											 ChunkHash,
-											 ChunkTargetPtrs,
-											 SequenceIndexChunksLeftToWriteCounters,
-											 NeedHashVerify);
+						std::vector<uint32_t> CompletedSequences =
+							CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
+						WriteCache.Close(CompletedSequences);
+						if (NeedHashVerify)
+						{
+							VerifyAndCompleteChunkSequencesAsync(TargetFolder,
+																 RemoteContent,
+																 RemoteLookup,
+																 CompletedSequences,
+																 Work,
+																 WritePool);
+						}
+						else
+						{
+							FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences);
+						}
 					}
 				}
-			},
-			Work.DefaultErrorFunction());
+			});
 	};
 
-	void UpdateFolder(BuildStorage&								Storage,
+	bool ReadStateFile(const std::filesystem::path& StateFilePath,
+					   FolderContent&				OutLocalFolderState,
+					   ChunkedFolderContent&		OutLocalContent)
+	{
+		ZEN_TRACE_CPU("ReadStateFile");
+		bool HasLocalState = false;
+		try
+		{
+			CbObject CurrentStateObject = LoadCompactBinaryObject(StateFilePath).Object;
+			if (CurrentStateObject)
+			{
+				Oid								  CurrentBuildId;
+				std::vector<Oid>				  SavedBuildPartIds;
+				std::vector<std::string>		  SavedBuildPartsNames;
+				std::vector<ChunkedFolderContent> SavedPartContents;
+				if (ReadStateObject(CurrentStateObject,
+									CurrentBuildId,
+									SavedBuildPartIds,
+									SavedBuildPartsNames,
+									SavedPartContents,
+									OutLocalFolderState))
+				{
+					if (!SavedPartContents.empty())
+					{
+						if (SavedPartContents.size() == 1)
+						{
+							OutLocalContent = std::move(SavedPartContents[0]);
+						}
+						else
+						{
+							OutLocalContent =
+								MergeChunkedFolderContents(SavedPartContents[0],
+														   std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1));
+						}
+						HasLocalState = true;
+					}
+				}
+			}
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_CONSOLE("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
+		}
+		return HasLocalState;
+	}
+
+	FolderContent GetValidFolderContent(GetFolderContentStatistics&											   LocalFolderScanStats,
+										const std::filesystem::path&										   Path,
+										std::span<const std::filesystem::path>								   PathsToCheck,
+										std::function<void(uint64_t PathCount, uint64_t CompletedPathCount)>&& ProgressCallback)
+	{
+		ZEN_TRACE_CPU("GetValidFolderContent");
+		FolderContent  Result;
+		const uint32_t PathCount = gsl::narrow<uint32_t>(PathsToCheck.size());
+
+		Result.Paths.resize(PathCount);
+		Result.RawSizes.resize(PathCount);
+		Result.Attributes.resize(PathCount);
+		Result.ModificationTicks.resize(PathCount);
+
+		{
+			Stopwatch Timer;
+			auto _ = MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
+
+			ParallelWork		  Work(AbortFlag, PauseFlag);
+			std::atomic<uint64_t> CompletedPathCount = 0;
+			uint32_t			  PathIndex			 = 0;
+
+			while (PathIndex < PathCount)
+			{
+				uint32_t PathRangeCount = Min(128u, PathCount - PathIndex);
+				Work.ScheduleWork(GetIOWorkerPool(),
+								  [PathIndex, PathRangeCount, &PathsToCheck, &Path, &Result, &CompletedPathCount, &LocalFolderScanStats](
+									  std::atomic<bool>&) {
+									  for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount;
+										   PathRangeIndex++)
+									  {
+										  const std::filesystem::path& FilePath		 = PathsToCheck[PathRangeIndex];
+										  std::filesystem::path		   LocalFilePath = (Path / FilePath).make_preferred();
+										  if (TryGetFileProperties(LocalFilePath,
+																   Result.RawSizes[PathRangeIndex],
+																   Result.ModificationTicks[PathRangeIndex],
+																   Result.Attributes[PathRangeIndex]))
+										  {
+											  Result.Paths[PathRangeIndex] = std::move(FilePath);
+											  LocalFolderScanStats.FoundFileCount++;
+											  LocalFolderScanStats.FoundFileByteCount += Result.RawSizes[PathRangeIndex];
+											  LocalFolderScanStats.AcceptedFileCount++;
+											  LocalFolderScanStats.AcceptedFileByteCount += Result.RawSizes[PathRangeIndex];
+										  }
+										  CompletedPathCount++;
+									  }
+								  });
+				PathIndex += PathRangeCount;
+			}
+			Work.Wait(200, [&](bool, bool, ptrdiff_t) {
+				if (ProgressCallback)
+				{
+					ProgressCallback(PathCount, CompletedPathCount.load());
+				}
+			});
+		}
+
+		uint32_t WritePathIndex = 0;
+		for (uint32_t ReadPathIndex = 0; ReadPathIndex < PathCount; ReadPathIndex++)
+		{
+			if (!Result.Paths[ReadPathIndex].empty())
+			{
+				if (WritePathIndex < ReadPathIndex)
+				{
+					Result.Paths[WritePathIndex]			 = std::move(Result.Paths[ReadPathIndex]);
+					Result.RawSizes[WritePathIndex]			 = Result.RawSizes[ReadPathIndex];
+					Result.Attributes[WritePathIndex]		 = Result.Attributes[ReadPathIndex];
+					Result.ModificationTicks[WritePathIndex] = Result.ModificationTicks[ReadPathIndex];
+				}
+				WritePathIndex++;
+			}
+		}
+
+		Result.Paths.resize(WritePathIndex);
+		Result.RawSizes.resize(WritePathIndex);
+		Result.Attributes.resize(WritePathIndex);
+		Result.ModificationTicks.resize(WritePathIndex);
+		return Result;
+	}
+
+	void UpdateFolder(const std::filesystem::path&				SystemRootDir,
+					  StorageInstance&							Storage,
 					  const Oid&								BuildId,
 					  const std::filesystem::path&				Path,
+					  const std::filesystem::path&				ZenFolderPath,
 					  const std::uint64_t						LargeAttachmentSize,
 					  const std::uint64_t						PreferredMultipartChunkSize,
 					  const ChunkedFolderContent&				LocalContent,
@@ -4324,19 +5707,18 @@ namespace {
 					  const std::vector<IoHash>&				LooseChunkHashes,
 					  bool										AllowPartialBlockRequests,
 					  bool										WipeTargetFolder,
-					  FolderContent&							OutLocalFolderState)
+					  bool										PrimeCacheOnly,
+					  bool										EnableScavenging,
+					  FolderContent&							OutLocalFolderState,
+					  DiskStatistics&							DiskStats,
+					  CacheMappingStatistics&					CacheMappingStats,
+					  DownloadStatistics&						DownloadStats,
+					  WriteChunkStatistics&						WriteChunkStats,
+					  RebuildFolderStateStatistics&				RebuildFolderStateStats)
 	{
 		ZEN_TRACE_CPU("UpdateFolder");
 
-		ZEN_UNUSED(WipeTargetFolder);
-		std::atomic<uint64_t> DownloadedBlocks		   = 0;
-		std::atomic<uint64_t> BlockBytes			   = 0;
-		std::atomic<uint64_t> DownloadedChunks		   = 0;
-		std::atomic<uint64_t> LooseChunksBytes		   = 0;
-		std::atomic<uint64_t> WriteToDiskBytes		   = 0;
-		std::atomic<uint64_t> MultipartAttachmentCount = 0;
-
-		DiskStatistics DiskStats;
+		ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests)));
 
 		Stopwatch IndexTimer;
 
@@ -4346,23 +5728,22 @@ namespace {
 
 		ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
 
-		const std::filesystem::path CacheFolderPath = Path / ZenTempCacheFolderName;
+		const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath);
 
 		Stopwatch CacheMappingTimer;
 
 		std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size());
-		//		std::vector<bool> RemoteSequenceIndexIsCachedFlags(RemoteContent.ChunkedContent.SequenceRawHashes.size(), false);
-		std::vector<bool> RemoteChunkIndexNeedsCopyFromLocalFileFlags(RemoteContent.ChunkedContent.ChunkHashes.size());
-		// Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly)
-		std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size());
+		std::vector<bool>				   RemoteChunkIndexNeedsCopyFromLocalFileFlags(RemoteContent.ChunkedContent.ChunkHashes.size());
+		std::vector<std::atomic<bool>>	   RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size());
 
 		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound;
 		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound;
-		uint64_t										 CachedChunkHashesByteCountFound	= 0;
-		uint64_t										 CachedSequenceHashesByteCountFound = 0;
+		if (!PrimeCacheOnly)
 		{
 			ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache");
 
+			Stopwatch CacheTimer;
+
 			DirectoryContent CacheDirContent;
 			GetDirectoryContent(CacheFolderPath,
 								DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
@@ -4380,7 +5761,8 @@ namespace {
 						if (ChunkSize == CacheDirContent.FileSizes[Index])
 						{
 							CachedChunkHashesFound.insert({FileHash, ChunkIndex});
-							CachedChunkHashesByteCountFound += ChunkSize;
+							CacheMappingStats.CacheChunkCount++;
+							CacheMappingStats.CacheChunkByteCount += ChunkSize;
 							continue;
 						}
 					}
@@ -4393,20 +5775,30 @@ namespace {
 						if (SequenceSize == CacheDirContent.FileSizes[Index])
 						{
 							CachedSequenceHashesFound.insert({FileHash, SequenceIndex});
-							CachedSequenceHashesByteCountFound += SequenceSize;
+							CacheMappingStats.CacheSequenceHashesCount++;
+							CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize;
+
+							const std::filesystem::path CacheFilePath =
+								GetFinalChunkedSequenceFileName(CacheFolderPath,
+																RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+							ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
+
 							continue;
 						}
 					}
 				}
-				std::filesystem::remove(CacheDirContent.Files[Index]);
+				TryRemoveFile(CacheDirContent.Files[Index]);
 			}
+			CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
 		}
 
 		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound;
-		uint64_t										 CachedBlocksByteCountFound = 0;
+		if (!PrimeCacheOnly)
 		{
 			ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache");
 
+			Stopwatch CacheTimer;
+
 			tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> AllBlockSizes;
 			AllBlockSizes.reserve(BlockDescriptions.size());
 			for (uint32_t BlockIndex = 0; BlockIndex < BlockDescriptions.size(); BlockIndex++)
@@ -4416,7 +5808,7 @@ namespace {
 			}
 
 			DirectoryContent BlockDirContent;
-			GetDirectoryContent(Path / ZenTempBlockFolderName,
+			GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath),
 								DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
 								BlockDirContent);
 			CachedBlocksFound.reserve(BlockDirContent.Files.size());
@@ -4438,64 +5830,335 @@ namespace {
 						if (BlockSize == BlockDirContent.FileSizes[Index])
 						{
 							CachedBlocksFound.insert({FileHash, BlockIndex});
-							CachedBlocksByteCountFound += BlockSize;
+							CacheMappingStats.CacheBlockCount++;
+							CacheMappingStats.CacheBlocksByteCount += BlockSize;
 							continue;
 						}
 					}
 				}
-				std::filesystem::remove(BlockDirContent.Files[Index]);
+				TryRemoveFile(BlockDirContent.Files[Index]);
 			}
+
+			CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
 		}
 
-		std::vector<uint32_t> LocalPathIndexesMatchingSequenceIndexes;
-		uint64_t			  LocalPathIndexesByteCountMatchingSequenceIndexes = 0;
-		// Pick up all whole files we can use from current local state
+		std::vector<uint32_t>							 LocalPathIndexesMatchingSequenceIndexes;
+		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceIndexesLeftToFindToRemoteIndex;
+
+		if (!PrimeCacheOnly)
 		{
-			ZEN_TRACE_CPU("UpdateFolder_CheckLocalChunks");
+			// Pick up all whole files we can use from current local state
+			ZEN_TRACE_CPU("UpdateFolder_GetLocalSequences");
+
+			Stopwatch LocalTimer;
+
 			for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size();
 				 RemoteSequenceIndex++)
 			{
-				const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+				const IoHash&  RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+				const uint32_t RemotePathIndex		 = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex);
+				const uint64_t RemoteRawSize		 = RemoteContent.RawSizes[RemotePathIndex];
 				if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash);
 					CacheSequenceIt != CachedSequenceHashesFound.end())
 				{
-					//				const uint32_t RemoteSequenceIndex = CacheSequenceIt->second;
-					//				const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex);
-					//				RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex];
+					const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+					ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
+					ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize));
 				}
 				else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash);
 						 CacheChunkIt != CachedChunkHashesFound.end())
 				{
-					//				const uint32_t RemoteChunkIndex = CacheChunkIt->second;
-					//				const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex);
-					//				RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex];
+					const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+					ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
+					ZEN_CONSOLE_VERBOSE("Found chunk {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize));
 				}
 				else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash);
 						 It != LocalLookup.RawHashToSequenceIndex.end())
 				{
-					const uint32_t LocalSequenceIndex = It->second;
-					const uint32_t LocalPathIndex	  = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex);
-					uint64_t	   RawSize			  = LocalContent.RawSizes[LocalPathIndex];
+					const uint32_t				LocalSequenceIndex = It->second;
+					const uint32_t				LocalPathIndex	   = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex);
+					const std::filesystem::path LocalFilePath	   = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+					ZEN_ASSERT_SLOW(IsFile(LocalFilePath));
 					LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex);
-					LocalPathIndexesByteCountMatchingSequenceIndexes += RawSize;
+					CacheMappingStats.LocalPathsMatchingSequencesCount++;
+					CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize;
+					ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, LocalFilePath, NiceBytes(RemoteRawSize));
 				}
 				else
 				{
 					// We must write the sequence
-					SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] =
-						RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
+					const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
+					SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
+					SequenceIndexesLeftToFindToRemoteIndex.insert({RemoteSequenceRawHash, RemoteSequenceIndex});
+				}
+			}
+
+			CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
+		}
+		else
+		{
+			for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size();
+				 RemoteSequenceIndex++)
+			{
+				const uint32_t ChunkCount									= RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
+				SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
+			}
+		}
+
+		std::vector<ChunkedFolderContent>  ScavengedContents;
+		std::vector<ChunkedContentLookup>  ScavengedLookups;
+		std::vector<std::filesystem::path> ScavengedPaths;
+
+		struct ScavengeCopyOperation
+		{
+			uint32_t ScavengedContentIndex = (uint32_t)-1;
+			uint32_t ScavengedPathIndex	   = (uint32_t)-1;
+			uint32_t RemoteSequenceIndex   = (uint32_t)-1;
+			uint64_t RawSize			   = (uint32_t)-1;
+		};
+
+		std::vector<ScavengeCopyOperation> ScavengeCopyOperations;
+		uint64_t						   ScavengedPathsCount = 0;
+
+		if (!PrimeCacheOnly && EnableScavenging)
+		{
+			ZEN_TRACE_CPU("UpdateFolder_GetScavengedSequences");
+
+			Stopwatch ScavengeTimer;
+
+			if (!SequenceIndexesLeftToFindToRemoteIndex.empty())
+			{
+				std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(SystemRootDir);
+				auto EraseIt = std::remove_if(ScavengeSources.begin(), ScavengeSources.end(), [&Path](const ScavengeSource& Source) {
+					return Source.Path == Path;
+				});
+				ScavengeSources.erase(EraseIt, ScavengeSources.end());
+
+				const size_t ScavengePathCount = ScavengeSources.size();
+
+				ScavengedContents.resize(ScavengePathCount);
+				ScavengedLookups.resize(ScavengePathCount);
+				ScavengedPaths.resize(ScavengePathCount);
+
+				ProgressBar	 ScavengeProgressBar(ProgressMode, "Scavenging");
+				ParallelWork Work(AbortFlag, PauseFlag);
+
+				std::atomic<uint64_t> PathsFound(0);
+				std::atomic<uint64_t> ChunksFound(0);
+				std::atomic<uint64_t> PathsScavenged(0);
+
+				for (size_t ScavengeIndex = 0; ScavengeIndex < ScavengePathCount; ScavengeIndex++)
+				{
+					Work.ScheduleWork(
+						GetIOWorkerPool(),
+						[&RemoteLookup,
+						 &ScavengeSources,
+						 &ScavengedContents,
+						 &ScavengedPaths,
+						 &ScavengedLookups,
+						 &PathsFound,
+						 &ChunksFound,
+						 &PathsScavenged,
+						 ScavengeIndex](std::atomic<bool>&) {
+							if (!AbortFlag)
+							{
+								const ScavengeSource& Source = ScavengeSources[ScavengeIndex];
+
+								ChunkedFolderContent&  ScavengedLocalContent = ScavengedContents[ScavengeIndex];
+								std::filesystem::path& ScavengePath			 = ScavengedPaths[ScavengeIndex];
+
+								FolderContent LocalFolderState;
+								if (ReadStateFile(Source.StateFilePath, LocalFolderState, ScavengedLocalContent))
+								{
+									if (IsDir(Source.Path))
+									{
+										ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengeIndex];
+										ScavengedLookup						  = BuildChunkedContentLookup(ScavengedLocalContent);
+
+										std::vector<uint32_t> PathIndexesToScavange;
+										uint32_t ScavengedStatePathCount = gsl::narrow<uint32_t>(ScavengedLocalContent.Paths.size());
+										PathIndexesToScavange.reserve(ScavengedStatePathCount);
+										for (uint32_t ScavengedStatePathIndex = 0; ScavengedStatePathIndex < ScavengedStatePathCount;
+											 ScavengedStatePathIndex++)
+										{
+											const IoHash& SequenceHash = ScavengedLocalContent.RawHashes[ScavengedStatePathIndex];
+											if (auto ScavengeSequenceIt = ScavengedLookup.RawHashToSequenceIndex.find(SequenceHash);
+												ScavengeSequenceIt != ScavengedLookup.RawHashToSequenceIndex.end())
+											{
+												const uint32_t ScavengeSequenceIndex = ScavengeSequenceIt->second;
+												if (RemoteLookup.RawHashToSequenceIndex.contains(SequenceHash))
+												{
+													PathIndexesToScavange.push_back(ScavengedStatePathIndex);
+												}
+												else
+												{
+													const uint32_t ScavengeChunkCount =
+														ScavengedLocalContent.ChunkedContent.ChunkCounts[ScavengeSequenceIndex];
+													for (uint32_t ScavengeChunkIndexOffset = 0;
+														 ScavengeChunkIndexOffset < ScavengeChunkCount;
+														 ScavengeChunkIndexOffset++)
+													{
+														const size_t ScavengeChunkOrderIndex =
+															ScavengedLookup.ChunkSequenceLocationOffset[ScavengeSequenceIndex] +
+															ScavengeChunkIndexOffset;
+														const uint32_t ScavengeChunkIndex =
+															ScavengedLocalContent.ChunkedContent.ChunkOrders[ScavengeChunkOrderIndex];
+														const IoHash& ScavengeChunkHash =
+															ScavengedLocalContent.ChunkedContent.ChunkHashes[ScavengeChunkIndex];
+														if (RemoteLookup.ChunkHashToChunkIndex.contains(ScavengeChunkHash))
+														{
+															PathIndexesToScavange.push_back(ScavengedStatePathIndex);
+															break;
+														}
+													}
+												}
+											}
+										}
+
+										if (!PathIndexesToScavange.empty())
+										{
+											std::vector<std::filesystem::path> PathsToScavenge;
+											PathsToScavenge.reserve(PathIndexesToScavange.size());
+											for (uint32_t ScavengedStatePathIndex : PathIndexesToScavange)
+											{
+												PathsToScavenge.push_back(ScavengedLocalContent.Paths[ScavengedStatePathIndex]);
+											}
+
+											GetFolderContentStatistics ScavengedFolderScanStats;
+
+											FolderContent ValidFolderContent =
+												GetValidFolderContent(ScavengedFolderScanStats, Source.Path, PathsToScavenge, {});
+
+											if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent))
+											{
+												std::vector<std::filesystem::path> DeletedPaths;
+												FolderContent					   UpdatedContent =
+													GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths);
+
+												// If the files are modified since the state was saved we ignore the files since we don't
+												// want to incur the cost of scanning/hashing scavenged files
+												DeletedPaths.insert(DeletedPaths.end(),
+																	UpdatedContent.Paths.begin(),
+																	UpdatedContent.Paths.end());
+												if (!DeletedPaths.empty())
+												{
+													ScavengedLocalContent =
+														DeletePathsFromChunkedContent(ScavengedLocalContent, ScavengedLookup, DeletedPaths);
+													ScavengedLookup = BuildChunkedContentLookup(ScavengedLocalContent);
+												}
+											}
+
+											if (!ScavengedLocalContent.Paths.empty())
+											{
+												ScavengePath = Source.Path;
+												PathsFound += ScavengedLocalContent.Paths.size();
+												ChunksFound += ScavengedLocalContent.ChunkedContent.ChunkHashes.size();
+											}
+										}
+
+										if (ScavengePath.empty())
+										{
+											ScavengedLocalContent			= {};
+											ScavengedLookups[ScavengeIndex] = {};
+											ScavengedPaths[ScavengeIndex].clear();
+										}
+									}
+								}
+								PathsScavenged++;
+							}
+						});
+				}
+				{
+					ZEN_TRACE_CPU("ScavengeScan_Wait");
+
+					Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+						ZEN_UNUSED(PendingWork);
+						std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavanging",
+														  PathsScavenged.load(),
+														  ScavengePathCount,
+														  PathsFound.load(),
+														  ChunksFound.load());
+						ScavengeProgressBar.UpdateState({.Task			 = "Scavenging       ",
+														 .Details		 = Details,
+														 .TotalCount	 = ScavengePathCount,
+														 .RemainingCount = ScavengePathCount - PathsScavenged.load(),
+														 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+														false);
+					});
+				}
+
+				ScavengeProgressBar.Finish();
+				if (AbortFlag)
+				{
+					return;
+				}
+
+				for (uint32_t ScavengedContentIndex = 0;
+					 ScavengedContentIndex < ScavengedContents.size() && (!SequenceIndexesLeftToFindToRemoteIndex.empty());
+					 ScavengedContentIndex++)
+				{
+					const std::filesystem::path& ScavengePath = ScavengedPaths[ScavengedContentIndex];
+					if (!ScavengePath.empty())
+					{
+						const ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengedContentIndex];
+						const ChunkedContentLookup& ScavengedLookup		  = ScavengedLookups[ScavengedContentIndex];
+
+						for (uint32_t ScavengedSequenceIndex = 0;
+							 ScavengedSequenceIndex < ScavengedLocalContent.ChunkedContent.SequenceRawHashes.size();
+							 ScavengedSequenceIndex++)
+						{
+							const IoHash& SequenceRawHash = ScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex];
+							if (auto It = SequenceIndexesLeftToFindToRemoteIndex.find(SequenceRawHash);
+								It != SequenceIndexesLeftToFindToRemoteIndex.end())
+							{
+								const uint32_t RemoteSequenceIndex = It->second;
+								const uint64_t RawSize =
+									RemoteContent.RawSizes[RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]];
+								ZEN_ASSERT(RawSize > 0);
+
+								const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex];
+								ZEN_ASSERT_SLOW(IsFile((ScavengePath / ScavengedLocalContent.Paths[ScavengedPathIndex]).make_preferred()));
+
+								ScavengeCopyOperations.push_back({.ScavengedContentIndex = ScavengedContentIndex,
+																  .ScavengedPathIndex	 = ScavengedPathIndex,
+																  .RemoteSequenceIndex	 = RemoteSequenceIndex,
+																  .RawSize				 = RawSize});
+
+								SequenceIndexesLeftToFindToRemoteIndex.erase(SequenceRawHash);
+								SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0;
+
+								CacheMappingStats.ScavengedPathsMatchingSequencesCount++;
+								CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize;
+							}
+						}
+						ScavengedPathsCount++;
+					}
 				}
 			}
+			CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
 		}
+
+		uint32_t RemainingChunkCount = 0;
+		for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
+		{
+			uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+			if (ChunkWriteCount > 0)
+			{
+				RemainingChunkCount++;
+			}
+		}
+
 		// Pick up all chunks in current local state
+		// TODO: Rename to LocalStateCopyData
 		struct CacheCopyData
 		{
-			uint32_t														LocalSequenceIndex = (uint32_t)-1;
+			uint32_t														ScavengeSourceIndex = (uint32_t)-1;
+			uint32_t														SourceSequenceIndex = (uint32_t)-1;
 			std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> TargetChunkLocationPtrs;
 			struct ChunkTarget
 			{
 				uint32_t TargetChunkLocationCount = (uint32_t)-1;
-				uint64_t ChunkRawSize			  = (uint64_t)-1;
+				uint32_t RemoteChunkIndex		  = (uint32_t)-1;
 				uint64_t CacheFileOffset		  = (uint64_t)-1;
 			};
 			std::vector<ChunkTarget> ChunkTargets;
@@ -4503,13 +6166,15 @@ namespace {
 
 		tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCacheCopyDataIndex;
 		std::vector<CacheCopyData>					   CacheCopyDatas;
-		uint64_t									   LocalChunkHashesMatchingRemoteCount	   = 0;
-		uint64_t									   LocalChunkHashesMatchingRemoteByteCount = 0;
 
+		if (!PrimeCacheOnly)
 		{
 			ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks");
 
-			for (uint32_t LocalSequenceIndex = 0; LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size();
+			Stopwatch LocalTimer;
+
+			for (uint32_t LocalSequenceIndex = 0;
+				 LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0);
 				 LocalSequenceIndex++)
 			{
 				const IoHash&  LocalSequenceRawHash = LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex];
@@ -4537,7 +6202,7 @@ namespace {
 								{
 									CacheCopyData::ChunkTarget Target = {
 										.TargetChunkLocationCount = gsl::narrow<uint32_t>(ChunkTargetPtrs.size()),
-										.ChunkRawSize			  = LocalChunkRawSize,
+										.RemoteChunkIndex		  = RemoteChunkIndex,
 										.CacheFileOffset		  = SourceOffset};
 									if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalSequenceRawHash);
 										CopySourceIt != RawHashToCacheCopyDataIndex.end())
@@ -4547,7 +6212,8 @@ namespace {
 										{
 											RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size());
 											CacheCopyDatas.push_back(
-												CacheCopyData{.LocalSequenceIndex	   = LocalSequenceIndex,
+												CacheCopyData{.ScavengeSourceIndex	   = (uint32_t)-1,
+															  .SourceSequenceIndex	   = LocalSequenceIndex,
 															  .TargetChunkLocationPtrs = ChunkTargetPtrs,
 															  .ChunkTargets			   = std::vector<CacheCopyData::ChunkTarget>{Target}});
 										}
@@ -4563,13 +6229,15 @@ namespace {
 									{
 										RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size());
 										CacheCopyDatas.push_back(
-											CacheCopyData{.LocalSequenceIndex	   = LocalSequenceIndex,
+											CacheCopyData{.ScavengeSourceIndex	   = (uint32_t)-1,
+														  .SourceSequenceIndex	   = LocalSequenceIndex,
 														  .TargetChunkLocationPtrs = ChunkTargetPtrs,
 														  .ChunkTargets			   = std::vector<CacheCopyData::ChunkTarget>{Target}});
 									}
-									LocalChunkHashesMatchingRemoteByteCount += LocalChunkRawSize;
-									LocalChunkHashesMatchingRemoteCount++;
+									CacheMappingStats.LocalChunkMatchingRemoteCount++;
+									CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize;
 									RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
+									RemainingChunkCount--;
 								}
 							}
 						}
@@ -4577,48 +6245,154 @@ namespace {
 					}
 				}
 			}
+			CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
 		}
 
-		if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty() ||
-			!LocalPathIndexesMatchingSequenceIndexes.empty() || LocalChunkHashesMatchingRemoteCount > 0)
+		if (!PrimeCacheOnly)
 		{
-			ZEN_CONSOLE(
-				"Cache: {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks. Local state: {} ({}) chunk sequences, {} ({}) chunks",
-				CachedSequenceHashesFound.size(),
-				NiceBytes(CachedSequenceHashesByteCountFound),
-				CachedChunkHashesFound.size(),
-				NiceBytes(CachedChunkHashesByteCountFound),
-				CachedBlocksFound.size(),
-				NiceBytes(CachedBlocksByteCountFound),
-				LocalPathIndexesMatchingSequenceIndexes.size(),
-				NiceBytes(LocalPathIndexesByteCountMatchingSequenceIndexes),
-				LocalChunkHashesMatchingRemoteCount,
-				NiceBytes(LocalChunkHashesMatchingRemoteByteCount));
+			ZEN_TRACE_CPU("UpdateFolder_GetScavengeChunks");
+
+			Stopwatch ScavengeTimer;
+
+			for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (RemainingChunkCount > 0);
+				 ScavengedContentIndex++)
+			{
+				const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengedContentIndex];
+				//				const std::filesystem::path& ScavengedPath = ScavengedPaths[ScavengedContentIndex];
+				const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex];
+
+				for (uint32_t ScavengedSequenceIndex = 0;
+					 ScavengedSequenceIndex < ScavengedContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0);
+					 ScavengedSequenceIndex++)
+				{
+					const IoHash&  ScavengedSequenceRawHash = ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex];
+					const uint32_t ScavengedOrderOffset		= ScavengedLookup.SequenceIndexChunkOrderOffset[ScavengedSequenceIndex];
+
+					{
+						uint64_t	   SourceOffset		   = 0;
+						const uint32_t ScavengedChunkCount = ScavengedContent.ChunkedContent.ChunkCounts[ScavengedSequenceIndex];
+						for (uint32_t ScavengedOrderIndex = 0; ScavengedOrderIndex < ScavengedChunkCount; ScavengedOrderIndex++)
+						{
+							const uint32_t ScavengedChunkIndex =
+								ScavengedContent.ChunkedContent.ChunkOrders[ScavengedOrderOffset + ScavengedOrderIndex];
+							const IoHash&  ScavengedChunkHash	 = ScavengedContent.ChunkedContent.ChunkHashes[ScavengedChunkIndex];
+							const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex];
+
+							if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(ScavengedChunkHash);
+								RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end())
+							{
+								const uint32_t RemoteChunkIndex = RemoteChunkIt->second;
+								if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+								{
+									std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
+										GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+
+									if (!ChunkTargetPtrs.empty())
+									{
+										CacheCopyData::ChunkTarget Target = {
+											.TargetChunkLocationCount = gsl::narrow<uint32_t>(ChunkTargetPtrs.size()),
+											.RemoteChunkIndex		  = RemoteChunkIndex,
+											.CacheFileOffset		  = SourceOffset};
+										if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(ScavengedSequenceRawHash);
+											CopySourceIt != RawHashToCacheCopyDataIndex.end())
+										{
+											CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second];
+											if (Data.TargetChunkLocationPtrs.size() > 1024)
+											{
+												RawHashToCacheCopyDataIndex.insert_or_assign(ScavengedSequenceRawHash,
+																							 CacheCopyDatas.size());
+												CacheCopyDatas.push_back(
+													CacheCopyData{.ScavengeSourceIndex	   = ScavengedContentIndex,
+																  .SourceSequenceIndex	   = ScavengedSequenceIndex,
+																  .TargetChunkLocationPtrs = ChunkTargetPtrs,
+																  .ChunkTargets = std::vector<CacheCopyData::ChunkTarget>{Target}});
+											}
+											else
+											{
+												Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(),
+																					ChunkTargetPtrs.begin(),
+																					ChunkTargetPtrs.end());
+												Data.ChunkTargets.push_back(Target);
+											}
+										}
+										else
+										{
+											RawHashToCacheCopyDataIndex.insert_or_assign(ScavengedSequenceRawHash, CacheCopyDatas.size());
+											CacheCopyDatas.push_back(
+												CacheCopyData{.ScavengeSourceIndex	   = ScavengedContentIndex,
+															  .SourceSequenceIndex	   = ScavengedSequenceIndex,
+															  .TargetChunkLocationPtrs = ChunkTargetPtrs,
+															  .ChunkTargets			   = std::vector<CacheCopyData::ChunkTarget>{Target}});
+										}
+										CacheMappingStats.ScavengedChunkMatchingRemoteCount++;
+										CacheMappingStats.ScavengedChunkMatchingRemoteByteCount += ScavengedChunkRawSize;
+										RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
+										RemainingChunkCount--;
+									}
+								}
+							}
+							SourceOffset += ScavengedChunkRawSize;
+						}
+					}
+				}
+			}
+			CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
+		}
+
+		if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
+		{
+			ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
+						CachedSequenceHashesFound.size(),
+						NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
+						CachedChunkHashesFound.size(),
+						NiceBytes(CacheMappingStats.CacheChunkByteCount),
+						CachedBlocksFound.size(),
+						NiceBytes(CacheMappingStats.CacheBlocksByteCount),
+						NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
 		}
 
-		uint32_t ChunkCountToWrite = 0;
+		if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
+		{
+			ZEN_CONSOLE("Local state   : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
+						LocalPathIndexesMatchingSequenceIndexes.size(),
+						NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
+						CacheMappingStats.LocalChunkMatchingRemoteCount,
+						NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
+						NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
+		}
+		if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
+		{
+			ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
+						ScavengedPathsCount,
+						CacheMappingStats.ScavengedPathsMatchingSequencesCount,
+						NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
+						CacheMappingStats.ScavengedChunkMatchingRemoteCount,
+						NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
+						NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+		}
+
+		uint64_t BytesToWrite = 0;
+
 		for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
 		{
-			if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
-			{
-				ChunkCountToWrite++;
-			}
-			else
+			uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+			if (ChunkWriteCount > 0)
 			{
-				std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
-					GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
-				if (!ChunkTargetPtrs.empty())
+				BytesToWrite += RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount;
+				if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
 				{
 					RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true;
-					ChunkCountToWrite++;
 				}
 			}
 		}
 
+		for (const ScavengeCopyOperation& ScavengeCopyOp : ScavengeCopyOperations)
+		{
+			BytesToWrite += ScavengeCopyOp.RawSize;
+		}
+
 		uint64_t			  TotalRequestCount	  = 0;
-		std::atomic<uint64_t> RequestsComplete	  = 0;
-		std::atomic<uint32_t> ChunkCountWritten	  = 0;
-		std::atomic<uint64_t> TotalPartWriteCount = 0;
+		uint64_t			  TotalPartWriteCount = 0;
 		std::atomic<uint64_t> WritePartsComplete  = 0;
 
 		{
@@ -4629,13 +6403,11 @@ namespace {
 			FilteredRate FilteredDownloadedBytesPerSecond;
 			FilteredRate FilteredWrittenBytesPerSecond;
 
-			WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst);	//  GetSyncWorkerPool();  //
-			WorkerThreadPool& WritePool	  = GetMediumWorkerPool(EWorkloadType::Burst);	// GetSyncWorkerPool();  //
+			WorkerThreadPool& NetworkPool = GetNetworkPool();
+			WorkerThreadPool& WritePool	  = GetIOWorkerPool();
 
-			ProgressBar	  WriteProgressBar(UsePlainProgress);
-			ParallellWork Work(AbortFlag);
-
-			std::atomic<uint64_t> BytesDownloaded = 0;
+			ProgressBar	 WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing");
+			ParallelWork Work(AbortFlag, PauseFlag);
 
 			struct LooseChunkHashWorkData
 			{
@@ -4645,6 +6417,7 @@ namespace {
 
 			std::vector<LooseChunkHashWorkData> LooseChunkHashWorks;
 			TotalPartWriteCount += CacheCopyDatas.size();
+			TotalPartWriteCount += ScavengeCopyOperations.size();
 
 			for (const IoHash ChunkHash : LooseChunkHashes)
 			{
@@ -4653,7 +6426,7 @@ namespace {
 				const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second;
 				if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
 				{
-					ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash);
+					ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash);
 					continue;
 				}
 				bool NeedsCopy = true;
@@ -4664,7 +6437,7 @@ namespace {
 
 					if (ChunkTargetPtrs.empty())
 					{
-						ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash);
+						ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash);
 					}
 					else
 					{
@@ -4718,164 +6491,741 @@ namespace {
 
 			std::vector<uint32_t> FullBlockWorks;
 
-			size_t	 BlocksNeededCount	= 0;
-			uint64_t AllBlocksSize		= 0;
-			uint64_t AllBlocksFetch		= 0;
-			uint64_t AllBlocksSlack		= 0;
-			uint64_t AllBlockRequests	= 0;
-			uint64_t AllBlockChunksSize = 0;
 			for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++)
 			{
 				const ChunkBlockDescription& BlockDescription	   = BlockDescriptions[BlockIndex];
 				const std::vector<uint32_t>	 BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription);
 				if (!BlockChunkIndexNeeded.empty())
 				{
-					bool UsingCachedBlock = false;
-					if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end())
+					if (PrimeCacheOnly)
 					{
-						ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet");
-
+						TotalRequestCount++;
 						TotalPartWriteCount++;
 
-						std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString();
-						if (std::filesystem::exists(BlockPath))
-						{
-							CachedChunkBlockIndexes.push_back(BlockIndex);
-							UsingCachedBlock = true;
-						}
+						FullBlockWorks.push_back(BlockIndex);
 					}
-
-					if (!UsingCachedBlock)
+					else
 					{
-						bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size();
-						bool CanDoPartialBlockDownload =
-							(BlockDescription.HeaderSize > 0) &&
-							(BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size());
-						if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload)
+						bool UsingCachedBlock = false;
+						if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end())
 						{
-							std::vector<BlockRangeDescriptor> BlockRanges;
+							ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet");
 
-							ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis");
+							TotalPartWriteCount++;
 
-							uint32_t NeedBlockChunkIndexOffset = 0;
-							uint32_t ChunkBlockIndex		   = 0;
-							uint32_t CurrentOffset =
-								gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize);
+							std::filesystem::path BlockPath =
+								ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+							if (IsFile(BlockPath))
+							{
+								CachedChunkBlockIndexes.push_back(BlockIndex);
+								UsingCachedBlock = true;
+							}
+						}
 
-							BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex};
-							while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() &&
-								   ChunkBlockIndex < BlockDescription.ChunkRawHashes.size())
+						if (!UsingCachedBlock)
+						{
+							bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size();
+							bool CanDoPartialBlockDownload =
+								(BlockDescription.HeaderSize > 0) &&
+								(BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size());
+							if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload)
 							{
-								const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex];
-								if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset])
+								std::vector<BlockRangeDescriptor> BlockRanges;
+
+								ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis");
+
+								uint32_t NeedBlockChunkIndexOffset = 0;
+								uint32_t ChunkBlockIndex		   = 0;
+								uint32_t CurrentOffset =
+									gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize);
+
+								const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(),
+																				BlockDescription.ChunkCompressedLengths.end(),
+																				std::uint64_t(CurrentOffset));
+
+								BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex};
+								while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() &&
+									   ChunkBlockIndex < BlockDescription.ChunkRawHashes.size())
 								{
-									if (NextRange.RangeLength > 0)
+									const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex];
+									if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset])
+									{
+										if (NextRange.RangeLength > 0)
+										{
+											BlockRanges.push_back(NextRange);
+											NextRange = {.BlockIndex = BlockIndex};
+										}
+										ChunkBlockIndex++;
+										CurrentOffset += ChunkCompressedLength;
+									}
+									else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset])
+									{
+										if (NextRange.RangeLength == 0)
+										{
+											NextRange.RangeStart		   = CurrentOffset;
+											NextRange.ChunkBlockIndexStart = ChunkBlockIndex;
+										}
+										NextRange.RangeLength += ChunkCompressedLength;
+										NextRange.ChunkBlockIndexCount++;
+										ChunkBlockIndex++;
+										CurrentOffset += ChunkCompressedLength;
+										NeedBlockChunkIndexOffset++;
+									}
+									else
 									{
-										BlockRanges.push_back(NextRange);
-										NextRange = {.BlockIndex = BlockIndex};
+										ZEN_ASSERT(false);
 									}
-									ChunkBlockIndex++;
-									CurrentOffset += ChunkCompressedLength;
 								}
-								else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset])
+								if (NextRange.RangeLength > 0)
 								{
-									AllBlockChunksSize += ChunkCompressedLength;
-									if (NextRange.RangeLength == 0)
+									BlockRanges.push_back(NextRange);
+								}
+
+								ZEN_ASSERT(!BlockRanges.empty());
+
+								std::vector<BlockRangeDescriptor> CollapsedBlockRanges;
+								auto							  It = BlockRanges.begin();
+								CollapsedBlockRanges.push_back(*It++);
+								while (It != BlockRanges.end())
+								{
+									BlockRangeDescriptor& LastRange		= CollapsedBlockRanges.back();
+									uint64_t			  Slack			= It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength);
+									uint64_t			  BothRangeSize = It->RangeLength + LastRange.RangeLength;
+									if (Slack <= Max(BothRangeSize / 8, 64u * 1024u))  // Made up heuristic - we'll see how it pans out
+									{
+										LastRange.ChunkBlockIndexCount =
+											(It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart;
+										LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart;
+									}
+									else
 									{
-										NextRange.RangeStart		   = CurrentOffset;
-										NextRange.ChunkBlockIndexStart = ChunkBlockIndex;
+										CollapsedBlockRanges.push_back(*It);
 									}
-									NextRange.RangeLength += ChunkCompressedLength;
-									NextRange.ChunkBlockIndexCount++;
-									ChunkBlockIndex++;
-									CurrentOffset += ChunkCompressedLength;
-									NeedBlockChunkIndexOffset++;
+									++It;
+								}
+
+								const std::uint64_t WantedSize = std::accumulate(
+									CollapsedBlockRanges.begin(),
+									CollapsedBlockRanges.end(),
+									uint64_t(0),
+									[](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; });
+								ZEN_ASSERT(WantedSize <= TotalBlockSize);
+								if (WantedSize > ((TotalBlockSize * 95) / 100))
+								{
+									ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block",
+														NiceBytes(WantedSize),
+														BlockDescription.BlockHash,
+														NiceBytes(TotalBlockSize));
+									TotalRequestCount++;
+									TotalPartWriteCount++;
+
+									FullBlockWorks.push_back(BlockIndex);
+								}
+								else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1)
+								{
+									ZEN_CONSOLE_VERBOSE(
+										"Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block",
+										NiceBytes(WantedSize),
+										BlockDescription.BlockHash,
+										NiceBytes(TotalBlockSize),
+										CollapsedBlockRanges.size());
+									TotalRequestCount++;
+									TotalPartWriteCount++;
+
+									FullBlockWorks.push_back(BlockIndex);
+								}
+								else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16))
+								{
+									ZEN_CONSOLE_VERBOSE(
+										"Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block",
+										NiceBytes(WantedSize),
+										BlockDescription.BlockHash,
+										NiceBytes(TotalBlockSize),
+										CollapsedBlockRanges.size());
+									TotalRequestCount++;
+									TotalPartWriteCount++;
+
+									FullBlockWorks.push_back(BlockIndex);
+								}
+								else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48))
+								{
+									ZEN_CONSOLE_VERBOSE(
+										"Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block",
+										NiceBytes(WantedSize),
+										BlockDescription.BlockHash,
+										NiceBytes(TotalBlockSize),
+										CollapsedBlockRanges.size());
+									TotalRequestCount++;
+									TotalPartWriteCount++;
+
+									FullBlockWorks.push_back(BlockIndex);
+								}
+								else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64))
+								{
+									ZEN_CONSOLE_VERBOSE(
+										"Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block",
+										NiceBytes(WantedSize),
+										BlockDescription.BlockHash,
+										NiceBytes(TotalBlockSize),
+										CollapsedBlockRanges.size());
+									TotalRequestCount++;
+									TotalPartWriteCount++;
+
+									FullBlockWorks.push_back(BlockIndex);
 								}
 								else
 								{
-									ZEN_ASSERT(false);
+									if (WantedSize > ((TotalBlockSize * 5) / 10))
+									{
+										ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block",
+															(WantedSize * 100) / TotalBlockSize,
+															NiceBytes(WantedSize),
+															BlockDescription.BlockHash,
+															NiceBytes(TotalBlockSize),
+															CollapsedBlockRanges.size());
+									}
+									TotalRequestCount += CollapsedBlockRanges.size();
+									TotalPartWriteCount += CollapsedBlockRanges.size();
+
+									BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end());
 								}
 							}
-							AllBlocksSize += CurrentOffset;
-							if (NextRange.RangeLength > 0)
+							else
 							{
-								BlockRanges.push_back(NextRange);
+								TotalRequestCount++;
+								TotalPartWriteCount++;
+
+								FullBlockWorks.push_back(BlockIndex);
 							}
+						}
+					}
+				}
+				else
+				{
+					ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash);
+				}
+			}
 
-							ZEN_ASSERT(!BlockRanges.empty());
-							std::vector<BlockRangeDescriptor> CollapsedBlockRanges;
-							auto							  It = BlockRanges.begin();
-							CollapsedBlockRanges.push_back(*It++);
-							uint64_t TotalSlack = 0;
-							while (It != BlockRanges.end())
+			struct BlobsExistsResult
+			{
+				tsl::robin_set<IoHash> ExistingBlobs;
+				uint64_t			   ElapsedTimeMs = 0;
+			};
+
+			BlobsExistsResult ExistsResult;
+
+			if (Storage.BuildCacheStorage)
+			{
+				ZEN_TRACE_CPU("BlobCacheExistCheck");
+				Stopwatch Timer;
+
+				tsl::robin_set<IoHash> BlobHashesSet;
+
+				BlobHashesSet.reserve(LooseChunkHashWorks.size() + FullBlockWorks.size());
+				for (LooseChunkHashWorkData& LooseChunkHashWork : LooseChunkHashWorks)
+				{
+					BlobHashesSet.insert(RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]);
+				}
+				for (const BlockRangeDescriptor& BlockRange : BlockRangeWorks)
+				{
+					const uint32_t				 BlockIndex		  = BlockRange.BlockIndex;
+					const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+					BlobHashesSet.insert(BlockDescription.BlockHash);
+				}
+				for (uint32_t BlockIndex : FullBlockWorks)
+				{
+					const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+					BlobHashesSet.insert(BlockDescription.BlockHash);
+				}
+
+				if (!BlobHashesSet.empty())
+				{
+					const std::vector<IoHash>							   BlobHashes(BlobHashesSet.begin(), BlobHashesSet.end());
+					const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult =
+						Storage.BuildCacheStorage->BlobsExists(BuildId, BlobHashes);
+
+					if (CacheExistsResult.size() == BlobHashes.size())
+					{
+						ExistsResult.ExistingBlobs.reserve(CacheExistsResult.size());
+						for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++)
+						{
+							if (CacheExistsResult[BlobIndex].HasBody)
 							{
-								BlockRangeDescriptor& LastRange		= CollapsedBlockRanges.back();
-								uint64_t			  Slack			= It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength);
-								uint64_t			  BothRangeSize = It->RangeLength + LastRange.RangeLength;
-								if (Slack <= Max(BothRangeSize / 8, 64u * 1024u))  // Made up heuristic - we'll see how it pans out
+								ExistsResult.ExistingBlobs.insert(BlobHashes[BlobIndex]);
+							}
+						}
+					}
+					ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs();
+					if (!ExistsResult.ExistingBlobs.empty())
+					{
+						ZEN_CONSOLE("Remote cache  : Found {} out of {} needed blobs in {}",
+									ExistsResult.ExistingBlobs.size(),
+									BlobHashes.size(),
+									NiceTimeSpanMs(ExistsResult.ElapsedTimeMs));
+					}
+				}
+			}
+
+			BufferedWriteFileCache WriteCache;
+
+			for (uint32_t ScavengeOpIndex = 0; ScavengeOpIndex < ScavengeCopyOperations.size(); ScavengeOpIndex++)
+			{
+				if (AbortFlag)
+				{
+					break;
+				}
+				if (!PrimeCacheOnly)
+				{
+					Work.ScheduleWork(
+						WritePool,
+						[&RemoteContent,
+						 &CacheFolderPath,
+						 &ScavengedPaths,
+						 &ScavengeCopyOperations,
+						 &ScavengedContents,
+						 &FilteredWrittenBytesPerSecond,
+						 ScavengeOpIndex,
+						 &WritePartsComplete,
+						 TotalPartWriteCount,
+						 &DiskStats](std::atomic<bool>&) mutable {
+							if (!AbortFlag)
+							{
+								ZEN_TRACE_CPU("UpdateFolder_WriteScavenged");
+
+								FilteredWrittenBytesPerSecond.Start();
+
+								const ScavengeCopyOperation& ScavengeOp		  = ScavengeCopyOperations[ScavengeOpIndex];
+								const ChunkedFolderContent&	 ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex];
+								const std::filesystem::path	 ScavengedPath	  = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex];
+
+								const std::filesystem::path ScavengedFilePath =
+									(ScavengedPaths[ScavengeOp.ScavengedContentIndex] / ScavengedPath).make_preferred();
+								ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize);
+
+								const IoHash& RemoteSequenceRawHash =
+									RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex];
+								const std::filesystem::path TempFilePath =
+									GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+
+								const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedContentIndex];
+								CopyFile(ScavengedFilePath, TempFilePath, RawSize, DiskStats.WriteCount, DiskStats.WriteByteCount);
+
+								const std::filesystem::path CacheFilePath =
+									GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+								RenameFile(TempFilePath, CacheFilePath);
+
+								WritePartsComplete++;
+								if (WritePartsComplete == TotalPartWriteCount)
 								{
-									LastRange.ChunkBlockIndexCount =
-										(It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart;
-									LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart;
-									TotalSlack += Slack;
+									FilteredWrittenBytesPerSecond.Stop();
 								}
-								else
+							}
+						});
+				}
+			}
+
+			for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++)
+			{
+				if (AbortFlag)
+				{
+					break;
+				}
+
+				LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex];
+
+				std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
+					std::move(LooseChunkHashWork.ChunkTargetPtrs);
+				const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex;
+
+				if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]))
+				{
+					DownloadStats.RequestsCompleteCount++;
+					continue;
+				}
+
+				Work.ScheduleWork(
+					WritePool,
+					[&Storage,
+					 &Path,
+					 &ZenFolderPath,
+					 &RemoteContent,
+					 &RemoteLookup,
+					 &CacheFolderPath,
+					 &SequenceIndexChunksLeftToWriteCounters,
+					 &Work,
+					 &WritePool,
+					 &NetworkPool,
+					 PrimeCacheOnly,
+					 &ExistsResult,
+					 &DiskStats,
+					 &DownloadStats,
+					 &WriteChunkStats,
+					 &WritePartsComplete,
+					 RemoteChunkIndex,
+					 ChunkTargetPtrs,
+					 BuildId = Oid(BuildId),
+					 LargeAttachmentSize,
+					 PreferredMultipartChunkSize,
+					 TotalRequestCount,
+					 TotalPartWriteCount,
+					 &WriteCache,
+					 &FilteredDownloadedBytesPerSecond,
+					 &FilteredWrittenBytesPerSecond](std::atomic<bool>&) mutable {
+						if (!AbortFlag)
+						{
+							ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded");
+							std::filesystem::path ExistingCompressedChunkPath;
+							if (!PrimeCacheOnly)
+							{
+								const IoHash&		  ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+								std::filesystem::path CompressedChunkPath =
+									ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString();
+								if (IsFile(CompressedChunkPath))
 								{
-									CollapsedBlockRanges.push_back(*It);
+									IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath);
+									if (ExistingCompressedPart)
+									{
+										IoHash	 RawHash;
+										uint64_t RawSize;
+										if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize))
+										{
+											DownloadStats.RequestsCompleteCount++;
+											if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+											{
+												FilteredDownloadedBytesPerSecond.Stop();
+											}
+											ExistingCompressedChunkPath = std::move(CompressedChunkPath);
+										}
+										else
+										{
+											std::error_code DummyEc;
+											RemoveFile(CompressedChunkPath, DummyEc);
+										}
+									}
 								}
-								++It;
 							}
+							if (!AbortFlag)
 
-							uint64_t TotalFetch = 0;
-							for (const BlockRangeDescriptor& Range : CollapsedBlockRanges)
 							{
-								TotalFetch += Range.RangeLength;
-							}
+								if (!ExistingCompressedChunkPath.empty())
+								{
+									Work.ScheduleWork(
+										WritePool,
+										[&Path,
+										 &ZenFolderPath,
+										 &RemoteContent,
+										 &RemoteLookup,
+										 &CacheFolderPath,
+										 &SequenceIndexChunksLeftToWriteCounters,
+										 &WriteCache,
+										 &Work,
+										 &WritePool,
+										 &DiskStats,
+										 &WriteChunkStats,
+										 &WritePartsComplete,
+										 TotalPartWriteCount,
+										 &FilteredWrittenBytesPerSecond,
+										 RemoteChunkIndex,
+										 ChunkTargetPtrs,
+										 CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic<bool>&) mutable {
+											if (!AbortFlag)
+											{
+												ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded");
 
-							AllBlocksFetch += TotalFetch;
-							AllBlocksSlack += TotalSlack;
-							BlocksNeededCount++;
-							AllBlockRequests += CollapsedBlockRanges.size();
+												FilteredWrittenBytesPerSecond.Start();
 
-							TotalRequestCount += CollapsedBlockRanges.size();
-							TotalPartWriteCount += CollapsedBlockRanges.size();
+												const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
 
-							BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end());
-						}
-						else
-						{
-							BlocksNeededCount++;
-							TotalRequestCount++;
-							TotalPartWriteCount++;
+												IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
+												if (!CompressedPart)
+												{
+													throw std::runtime_error(
+														fmt::format("Could not open dowloaded compressed chunk {} from {}",
+																	ChunkHash,
+																	CompressedChunkPath));
+												}
+
+												std::filesystem::path TargetFolder	 = ZenTempCacheFolderPath(ZenFolderPath);
+												bool				  NeedHashVerify = WriteCompressedChunk(TargetFolder,
+																							RemoteContent,
+																							RemoteLookup,
+																							ChunkHash,
+																							ChunkTargetPtrs,
+																							WriteCache,
+																							std::move(CompressedPart),
+																							DiskStats);
+												WritePartsComplete++;
+
+												if (!AbortFlag)
+												{
+													if (WritePartsComplete == TotalPartWriteCount)
+													{
+														FilteredWrittenBytesPerSecond.Stop();
+													}
+
+													TryRemoveFile(CompressedChunkPath);
 
-							FullBlockWorks.push_back(BlockIndex);
+													std::vector<uint32_t> CompletedSequences =
+														CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
+													WriteCache.Close(CompletedSequences);
+													if (NeedHashVerify)
+													{
+														VerifyAndCompleteChunkSequencesAsync(TargetFolder,
+																							 RemoteContent,
+																							 RemoteLookup,
+																							 CompletedSequences,
+																							 Work,
+																							 WritePool);
+													}
+													else
+													{
+														FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences);
+													}
+												}
+											}
+										});
+								}
+								else
+								{
+									Work.ScheduleWork(
+										NetworkPool,
+										[&Path,
+										 &ZenFolderPath,
+										 &Storage,
+										 BuildId = Oid(BuildId),
+										 PrimeCacheOnly,
+										 &RemoteContent,
+										 &RemoteLookup,
+										 &ExistsResult,
+										 &SequenceIndexChunksLeftToWriteCounters,
+										 &WriteCache,
+										 &Work,
+										 &WritePool,
+										 &NetworkPool,
+										 &DiskStats,
+										 &WriteChunkStats,
+										 &WritePartsComplete,
+										 TotalPartWriteCount,
+										 TotalRequestCount,
+										 &FilteredDownloadedBytesPerSecond,
+										 &FilteredWrittenBytesPerSecond,
+										 LargeAttachmentSize,
+										 PreferredMultipartChunkSize,
+										 RemoteChunkIndex,
+										 ChunkTargetPtrs,
+										 &DownloadStats](std::atomic<bool>&) mutable {
+											if (!AbortFlag)
+											{
+												const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+												FilteredDownloadedBytesPerSecond.Start();
+												IoBuffer   BuildBlob;
+												const bool ExistsInCache =
+													Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash);
+												if (ExistsInCache)
+												{
+													BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash);
+												}
+												if (BuildBlob)
+												{
+													uint64_t BlobSize = BuildBlob.GetSize();
+													DownloadStats.DownloadedChunkCount++;
+													DownloadStats.DownloadedChunkByteCount += BlobSize;
+													DownloadStats.RequestsCompleteCount++;
+													if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+													{
+														FilteredDownloadedBytesPerSecond.Stop();
+													}
+													AsyncWriteDownloadedChunk(ZenFolderPath,
+																			  RemoteContent,
+																			  RemoteLookup,
+																			  RemoteChunkIndex,
+																			  std::move(ChunkTargetPtrs),
+																			  WriteCache,
+																			  Work,
+																			  WritePool,
+																			  std::move(BuildBlob),
+																			  SequenceIndexChunksLeftToWriteCounters,
+																			  WritePartsComplete,
+																			  TotalPartWriteCount,
+																			  FilteredWrittenBytesPerSecond,
+																			  DiskStats);
+												}
+												else
+												{
+													if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize)
+													{
+														ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk");
+														DownloadLargeBlob(*Storage.BuildStorage,
+																		  ZenTempDownloadFolderPath(ZenFolderPath),
+																		  BuildId,
+																		  ChunkHash,
+																		  PreferredMultipartChunkSize,
+																		  Work,
+																		  NetworkPool,
+																		  DownloadStats,
+																		  [&Storage,
+																		   &ZenFolderPath,
+																		   &RemoteContent,
+																		   &RemoteLookup,
+																		   BuildId,
+																		   PrimeCacheOnly,
+																		   &SequenceIndexChunksLeftToWriteCounters,
+																		   &WriteCache,
+																		   &Work,
+																		   &WritePool,
+																		   ChunkHash,
+																		   TotalPartWriteCount,
+																		   TotalRequestCount,
+																		   &WritePartsComplete,
+																		   &FilteredWrittenBytesPerSecond,
+																		   &FilteredDownloadedBytesPerSecond,
+																		   &DownloadStats,
+																		   &DiskStats,
+																		   RemoteChunkIndex,
+																		   ChunkTargetPtrs](IoBuffer&& Payload) mutable {
+																			  DownloadStats.RequestsCompleteCount++;
+																			  if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+																			  {
+																				  FilteredDownloadedBytesPerSecond.Stop();
+																			  }
+																			  if (Payload && Storage.BuildCacheStorage)
+																			  {
+																				  Storage.BuildCacheStorage->PutBuildBlob(
+																					  BuildId,
+																					  ChunkHash,
+																					  ZenContentType::kCompressedBinary,
+																					  CompositeBuffer(SharedBuffer(Payload)));
+																			  }
+																			  if (!PrimeCacheOnly)
+																			  {
+																				  if (!AbortFlag)
+																				  {
+																					  AsyncWriteDownloadedChunk(
+																						  ZenFolderPath,
+																						  RemoteContent,
+																						  RemoteLookup,
+																						  RemoteChunkIndex,
+																						  std::move(ChunkTargetPtrs),
+																						  WriteCache,
+																						  Work,
+																						  WritePool,
+																						  std::move(Payload),
+																						  SequenceIndexChunksLeftToWriteCounters,
+																						  WritePartsComplete,
+																						  TotalPartWriteCount,
+																						  FilteredWrittenBytesPerSecond,
+																						  DiskStats);
+																				  }
+																			  }
+																		  });
+													}
+													else
+													{
+														ZEN_TRACE_CPU("UpdateFolder_GetChunk");
+														BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash);
+														if (BuildBlob && Storage.BuildCacheStorage)
+														{
+															Storage.BuildCacheStorage->PutBuildBlob(
+																BuildId,
+																ChunkHash,
+																BuildBlob.GetContentType(),
+																CompositeBuffer(SharedBuffer(BuildBlob)));
+														}
+														if (!BuildBlob)
+														{
+															throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash));
+														}
+														if (!PrimeCacheOnly)
+														{
+															if (!AbortFlag)
+															{
+																uint64_t BlobSize = BuildBlob.GetSize();
+																DownloadStats.DownloadedChunkCount++;
+																DownloadStats.DownloadedChunkByteCount += BlobSize;
+																DownloadStats.RequestsCompleteCount++;
+																if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+																{
+																	FilteredDownloadedBytesPerSecond.Stop();
+																}
+																AsyncWriteDownloadedChunk(ZenFolderPath,
+																						  RemoteContent,
+																						  RemoteLookup,
+																						  RemoteChunkIndex,
+																						  std::move(ChunkTargetPtrs),
+																						  WriteCache,
+																						  Work,
+																						  WritePool,
+																						  std::move(BuildBlob),
+																						  SequenceIndexChunksLeftToWriteCounters,
+																						  WritePartsComplete,
+																						  TotalPartWriteCount,
+																						  FilteredWrittenBytesPerSecond,
+																						  DiskStats);
+															}
+														}
+													}
+												}
+											}
+										});
+								}
+							}
 						}
-					}
-				}
-				else
-				{
-					ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash);
-				}
+					});
 			}
 
 			for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++)
 			{
+				ZEN_ASSERT(!PrimeCacheOnly);
 				if (AbortFlag)
 				{
 					break;
 				}
 
 				Work.ScheduleWork(
-					WritePool,	// GetSyncWorkerPool(),//
-					[&, CopyDataIndex](std::atomic<bool>&) {
+					WritePool,
+					[&Path,
+					 &LocalContent,
+					 &RemoteContent,
+					 &RemoteLookup,
+					 &CacheFolderPath,
+					 &LocalLookup,
+					 &SequenceIndexChunksLeftToWriteCounters,
+					 &WriteCache,
+					 &Work,
+					 &WritePool,
+					 &FilteredWrittenBytesPerSecond,
+					 &CacheCopyDatas,
+					 &ScavengedContents,
+					 &ScavengedLookups,
+					 &ScavengedPaths,
+					 &WritePartsComplete,
+					 TotalPartWriteCount,
+					 &DiskStats,
+					 CopyDataIndex](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("UpdateFolder_CopyLocal");
 
 							FilteredWrittenBytesPerSecond.Start();
-							const CacheCopyData& CopyData		= CacheCopyDatas[CopyDataIndex];
-							const uint32_t		 LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex];
-							const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+							const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex];
+
+							std::filesystem::path SourceFilePath;
+
+							if (CopyData.ScavengeSourceIndex == (uint32_t)-1)
+							{
+								const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
+								SourceFilePath				  = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+							}
+							else
+							{
+								const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex];
+								const ChunkedContentLookup& ScavengedLookup	 = ScavengedLookups[CopyData.ScavengeSourceIndex];
+								const std::filesystem::path ScavengedPath	 = ScavengedPaths[CopyData.ScavengeSourceIndex];
+								const uint32_t				ScavengedPathIndex =
+									ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
+								SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred();
+							}
+							ZEN_ASSERT_SLOW(IsFile(SourceFilePath));
 							ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty());
 
 							uint64_t CacheLocalFileBytesRead = 0;
@@ -4888,7 +7238,7 @@ namespace {
 							{
 								const ChunkedContentLookup::ChunkSequenceLocation* Target		   = nullptr;
 								uint64_t										   CacheFileOffset = (uint64_t)-1;
-								uint64_t										   ChunkSize	   = (uint64_t)-1;
+								uint32_t										   ChunkIndex	   = (uint32_t)-1;
 							};
 
 							std::vector<WriteOp> WriteOps;
@@ -4905,7 +7255,7 @@ namespace {
 									{
 										WriteOps.push_back(WriteOp{.Target			= Target,
 																   .CacheFileOffset = ChunkTarget.CacheFileOffset,
-																   .ChunkSize		= ChunkTarget.ChunkRawSize});
+																   .ChunkIndex		= ChunkTarget.RemoteChunkIndex});
 									}
 									TargetStart += ChunkTarget.TargetChunkLocationCount;
 								}
@@ -4931,281 +7281,129 @@ namespace {
 							{
 								ZEN_TRACE_CPU("Write");
 
-								BufferedOpenFile SourceFile(LocalFilePath);
-								WriteFileCache	 OpenFileCache;
-								for (const WriteOp& Op : WriteOps)
+								tsl::robin_set<uint32_t> ChunkIndexesWritten;
+
+								BufferedOpenFile			  SourceFile(SourceFilePath, DiskStats);
+								BufferedWriteFileCache::Local LocalWriter(WriteCache);
+
+								for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();)
 								{
 									if (AbortFlag)
 									{
 										break;
 									}
+									const WriteOp& Op = WriteOps[WriteOpIndex];
+
 									const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
 									ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <=
 											   RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]);
 									ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0);
-									const uint32_t	RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
-									const uint64_t	ChunkSize		= Op.ChunkSize;
-									CompositeBuffer ChunkSource		= SourceFile.GetRange(Op.CacheFileOffset, ChunkSize);
-
-									ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]);
-
-									OpenFileCache.WriteToFile<CompositeBuffer>(
-										RemoteSequenceIndex,
-										[&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) {
-											return GetTempChunkedSequenceFileName(
-												CacheFolderPath,
-												RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
-										},
-										ChunkSource,
-										Op.Target->Offset,
-										RemoteContent.RawSizes[RemotePathIndex]);
-									WriteToDiskBytes += ChunkSize;
-									CacheLocalFileBytesRead += ChunkSize;  // TODO: This should be the sum of unique chunk sizes?
-								}
-							}
-							if (!AbortFlag)
-							{
-								// Write tracking, updating this must be done without any files open (WriteFileCache)
-								for (const WriteOp& Op : WriteOps)
-								{
-									const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
-									if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1)
+									const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
+									const uint64_t ChunkSize	   = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex];
+
+									uint64_t ReadLength	 = ChunkSize;
+									size_t	 WriteCount	 = 1;
+									uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize;
+									uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize;
+									while ((WriteOpIndex + WriteCount) < WriteOps.size())
 									{
-										const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+										const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount];
+										if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex)
 										{
-											ZEN_TRACE_CPU("VerifyHash");
-											const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(
-												GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)));
-											if (VerifyChunkHash != SequenceRawHash)
-											{
-												throw std::runtime_error(
-													fmt::format("Written chunk sequence {} hash does not match expected hash {}",
-																VerifyChunkHash,
-																SequenceRawHash));
-											}
+											break;
 										}
-
-										ZEN_TRACE_CPU("rename");
-										ZEN_ASSERT_SLOW(
-											!std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)));
-										std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash),
-																GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash));
-									}
-								}
-
-								ChunkCountWritten += gsl::narrow<uint32_t>(CopyData.ChunkTargets.size());
-								ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]);
-							}
-							WritePartsComplete++;
-							if (WritePartsComplete == TotalPartWriteCount)
-							{
-								FilteredWrittenBytesPerSecond.Stop();
-							}
-						}
-					},
-					Work.DefaultErrorFunction());
-			}
-
-			for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++)
-			{
-				if (AbortFlag)
-				{
-					break;
-				}
-
-				LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex];
-
-				std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
-					std::move(LooseChunkHashWork.ChunkTargetPtrs);
-				const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex;
-
-				Work.ScheduleWork(
-					WritePool,	// NetworkPool,  // GetSyncWorkerPool(),//
-					[&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic<bool>&) mutable {
-						if (!AbortFlag)
-						{
-							ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded");
-							std::filesystem::path ExistingCompressedChunkPath;
-							{
-								const IoHash&		  ChunkHash			  = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
-								std::filesystem::path CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString();
-								if (std::filesystem::exists(CompressedChunkPath))
-								{
-									IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath);
-									if (ExistingCompressedPart)
-									{
-										IoHash	 RawHash;
-										uint64_t RawSize;
-										if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize))
+										if (NextOp.Target->Offset != OpTargetEnd)
 										{
-											LooseChunksBytes += ExistingCompressedPart.GetSize();
-											RequestsComplete++;
-											if (RequestsComplete == TotalRequestCount)
-											{
-												FilteredDownloadedBytesPerSecond.Stop();
-											}
-											ExistingCompressedChunkPath = std::move(CompressedChunkPath);
+											break;
 										}
-										else
+										if (NextOp.CacheFileOffset != OpSourceEnd)
 										{
-											std::error_code DummyEc;
-											std::filesystem::remove(CompressedChunkPath, DummyEc);
+											break;
 										}
-									}
-								}
-							}
-							if (!ExistingCompressedChunkPath.empty())
-							{
-								Work.ScheduleWork(
-									WritePool,	// WritePool, GetSyncWorkerPool()
-									[&Path,
-									 &RemoteContent,
-									 &RemoteLookup,
-									 &CacheFolderPath,
-									 &SequenceIndexChunksLeftToWriteCounters,
-									 &WriteToDiskBytes,
-									 &ChunkCountWritten,
-									 &WritePartsComplete,
-									 &TotalPartWriteCount,
-									 &FilteredWrittenBytesPerSecond,
-									 RemoteChunkIndex,
-									 ChunkTargetPtrs,
-									 CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic<bool>&) mutable {
-										if (!AbortFlag)
+										const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex];
+										if (ReadLength + NextChunkLength > 512u * 1024u)
 										{
-											ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded");
-
-											FilteredWrittenBytesPerSecond.Start();
-
-											const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+											break;
+										}
+										ReadLength += NextChunkLength;
+										OpSourceEnd += NextChunkLength;
+										OpTargetEnd += NextChunkLength;
+										WriteCount++;
+									}
 
-											IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
-											if (!CompressedPart)
-											{
-												throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}",
-																					 ChunkHash,
-																					 CompressedChunkPath));
-											}
+									CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength);
 
-											std::filesystem::path TargetFolder	 = Path / ZenTempCacheFolderName;
-											bool				  NeedHashVerify = WriteCompressedChunk(TargetFolder,
-																						RemoteContent,
-																						RemoteLookup,
-																						ChunkHash,
-																						ChunkTargetPtrs,
-																						std::move(CompressedPart),
-																						WriteToDiskBytes);
+									const uint64_t FileOffset = Op.Target->Offset;
 
-											if (!AbortFlag)
-											{
-												ChunkCountWritten++;
-												WritePartsComplete++;
-												if (WritePartsComplete == TotalPartWriteCount)
-												{
-													FilteredWrittenBytesPerSecond.Stop();
-												}
+									WriteSequenceChunk(CacheFolderPath,
+													   RemoteContent,
+													   LocalWriter,
+													   ChunkSource,
+													   RemoteSequenceIndex,
+													   FileOffset,
+													   RemotePathIndex,
+													   DiskStats);
 
-												std::filesystem::remove(CompressedChunkPath);
+									CacheLocalFileBytesRead += ReadLength;	// TODO: This should be the sum of unique chunk sizes?
 
-												CompleteChunkTargets(TargetFolder,
-																	 RemoteContent,
-																	 ChunkHash,
-																	 ChunkTargetPtrs,
-																	 SequenceIndexChunksLeftToWriteCounters,
-																	 NeedHashVerify);
-											}
-										}
-									},
-									Work.DefaultErrorFunction());
+									WriteOpIndex += WriteCount;
+								}
 							}
-							else
+							if (!AbortFlag)
 							{
-								FilteredDownloadedBytesPerSecond.Start();
-								const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
-								if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize)
-								{
-									ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk");
-									DownloadLargeBlob(Storage,
-													  Path / ZenTempDownloadFolderName,
-													  BuildId,
-													  ChunkHash,
-													  PreferredMultipartChunkSize,
-													  Work,
-													  NetworkPool,
-													  BytesDownloaded,
-													  MultipartAttachmentCount,
-													  [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable {
-														  RequestsComplete++;
-														  if (RequestsComplete == TotalRequestCount)
-														  {
-															  FilteredDownloadedBytesPerSecond.Stop();
-														  }
-														  AsyncWriteDownloadedChunk(Path,
-																					RemoteContent,
-																					RemoteLookup,
-																					RemoteChunkIndex,
-																					std::move(ChunkTargetPtrs),
-																					Work,
-																					WritePool,
-																					std::move(Payload),
-																					SequenceIndexChunksLeftToWriteCounters,
-																					WriteToDiskBytes,
-																					ChunkCountWritten,
-																					WritePartsComplete,
-																					TotalPartWriteCount,
-																					LooseChunksBytes,
-																					FilteredWrittenBytesPerSecond);
-													  });
-								}
-								else
+								// Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local)
+								std::vector<uint32_t> CompletedChunkSequences;
+								for (const WriteOp& Op : WriteOps)
 								{
-									ZEN_TRACE_CPU("UpdateFolder_GetChunk");
-
-									IoBuffer BuildBlob = Storage.GetBuildBlob(BuildId, ChunkHash);
-									if (!BuildBlob)
-									{
-										throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash));
-									}
-									uint64_t BlobSize = BuildBlob.GetSize();
-									BytesDownloaded += BlobSize;
-
-									RequestsComplete++;
-									if (RequestsComplete == TotalRequestCount)
+									const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
+									if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
 									{
-										FilteredDownloadedBytesPerSecond.Stop();
+										CompletedChunkSequences.push_back(RemoteSequenceIndex);
 									}
-									AsyncWriteDownloadedChunk(Path,
-															  RemoteContent,
-															  RemoteLookup,
-															  RemoteChunkIndex,
-															  std::move(ChunkTargetPtrs),
-															  Work,
-															  WritePool,
-															  std::move(BuildBlob),
-															  SequenceIndexChunksLeftToWriteCounters,
-															  WriteToDiskBytes,
-															  ChunkCountWritten,
-															  WritePartsComplete,
-															  TotalPartWriteCount,
-															  LooseChunksBytes,
-															  FilteredWrittenBytesPerSecond);
 								}
+								WriteCache.Close(CompletedChunkSequences);
+								VerifyAndCompleteChunkSequencesAsync(CacheFolderPath,
+																	 RemoteContent,
+																	 RemoteLookup,
+																	 CompletedChunkSequences,
+																	 Work,
+																	 WritePool);
+								ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath);
+							}
+							WritePartsComplete++;
+							if (WritePartsComplete == TotalPartWriteCount)
+							{
+								FilteredWrittenBytesPerSecond.Stop();
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			}
 
 			for (uint32_t BlockIndex : CachedChunkBlockIndexes)
 			{
+				ZEN_ASSERT(!PrimeCacheOnly);
 				if (AbortFlag)
 				{
 					break;
 				}
 
 				Work.ScheduleWork(
-					WritePool,	// GetSyncWorkerPool(),  // WritePool,
-					[&, BlockIndex](std::atomic<bool>&) mutable {
+					WritePool,
+					[&ZenFolderPath,
+					 &CacheFolderPath,
+					 &RemoteContent,
+					 &RemoteLookup,
+					 &RemoteChunkIndexNeedsCopyFromSourceFlags,
+					 &SequenceIndexChunksLeftToWriteCounters,
+					 &WriteCache,
+					 &Work,
+					 &WritePool,
+					 &BlockDescriptions,
+					 &FilteredWrittenBytesPerSecond,
+					 &DiskStats,
+					 &WritePartsComplete,
+					 TotalPartWriteCount,
+					 BlockIndex](std::atomic<bool>&) mutable {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock");
@@ -5213,41 +7411,50 @@ namespace {
 							const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
 							FilteredWrittenBytesPerSecond.Start();
 
-							std::filesystem::path BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString();
-							IoBuffer			  BlockBuffer	 = IoBufferBuilder::MakeFromFile(BlockChunkPath);
+							std::filesystem::path BlockChunkPath =
+								ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+							IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
 							if (!BlockBuffer)
 							{
 								throw std::runtime_error(
 									fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath));
 							}
 
-							if (!WriteBlockToDisk(CacheFolderPath,
-												  RemoteContent,
-												  BlockDescription,
-												  SequenceIndexChunksLeftToWriteCounters,
-												  CompositeBuffer(std::move(BlockBuffer)),
-												  RemoteLookup,
-												  RemoteChunkIndexNeedsCopyFromSourceFlags,
-												  ChunkCountWritten,
-												  WriteToDiskBytes))
-							{
-								std::error_code DummyEc;
-								std::filesystem::remove(BlockChunkPath, DummyEc);
-								throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash));
-							}
-							WritePartsComplete++;
-							std::filesystem::remove(BlockChunkPath);
-							if (WritePartsComplete == TotalPartWriteCount)
+							if (!AbortFlag)
 							{
-								FilteredWrittenBytesPerSecond.Stop();
+								if (!WriteBlockToDisk(CacheFolderPath,
+													  RemoteContent,
+													  BlockDescription,
+													  SequenceIndexChunksLeftToWriteCounters,
+													  Work,
+													  WritePool,
+													  CompositeBuffer(std::move(BlockBuffer)),
+													  RemoteLookup,
+													  RemoteChunkIndexNeedsCopyFromSourceFlags,
+													  WriteCache,
+													  DiskStats))
+								{
+									std::error_code DummyEc;
+									RemoveFile(BlockChunkPath, DummyEc);
+									throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash));
+								}
+
+								TryRemoveFile(BlockChunkPath);
+
+								WritePartsComplete++;
+
+								if (WritePartsComplete == TotalPartWriteCount)
+								{
+									FilteredWrittenBytesPerSecond.Stop();
+								}
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			}
 
 			for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++)
 			{
+				ZEN_ASSERT(!PrimeCacheOnly);
 				if (AbortFlag)
 				{
 					break;
@@ -5256,8 +7463,29 @@ namespace {
 				ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1);
 				const uint32_t BlockIndex = BlockRange.BlockIndex;
 				Work.ScheduleWork(
-					NetworkPool,  // NetworkPool,  // GetSyncWorkerPool()
-					[&, BlockIndex, BlockRange](std::atomic<bool>&) {
+					NetworkPool,
+					[&Storage,
+					 &ZenFolderPath,
+					 BuildId,
+					 &RemoteLookup,
+					 &BlockDescriptions,
+					 &RemoteChunkIndexNeedsCopyFromSourceFlags,
+					 &CacheFolderPath,
+					 &RemoteContent,
+					 &SequenceIndexChunksLeftToWriteCounters,
+					 &ExistsResult,
+					 &WriteCache,
+					 &FilteredDownloadedBytesPerSecond,
+					 TotalRequestCount,
+					 &WritePartsComplete,
+					 TotalPartWriteCount,
+					 &FilteredWrittenBytesPerSecond,
+					 &DiskStats,
+					 &DownloadStats,
+					 &Work,
+					 &WritePool,
+					 BlockIndex,
+					 BlockRange](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock");
@@ -5265,134 +7493,167 @@ namespace {
 							const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
 
 							FilteredDownloadedBytesPerSecond.Start();
-							IoBuffer BlockBuffer =
-								Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength);
+							IoBuffer BlockBuffer;
+							if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash))
+							{
+								BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId,
+																					  BlockDescription.BlockHash,
+																					  BlockRange.RangeStart,
+																					  BlockRange.RangeLength);
+							}
 							if (!BlockBuffer)
 							{
-								throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash));
+								BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId,
+																				 BlockDescription.BlockHash,
+																				 BlockRange.RangeStart,
+																				 BlockRange.RangeLength);
 							}
-							uint64_t BlockSize = BlockBuffer.GetSize();
-							BytesDownloaded += BlockSize;
-							BlockBytes += BlockSize;
-							DownloadedBlocks++;
-							RequestsComplete++;
-							if (RequestsComplete == TotalRequestCount)
+							if (!BlockBuffer)
 							{
-								FilteredDownloadedBytesPerSecond.Stop();
+								throw std::runtime_error(fmt::format("Block {} is missing when fetching range {} -> {}",
+																	 BlockDescription.BlockHash,
+																	 BlockRange.RangeStart,
+																	 BlockRange.RangeStart + BlockRange.RangeLength));
 							}
-
-							std::filesystem::path BlockChunkPath;
-
-							// Check if the dowloaded block is file based and we can move it directly without rewriting it
+							if (!AbortFlag)
 							{
-								IoBufferFileReference FileRef;
-								if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) &&
-									(FileRef.FileChunkSize == BlockSize))
+								uint64_t BlockSize = BlockBuffer.GetSize();
+								DownloadStats.DownloadedBlockCount++;
+								DownloadStats.DownloadedBlockByteCount += BlockSize;
+								DownloadStats.RequestsCompleteCount++;
+								if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
 								{
-									ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock");
+									FilteredDownloadedBytesPerSecond.Stop();
+								}
 
-									std::error_code		  Ec;
-									std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
-									if (!Ec)
+								std::filesystem::path BlockChunkPath;
+
+								// Check if the dowloaded block is file based and we can move it directly without rewriting it
+								{
+									IoBufferFileReference FileRef;
+									if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) &&
+										(FileRef.FileChunkSize == BlockSize))
 									{
-										BlockBuffer.SetDeleteOnClose(false);
-										BlockBuffer	   = {};
-										BlockChunkPath = Path / ZenTempBlockFolderName /
-														 fmt::format("{}_{:x}_{:x}",
-																	 BlockDescription.BlockHash,
-																	 BlockRange.RangeStart,
-																	 BlockRange.RangeLength);
-										std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec);
-										if (Ec)
+										ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock");
+
+										std::error_code		  Ec;
+										std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
+										if (!Ec)
 										{
-											BlockChunkPath = std::filesystem::path{};
+											BlockBuffer.SetDeleteOnClose(false);
+											BlockBuffer	   = {};
+											BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
+																												 BlockDescription.BlockHash,
+																												 BlockRange.RangeStart,
+																												 BlockRange.RangeLength);
+											RenameFile(TempBlobPath, BlockChunkPath, Ec);
+											if (Ec)
+											{
+												BlockChunkPath = std::filesystem::path{};
 
-											// Re-open the temp file again
-											BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
-											BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true);
-											BlockBuffer.SetDeleteOnClose(true);
+												// Re-open the temp file again
+												BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
+												BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true);
+												BlockBuffer.SetDeleteOnClose(true);
+											}
 										}
 									}
 								}
-							}
 
-							if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u))
-							{
-								ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
-								// Could not be moved and rather large, lets store it on disk
-								BlockChunkPath =
-									Path / ZenTempBlockFolderName /
-									fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength);
-								TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
-								BlockBuffer = {};
-							}
+								if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u))
+								{
+									ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
+									// Could not be moved and rather large, lets store it on disk
+									BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
+																										 BlockDescription.BlockHash,
+																										 BlockRange.RangeStart,
+																										 BlockRange.RangeLength);
+									TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
+									BlockBuffer = {};
+								}
 
-							if (!AbortFlag)
-							{
-								Work.ScheduleWork(
-									WritePool,	// WritePool,	// GetSyncWorkerPool(),
-									[&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)](
-										std::atomic<bool>&) mutable {
-										if (!AbortFlag)
-										{
-											ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock");
+								if (!AbortFlag)
+								{
+									Work.ScheduleWork(
+										WritePool,
+										[&CacheFolderPath,
+										 &RemoteContent,
+										 &RemoteLookup,
+										 &BlockDescriptions,
+										 &RemoteChunkIndexNeedsCopyFromSourceFlags,
+										 &SequenceIndexChunksLeftToWriteCounters,
+										 &WritePartsComplete,
+										 &WriteCache,
+										 &Work,
+										 TotalPartWriteCount,
+										 &WritePool,
+										 &DiskStats,
+										 &FilteredWrittenBytesPerSecond,
+										 BlockIndex,
+										 BlockRange,
+										 BlockChunkPath,
+										 BlockPartialBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable {
+											if (!AbortFlag)
+											{
+												ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock");
 
-											const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+												const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
 
-											if (BlockChunkPath.empty())
-											{
-												ZEN_ASSERT(BlockPartialBuffer);
-											}
-											else
-											{
-												ZEN_ASSERT(!BlockPartialBuffer);
-												BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
-												if (!BlockPartialBuffer)
+												if (BlockChunkPath.empty())
 												{
-													throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}",
-																						 BlockDescription.BlockHash,
-																						 BlockChunkPath));
+													ZEN_ASSERT(BlockPartialBuffer);
+												}
+												else
+												{
+													ZEN_ASSERT(!BlockPartialBuffer);
+													BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
+													if (!BlockPartialBuffer)
+													{
+														throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}",
+																							 BlockDescription.BlockHash,
+																							 BlockChunkPath));
+													}
 												}
-											}
 
-											FilteredWrittenBytesPerSecond.Start();
-
-											if (!WritePartialBlockToDisk(
-													CacheFolderPath,
-													RemoteContent,
-													BlockDescription,
-													SequenceIndexChunksLeftToWriteCounters,
-													CompositeBuffer(std::move(BlockPartialBuffer)),
-													BlockRange.ChunkBlockIndexStart,
-													BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1,
-													RemoteLookup,
-													RemoteChunkIndexNeedsCopyFromSourceFlags,
-													ChunkCountWritten,
-													WriteToDiskBytes))
-											{
-												std::error_code DummyEc;
-												std::filesystem::remove(BlockChunkPath, DummyEc);
-												throw std::runtime_error(
-													fmt::format("Partial block {} is malformed", BlockDescription.BlockHash));
-											}
-											WritePartsComplete++;
+												FilteredWrittenBytesPerSecond.Start();
+
+												if (!WritePartialBlockToDisk(
+														CacheFolderPath,
+														RemoteContent,
+														BlockDescription,
+														SequenceIndexChunksLeftToWriteCounters,
+														Work,
+														WritePool,
+														CompositeBuffer(std::move(BlockPartialBuffer)),
+														BlockRange.ChunkBlockIndexStart,
+														BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1,
+														RemoteLookup,
+														RemoteChunkIndexNeedsCopyFromSourceFlags,
+														WriteCache,
+														DiskStats))
+												{
+													std::error_code DummyEc;
+													RemoveFile(BlockChunkPath, DummyEc);
+													throw std::runtime_error(
+														fmt::format("Partial block {} is malformed", BlockDescription.BlockHash));
+												}
 
-											if (!BlockChunkPath.empty())
-											{
-												std::filesystem::remove(BlockChunkPath);
-											}
+												if (!BlockChunkPath.empty())
+												{
+													TryRemoveFile(BlockChunkPath);
+												}
 
-											if (WritePartsComplete == TotalPartWriteCount)
-											{
-												FilteredWrittenBytesPerSecond.Stop();
+												WritePartsComplete++;
+												if (WritePartsComplete == TotalPartWriteCount)
+												{
+													FilteredWrittenBytesPerSecond.Stop();
+												}
 											}
-										}
-									},
-									Work.DefaultErrorFunction());
+										});
+								}
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			}
 
 			for (uint32_t BlockIndex : FullBlockWorks)
@@ -5401,9 +7662,38 @@ namespace {
 				{
 					break;
 				}
+
+				if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash))
+				{
+					DownloadStats.RequestsCompleteCount++;
+					continue;
+				}
+
 				Work.ScheduleWork(
-					NetworkPool,  // GetSyncWorkerPool(),  // NetworkPool,
-					[&, BlockIndex](std::atomic<bool>&) {
+					NetworkPool,
+					[&Storage,
+					 &ZenFolderPath,
+					 BuildId,
+					 PrimeCacheOnly,
+					 &BlockDescriptions,
+					 &WritePartsComplete,
+					 TotalPartWriteCount,
+					 &FilteredWrittenBytesPerSecond,
+					 &ExistsResult,
+					 &Work,
+					 &WritePool,
+					 &RemoteContent,
+					 &RemoteLookup,
+					 &WriteCache,
+					 &CacheFolderPath,
+					 &RemoteChunkIndexNeedsCopyFromSourceFlags,
+					 &SequenceIndexChunksLeftToWriteCounters,
+					 &FilteredDownloadedBytesPerSecond,
+					 &WriteChunkStats,
+					 &DiskStats,
+					 &DownloadStats,
+					 TotalRequestCount,
+					 BlockIndex](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("UpdateFolder_GetFullBlock");
@@ -5411,295 +7701,496 @@ namespace {
 							const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
 
 							FilteredDownloadedBytesPerSecond.Start();
-							IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash);
-							if (!BlockBuffer)
-							{
-								throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash));
-							}
-							uint64_t BlockSize = BlockBuffer.GetSize();
-							BytesDownloaded += BlockSize;
-							BlockBytes += BlockSize;
-							DownloadedBlocks++;
-							RequestsComplete++;
-							if (RequestsComplete == TotalRequestCount)
+
+							IoBuffer   BlockBuffer;
+							const bool ExistsInCache =
+								Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash);
+							if (ExistsInCache)
 							{
-								FilteredDownloadedBytesPerSecond.Stop();
+								BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash);
 							}
-
-							std::filesystem::path BlockChunkPath;
-
-							// Check if the dowloaded block is file based and we can move it directly without rewriting it
+							if (!BlockBuffer)
 							{
-								IoBufferFileReference FileRef;
-								if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) &&
-									(FileRef.FileChunkSize == BlockSize))
+								BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash);
+								if (BlockBuffer && Storage.BuildCacheStorage)
 								{
-									ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock");
-									std::error_code		  Ec;
-									std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
-									if (!Ec)
-									{
-										BlockBuffer.SetDeleteOnClose(false);
-										BlockBuffer	   = {};
-										BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString();
-										std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec);
-										if (Ec)
-										{
-											BlockChunkPath = std::filesystem::path{};
-
-											// Re-open the temp file again
-											BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
-											BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true);
-											BlockBuffer.SetDeleteOnClose(true);
-										}
-									}
+									Storage.BuildCacheStorage->PutBuildBlob(BuildId,
+																			BlockDescription.BlockHash,
+																			BlockBuffer.GetContentType(),
+																			CompositeBuffer(SharedBuffer(BlockBuffer)));
 								}
 							}
-
-							if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u))
+							if (!BlockBuffer)
 							{
-								ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
-								// Could not be moved and rather large, lets store it on disk
-								BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString();
-								TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
-								BlockBuffer = {};
+								throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash));
 							}
-
 							if (!AbortFlag)
 							{
-								Work.ScheduleWork(
-									WritePool,	// WritePool, GetSyncWorkerPool()
-									[&RemoteContent,
-									 &RemoteLookup,
-									 CacheFolderPath,
-									 &RemoteChunkIndexNeedsCopyFromSourceFlags,
-									 &SequenceIndexChunksLeftToWriteCounters,
-									 BlockIndex,
-									 &BlockDescriptions,
-									 &ChunkCountWritten,
-									 &WriteToDiskBytes,
-									 &WritePartsComplete,
-									 &TotalPartWriteCount,
-									 &FilteredWrittenBytesPerSecond,
-									 BlockChunkPath,
-									 BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable {
-										if (!AbortFlag)
-										{
-											ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock");
+								uint64_t BlockSize = BlockBuffer.GetSize();
+								DownloadStats.DownloadedBlockCount++;
+								DownloadStats.DownloadedBlockByteCount += BlockSize;
+								DownloadStats.RequestsCompleteCount++;
+								if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+								{
+									FilteredDownloadedBytesPerSecond.Stop();
+								}
 
-											const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+								if (!PrimeCacheOnly)
+								{
+									std::filesystem::path BlockChunkPath;
 
-											if (BlockChunkPath.empty())
-											{
-												ZEN_ASSERT(BlockBuffer);
-											}
-											else
+									// Check if the dowloaded block is file based and we can move it directly without rewriting it
+									{
+										IoBufferFileReference FileRef;
+										if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) &&
+											(FileRef.FileChunkSize == BlockSize))
+										{
+											ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock");
+											std::error_code		  Ec;
+											std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
+											if (!Ec)
 											{
-												ZEN_ASSERT(!BlockBuffer);
-												BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
-												if (!BlockBuffer)
+												BlockBuffer.SetDeleteOnClose(false);
+												BlockBuffer = {};
+												BlockChunkPath =
+													ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+												RenameFile(TempBlobPath, BlockChunkPath, Ec);
+												if (Ec)
 												{
-													throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}",
-																						 BlockDescription.BlockHash,
-																						 BlockChunkPath));
-												}
-											}
+													BlockChunkPath = std::filesystem::path{};
 
-											FilteredWrittenBytesPerSecond.Start();
-											if (!WriteBlockToDisk(CacheFolderPath,
-																  RemoteContent,
-																  BlockDescription,
-																  SequenceIndexChunksLeftToWriteCounters,
-																  CompositeBuffer(std::move(BlockBuffer)),
-																  RemoteLookup,
-																  RemoteChunkIndexNeedsCopyFromSourceFlags,
-																  ChunkCountWritten,
-																  WriteToDiskBytes))
-											{
-												std::error_code DummyEc;
-												std::filesystem::remove(BlockChunkPath, DummyEc);
-												throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash));
+													// Re-open the temp file again
+													BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
+													BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true);
+													BlockBuffer.SetDeleteOnClose(true);
+												}
 											}
-											WritePartsComplete++;
+										}
+									}
 
-											if (!BlockChunkPath.empty())
-											{
-												std::filesystem::remove(BlockChunkPath);
-											}
+									if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u))
+									{
+										ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
+										// Could not be moved and rather large, lets store it on disk
+										BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+										TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
+										BlockBuffer = {};
+									}
 
-											if (WritePartsComplete == TotalPartWriteCount)
-											{
-												FilteredWrittenBytesPerSecond.Stop();
-											}
-										}
-									},
-									Work.DefaultErrorFunction());
+									if (!AbortFlag)
+									{
+										Work.ScheduleWork(WritePool,
+														  [&Work,
+														   &WritePool,
+														   &RemoteContent,
+														   &RemoteLookup,
+														   CacheFolderPath,
+														   &RemoteChunkIndexNeedsCopyFromSourceFlags,
+														   &SequenceIndexChunksLeftToWriteCounters,
+														   BlockIndex,
+														   &BlockDescriptions,
+														   &WriteCache,
+														   &WriteChunkStats,
+														   &DiskStats,
+														   &WritePartsComplete,
+														   TotalPartWriteCount,
+														   &FilteredWrittenBytesPerSecond,
+														   BlockChunkPath,
+														   BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable {
+															  if (!AbortFlag)
+															  {
+																  ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock");
+
+																  const ChunkBlockDescription& BlockDescription =
+																	  BlockDescriptions[BlockIndex];
+
+																  if (BlockChunkPath.empty())
+																  {
+																	  ZEN_ASSERT(BlockBuffer);
+																  }
+																  else
+																  {
+																	  ZEN_ASSERT(!BlockBuffer);
+																	  BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
+																	  if (!BlockBuffer)
+																	  {
+																		  throw std::runtime_error(
+																			  fmt::format("Could not open dowloaded block {} from {}",
+																						  BlockDescription.BlockHash,
+																						  BlockChunkPath));
+																	  }
+																  }
+
+																  FilteredWrittenBytesPerSecond.Start();
+																  if (!WriteBlockToDisk(CacheFolderPath,
+																						RemoteContent,
+																						BlockDescription,
+																						SequenceIndexChunksLeftToWriteCounters,
+																						Work,
+																						WritePool,
+																						CompositeBuffer(std::move(BlockBuffer)),
+																						RemoteLookup,
+																						RemoteChunkIndexNeedsCopyFromSourceFlags,
+																						WriteCache,
+																						DiskStats))
+																  {
+																	  std::error_code DummyEc;
+																	  RemoveFile(BlockChunkPath, DummyEc);
+																	  throw std::runtime_error(
+																		  fmt::format("Block {} is malformed", BlockDescription.BlockHash));
+																  }
+
+																  if (!BlockChunkPath.empty())
+																  {
+																	  TryRemoveFile(BlockChunkPath);
+																  }
+
+																  WritePartsComplete++;
+
+																  if (WritePartsComplete == TotalPartWriteCount)
+																  {
+																	  FilteredWrittenBytesPerSecond.Stop();
+																  }
+															  }
+														  });
+									}
+								}
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 			}
 
-			ZEN_DEBUG("Fetching {} with {} slack (ideal {}) out of {} using {} requests for {} blocks",
-					  NiceBytes(AllBlocksFetch),
-					  NiceBytes(AllBlocksSlack),
-					  NiceBytes(AllBlockChunksSize),
-					  NiceBytes(AllBlocksSize),
-					  AllBlockRequests,
-					  BlocksNeededCount);
 			{
 				ZEN_TRACE_CPU("WriteChunks_Wait");
 
-				Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-					ZEN_UNUSED(IsAborted, PendingWork);
-					ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load());
-					FilteredWrittenBytesPerSecond.Update(WriteToDiskBytes.load());
-					FilteredDownloadedBytesPerSecond.Update(BytesDownloaded.load());
-					std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.",
-													  RequestsComplete.load(),
-													  TotalRequestCount,
-													  NiceBytes(BytesDownloaded.load()),
-													  NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8),
-													  ChunkCountWritten.load(),
-													  ChunkCountToWrite,
-													  NiceBytes(WriteToDiskBytes.load()),
-													  NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()));
-					WriteProgressBar.UpdateState({.Task			  = "Writing chunks   ",
-												  .Details		  = Details,
-												  .TotalCount	  = gsl::narrow<uint64_t>(ChunkCountToWrite),
-												  .RemainingCount = gsl::narrow<uint64_t>(ChunkCountToWrite - ChunkCountWritten.load())},
-												 false);
+				Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+					ZEN_UNUSED(PendingWork);
+					uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() +
+											   DownloadStats.DownloadedBlockByteCount.load() +
+											   +DownloadStats.DownloadedPartialBlockByteCount.load();
+					FilteredWrittenBytesPerSecond.Update(DiskStats.WriteByteCount.load());
+					FilteredDownloadedBytesPerSecond.Update(DownloadedBytes);
+					std::string DownloadRateString =
+						(DownloadStats.RequestsCompleteCount == TotalRequestCount)
+							? ""
+							: fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8));
+					std::string WriteDetails = PrimeCacheOnly ? ""
+															  : fmt::format(" {}/{} ({}B/s) written.",
+																			NiceBytes(DiskStats.WriteByteCount.load()),
+																			NiceBytes(BytesToWrite),
+																			NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()));
+					std::string Details		 = fmt::format("{}/{} ({}{}) downloaded.{}",
+													   DownloadStats.RequestsCompleteCount.load(),
+													   TotalRequestCount,
+													   NiceBytes(DownloadedBytes),
+													   DownloadRateString,
+													   WriteDetails);
+					WriteProgressBar.UpdateState(
+						{.Task			 = PrimeCacheOnly ? "Downloading      " : "Writing chunks   ",
+						 .Details		 = Details,
+						 .TotalCount	 = PrimeCacheOnly ? TotalRequestCount : BytesToWrite,
+						 .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load())
+														  : (BytesToWrite - DiskStats.WriteByteCount.load()),
+						 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+						false);
 				});
 			}
 
 			FilteredWrittenBytesPerSecond.Stop();
 			FilteredDownloadedBytesPerSecond.Stop();
 
+			WriteProgressBar.Finish();
 			if (AbortFlag)
 			{
 				return;
 			}
 
-			WriteProgressBar.Finish();
-
-			uint32_t RawSequencesMissingWriteCount = 0;
-			for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++)
+			if (!PrimeCacheOnly)
 			{
-				const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex];
-				if (SequenceIndexChunksLeftToWriteCounter.load() != 0)
+				uint32_t RawSequencesMissingWriteCount = 0;
+				for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++)
 				{
-					RawSequencesMissingWriteCount++;
-					const uint32_t				 PathIndex		= RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
-					const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex];
-					ZEN_ASSERT(!IncompletePath.empty());
-					const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
-					ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount);
+					const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex];
+					if (SequenceIndexChunksLeftToWriteCounter.load() != 0)
+					{
+						RawSequencesMissingWriteCount++;
+						const uint32_t				 PathIndex		= RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+						const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex];
+						ZEN_ASSERT(!IncompletePath.empty());
+						const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
+						ZEN_CONSOLE("{}: Max count {}, Current count {}",
+									IncompletePath,
+									ExpectedSequenceCount,
+									SequenceIndexChunksLeftToWriteCounter.load());
+						ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount);
+					}
 				}
+				ZEN_ASSERT(RawSequencesMissingWriteCount == 0);
 			}
-			ZEN_ASSERT(RawSequencesMissingWriteCount == 0);
 
+			const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() +
+											 +DownloadStats.DownloadedPartialBlockByteCount.load();
 			ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
-						NiceBytes(BytesDownloaded.load()),
-						NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), BytesDownloaded * 8)),
+						NiceBytes(DownloadedBytes),
+						NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
 						NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
-						NiceBytes(WriteToDiskBytes.load()),
-						NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), WriteToDiskBytes.load())),
+						NiceBytes(DiskStats.WriteByteCount.load()),
+						NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
 						NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
 						NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
-		}
 
-		std::vector<std::pair<IoHash, uint32_t>> Targets;
-		Targets.reserve(RemoteContent.Paths.size());
-		for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++)
-		{
-			Targets.push_back(std::make_pair(RemoteContent.RawHashes[RemotePathIndex], RemotePathIndex));
+			WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
+			WriteChunkStats.DownloadTimeUs				 = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
+			WriteChunkStats.WriteTimeUs					 = FilteredWrittenBytesPerSecond.GetElapsedTimeUS();
 		}
-		std::sort(Targets.begin(), Targets.end(), [](const std::pair<IoHash, uint32_t>& Lhs, const std::pair<IoHash, uint32_t>& Rhs) {
-			return Lhs.first < Rhs.first;
-		});
 
-		// Move all files we will reuse to cache folder
-		// TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place
-		if (!LocalPathIndexesMatchingSequenceIndexes.empty())
+		if (PrimeCacheOnly)
 		{
-			ZEN_TRACE_CPU("UpdateFolder_CacheReused");
-			uint64_t TotalFullFileSizeCached = 0;
-			for (uint32_t LocalPathIndex : LocalPathIndexesMatchingSequenceIndexes)
-			{
-				const IoHash&				RawHash		  = LocalContent.RawHashes[LocalPathIndex];
-				const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
-				const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
-				ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath));
-				SetFileReadOnly(LocalFilePath, false);
-				ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath));
-				std::filesystem::rename(LocalFilePath, CacheFilePath);
-				TotalFullFileSizeCached += std::filesystem::file_size(CacheFilePath);
-			}
-			ZEN_CONSOLE("Saved {} ({}) unchanged files in cache",
-						LocalPathIndexesMatchingSequenceIndexes.size(),
-						NiceBytes(TotalFullFileSizeCached));
+			return;
 		}
 
-		if (WipeTargetFolder)
-		{
-			ZEN_TRACE_CPU("UpdateFolder_WipeTarget");
+		tsl::robin_map<uint32_t, uint32_t> RemotePathIndexToLocalPathIndex;
+		RemotePathIndexToLocalPathIndex.reserve(RemoteContent.Paths.size());
 
-			// Clean target folder
-			ZEN_CONSOLE("Wiping {}", Path);
-			if (!CleanDirectory(Path, DefaultExcludeFolders))
-			{
-				ZEN_WARN("Some files in {} could not be removed", Path);
-			}
+		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceHashToLocalPathIndex;
+		std::vector<uint32_t>							 RemoveLocalPathIndexes;
+
+		if (AbortFlag)
+		{
+			return;
 		}
-		else
+
 		{
-			ZEN_TRACE_CPU("UpdateFolder_RemoveUnused");
+			ZEN_TRACE_CPU("UpdateFolder_PrepareTarget");
 
-			// Remove unused tracked files
-			tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex;
+			tsl::robin_set<IoHash, IoHash::Hasher> CachedRemoteSequences;
+			tsl::robin_map<std::string, uint32_t>  RemotePathToRemoteIndex;
 			RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size());
 			for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++)
 			{
 				RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex});
 			}
-			std::vector<std::filesystem::path> LocalFilesToRemove;
+
+			std::vector<uint32_t> FilesToCache;
+
+			uint64_t			  MatchCount		= 0;
+			uint64_t			  PathMismatchCount = 0;
+			uint64_t			  HashMismatchCount = 0;
+			std::atomic<uint64_t> CachedCount		= 0;
+			std::atomic<uint64_t> CachedByteCount	= 0;
+			uint64_t			  SkippedCount		= 0;
+			uint64_t			  DeleteCount		= 0;
 			for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++)
 			{
-				if (!RemotePathToRemoteIndex.contains(LocalContent.Paths[LocalPathIndex].generic_string()))
+				if (AbortFlag)
+				{
+					break;
+				}
+				const IoHash&				 RawHash   = LocalContent.RawHashes[LocalPathIndex];
+				const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex];
+
+				ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred()));
+
+				if (!WipeTargetFolder)
 				{
-					const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
-					if (std::filesystem::exists(LocalFilePath))
+					if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string());
+						RemotePathIt != RemotePathToRemoteIndex.end())
+					{
+						const uint32_t RemotePathIndex = RemotePathIt->second;
+						if (RemoteContent.RawHashes[RemotePathIndex] == RawHash)
+						{
+							// It is already in it's desired place
+							RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex;
+							SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex});
+							MatchCount++;
+							continue;
+						}
+						else
+						{
+							HashMismatchCount++;
+						}
+					}
+					else
 					{
-						LocalFilesToRemove.emplace_back(std::move(LocalFilePath));
+						PathMismatchCount++;
 					}
 				}
+				if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash))
+				{
+					if (!CachedRemoteSequences.contains(RawHash))
+					{
+						ZEN_TRACE_CPU("MoveToCache");
+						// We need it
+						FilesToCache.push_back(LocalPathIndex);
+						CachedRemoteSequences.insert(RawHash);
+					}
+					else
+					{
+						// We already have it
+						SkippedCount++;
+					}
+				}
+				else if (!WipeTargetFolder)
+				{
+					// We don't need it
+					RemoveLocalPathIndexes.push_back(LocalPathIndex);
+					DeleteCount++;
+				}
+			}
+
+			if (AbortFlag)
+			{
+				return;
 			}
-			if (!LocalFilesToRemove.empty())
+
 			{
-				ZEN_CONSOLE("Cleaning {} removed files from {}", LocalFilesToRemove.size(), Path);
-				for (const std::filesystem::path& LocalFilePath : LocalFilesToRemove)
+				ZEN_TRACE_CPU("UpdateFolder_CopyToCache");
+
+				Stopwatch Timer;
+
+				WorkerThreadPool& WritePool = GetIOWorkerPool();
+
+				ProgressBar	 CacheLocalProgressBar(ProgressMode, "Cache Local Data");
+				ParallelWork Work(AbortFlag, PauseFlag);
+
+				for (uint32_t LocalPathIndex : FilesToCache)
+				{
+					if (AbortFlag)
+					{
+						break;
+					}
+					Work.ScheduleWork(
+						WritePool,
+						[&Path, &LocalContent, &CacheFolderPath, &CachedCount, &CachedByteCount, LocalPathIndex](std::atomic<bool>&) {
+							ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache");
+							if (!AbortFlag)
+							{
+								const IoHash&				 RawHash	   = LocalContent.RawHashes[LocalPathIndex];
+								const std::filesystem::path& LocalPath	   = LocalContent.Paths[LocalPathIndex];
+								const std::filesystem::path	 CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
+								ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath));
+								const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred();
+								RenameFileWithRetry(LocalFilePath, CacheFilePath);
+								CachedCount++;
+								CachedByteCount += LocalContent.RawSizes[LocalPathIndex];
+							}
+						});
+				}
+
+				{
+					ZEN_TRACE_CPU("CacheLocal_Wait");
+
+					Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+						ZEN_UNUSED(PendingWork);
+						const uint64_t WorkTotal	= FilesToCache.size();
+						const uint64_t WorkComplete = CachedCount.load();
+						std::string	   Details		= fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(CachedByteCount));
+						CacheLocalProgressBar.UpdateState({.Task		   = "Caching local    ",
+														   .Details		   = Details,
+														   .TotalCount	   = gsl::narrow<uint64_t>(WorkTotal),
+														   .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete),
+														   .Status		   = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+														  false);
+					});
+				}
+
+				CacheLocalProgressBar.Finish();
+				if (AbortFlag)
 				{
-					SetFileReadOnly(LocalFilePath, false);
-					std::filesystem::remove(LocalFilePath);
+					return;
 				}
+
+				ZEN_DEBUG(
+					"Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, "
+					"Delete: {}",
+					MatchCount,
+					PathMismatchCount,
+					HashMismatchCount,
+					CachedCount.load(),
+					NiceBytes(CachedByteCount.load()),
+					SkippedCount,
+					DeleteCount);
+			}
+		}
+
+		if (WipeTargetFolder)
+		{
+			ZEN_TRACE_CPU("UpdateFolder_WipeTarget");
+			Stopwatch Timer;
+
+			// Clean target folder
+			if (!CleanDirectory(Path, DefaultExcludeFolders))
+			{
+				ZEN_WARN("Some files in {} could not be removed", Path);
 			}
+			RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+		}
+
+		if (AbortFlag)
+		{
+			return;
 		}
 
 		{
 			ZEN_TRACE_CPU("UpdateFolder_FinalizeTree");
+			Stopwatch Timer;
 
-			WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst);  // GetSyncWorkerPool();  //
+			WorkerThreadPool& WritePool = GetIOWorkerPool();
 
-			ProgressBar	  RebuildProgressBar(UsePlainProgress);
-			ParallellWork Work(AbortFlag);
+			ProgressBar	 RebuildProgressBar(ProgressMode, "Rebuild State");
+			ParallelWork Work(AbortFlag, PauseFlag);
 
 			OutLocalFolderState.Paths.resize(RemoteContent.Paths.size());
 			OutLocalFolderState.RawSizes.resize(RemoteContent.Paths.size());
 			OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size());
 			OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size());
 
+			std::atomic<uint64_t> DeletedCount = 0;
+
+			for (uint32_t LocalPathIndex : RemoveLocalPathIndexes)
+			{
+				if (AbortFlag)
+				{
+					break;
+				}
+				Work.ScheduleWork(WritePool, [&Path, &LocalContent, &DeletedCount, LocalPathIndex](std::atomic<bool>&) {
+					if (!AbortFlag)
+					{
+						ZEN_TRACE_CPU("FinalizeTree_Remove");
+						const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+						SetFileReadOnlyWithRetry(LocalFilePath, false);
+						RemoveFileWithRetry(LocalFilePath);
+						DeletedCount++;
+					}
+				});
+			}
+
 			std::atomic<uint64_t> TargetsComplete = 0;
 
+			struct FinalizeTarget
+			{
+				IoHash	 RawHash;
+				uint32_t RemotePathIndex;
+			};
+
+			std::vector<FinalizeTarget> Targets;
+			Targets.reserve(RemoteContent.Paths.size());
+			for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++)
+			{
+				Targets.push_back(FinalizeTarget{.RawHash = RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex});
+			}
+			std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) {
+				if (Lhs.RawHash < Rhs.RawHash)
+				{
+					return true;
+				}
+				else if (Lhs.RawHash > Rhs.RawHash)
+				{
+					return false;
+				}
+				return Lhs.RemotePathIndex < Rhs.RemotePathIndex;
+			});
+
 			size_t TargetOffset = 0;
 			while (TargetOffset < Targets.size())
 			{
@@ -5708,95 +8199,189 @@ namespace {
 					break;
 				}
 
-				size_t		  TargetCount = 1;
-				const IoHash& RawHash	  = Targets[TargetOffset].first;
-				while (Targets[TargetOffset + TargetCount].first == RawHash)
+				size_t TargetCount = 1;
+				while (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash)
 				{
 					TargetCount++;
 				}
 
 				Work.ScheduleWork(
-					WritePool,	// GetSyncWorkerPool(),//
-					[&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic<bool>&) {
+					WritePool,
+					[&Path,
+					 &LocalContent,
+					 &SequenceHashToLocalPathIndex,
+					 &RemoteContent,
+					 &RemoteLookup,
+					 &CacheFolderPath,
+					 &Targets,
+					 &RemotePathIndexToLocalPathIndex,
+					 &RebuildFolderStateStats,
+					 &OutLocalFolderState,
+					 BaseTargetOffset = TargetOffset,
+					 TargetCount,
+					 &TargetsComplete](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
 							ZEN_TRACE_CPU("FinalizeTree_Work");
 
-							size_t						 TargetOffset		   = BaseTargetOffset;
-							const IoHash&				 RawHash			   = Targets[TargetOffset].first;
-							const uint32_t				 FirstTargetPathIndex  = Targets[TargetOffset].second;
-							const std::filesystem::path& FirstTargetPath	   = RemoteContent.Paths[FirstTargetPathIndex];
-							OutLocalFolderState.Paths[FirstTargetPathIndex]	   = FirstTargetPath;
-							OutLocalFolderState.RawSizes[FirstTargetPathIndex] = RemoteContent.RawSizes[FirstTargetPathIndex];
-							const std::filesystem::path FirstTargetFilePath	   = (Path / FirstTargetPath).make_preferred();
+							size_t		  TargetOffset = BaseTargetOffset;
+							const IoHash& RawHash	   = Targets[TargetOffset].RawHash;
+
 							if (RawHash == IoHash::Zero)
 							{
-								if (std::filesystem::exists(FirstTargetFilePath))
-								{
-									SetFileReadOnly(FirstTargetFilePath, false);
-								}
-								CreateDirectories(FirstTargetFilePath.parent_path());
+								ZEN_TRACE_CPU("ZeroSize");
+								while (TargetOffset < (BaseTargetOffset + TargetCount))
 								{
-									BasicFile OutputFile;
-									OutputFile.Open(FirstTargetFilePath, BasicFile::Mode::kTruncate);
+									const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
+									ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
+									const std::filesystem::path& TargetPath		= RemoteContent.Paths[RemotePathIndex];
+									std::filesystem::path		 TargetFilePath = (Path / TargetPath).make_preferred();
+									if (!RemotePathIndexToLocalPathIndex[RemotePathIndex])
+									{
+										if (IsFileWithRetry(TargetFilePath))
+										{
+											SetFileReadOnlyWithRetry(TargetFilePath, false);
+										}
+										else
+										{
+											CreateDirectories(TargetFilePath.parent_path());
+										}
+										BasicFile OutputFile;
+										OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate);
+									}
+									OutLocalFolderState.Paths[RemotePathIndex]	  = TargetPath;
+									OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex];
+
+									OutLocalFolderState.Attributes[RemotePathIndex] =
+										RemoteContent.Attributes.empty()
+											? GetNativeFileAttributes(TargetFilePath)
+											: SetNativeFileAttributes(TargetFilePath,
+																	  RemoteContent.Platform,
+																	  RemoteContent.Attributes[RemotePathIndex]);
+									OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
+
+									TargetOffset++;
+									TargetsComplete++;
 								}
 							}
 							else
 							{
-								ZEN_TRACE_CPU("FinalizeTree_MoveIntoPlace");
-
-								const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
-								ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath));
-								CreateDirectories(FirstTargetFilePath.parent_path());
-								if (std::filesystem::exists(FirstTargetFilePath))
+								ZEN_TRACE_CPU("Files");
+								ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash));
+								const uint32_t				 FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex;
+								const std::filesystem::path& FirstTargetPath	  = RemoteContent.Paths[FirstRemotePathIndex];
+								std::filesystem::path		 FirstTargetFilePath  = (Path / FirstTargetPath).make_preferred();
+
+								if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex);
+									InPlaceIt != RemotePathIndexToLocalPathIndex.end())
 								{
-									SetFileReadOnly(FirstTargetFilePath, false);
+									ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath));
 								}
-								std::filesystem::rename(CacheFilePath, FirstTargetFilePath);
-							}
+								else
+								{
+									if (IsFileWithRetry(FirstTargetFilePath))
+									{
+										SetFileReadOnlyWithRetry(FirstTargetFilePath, false);
+									}
+									else
+									{
+										CreateDirectories(FirstTargetFilePath.parent_path());
+									}
+
+									if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash);
+										InplaceIt != SequenceHashToLocalPathIndex.end())
+									{
+										ZEN_TRACE_CPU("Copy");
+										const uint32_t				 LocalPathIndex = InplaceIt->second;
+										const std::filesystem::path& SourcePath		= LocalContent.Paths[LocalPathIndex];
+										std::filesystem::path		 SourceFilePath = (Path / SourcePath).make_preferred();
+										ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath));
+
+										ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath);
+										const uint64_t		  RawSize = LocalContent.RawSizes[LocalPathIndex];
+										std::atomic<uint64_t> WriteCount;
+										std::atomic<uint64_t> WriteByteCount;
+										CopyFile(SourceFilePath, FirstTargetFilePath, RawSize, WriteCount, WriteByteCount);
+										RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
+									}
+									else
+									{
+										ZEN_TRACE_CPU("Rename");
+										const std::filesystem::path CacheFilePath =
+											GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
+										ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath));
 
-							OutLocalFolderState.Attributes[FirstTargetPathIndex] =
-								RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath)
-																 : SetNativeFileAttributes(FirstTargetFilePath,
-																						   RemoteContent.Platform,
-																						   RemoteContent.Attributes[FirstTargetPathIndex]);
-							OutLocalFolderState.ModificationTicks[FirstTargetPathIndex] = GetModificationTickFromPath(FirstTargetFilePath);
+										RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
 
-							TargetOffset++;
-							TargetsComplete++;
-							while (TargetOffset < (BaseTargetOffset + TargetCount))
-							{
-								ZEN_TRACE_CPU("FinalizeTree_Copy");
-
-								ZEN_ASSERT(Targets[TargetOffset].first == RawHash);
-								ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath));
-								const uint32_t				 ExtraTargetPathIndex  = Targets[TargetOffset].second;
-								const std::filesystem::path& ExtraTargetPath	   = RemoteContent.Paths[ExtraTargetPathIndex];
-								const std::filesystem::path	 ExtraTargetFilePath   = (Path / ExtraTargetPath).make_preferred();
-								OutLocalFolderState.Paths[ExtraTargetPathIndex]	   = ExtraTargetPath;
-								OutLocalFolderState.RawSizes[ExtraTargetPathIndex] = RemoteContent.RawSizes[ExtraTargetPathIndex];
-								CreateDirectories(ExtraTargetFilePath.parent_path());
-								if (std::filesystem::exists(ExtraTargetFilePath))
-								{
-									SetFileReadOnly(ExtraTargetFilePath, false);
+										RebuildFolderStateStats.FinalizeTreeFilesMovedCount++;
+									}
 								}
-								CopyFile(FirstTargetFilePath, ExtraTargetFilePath, {.EnableClone = false});
 
-								OutLocalFolderState.Attributes[ExtraTargetPathIndex] =
+								OutLocalFolderState.Paths[FirstRemotePathIndex]	   = FirstTargetPath;
+								OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex];
+
+								OutLocalFolderState.Attributes[FirstRemotePathIndex] =
 									RemoteContent.Attributes.empty()
-										? GetNativeFileAttributes(ExtraTargetFilePath)
-										: SetNativeFileAttributes(ExtraTargetFilePath,
+										? GetNativeFileAttributes(FirstTargetFilePath)
+										: SetNativeFileAttributes(FirstTargetFilePath,
 																  RemoteContent.Platform,
-																  RemoteContent.Attributes[ExtraTargetPathIndex]);
-								OutLocalFolderState.ModificationTicks[ExtraTargetPathIndex] =
-									GetModificationTickFromPath(ExtraTargetFilePath);
+																  RemoteContent.Attributes[FirstRemotePathIndex]);
+								OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] =
+									GetModificationTickFromPath(FirstTargetFilePath);
 
 								TargetOffset++;
 								TargetsComplete++;
+
+								while (TargetOffset < (BaseTargetOffset + TargetCount))
+								{
+									const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
+									ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
+									const std::filesystem::path& TargetPath		= RemoteContent.Paths[RemotePathIndex];
+									std::filesystem::path		 TargetFilePath = (Path / TargetPath).make_preferred();
+
+									if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex);
+										InPlaceIt != RemotePathIndexToLocalPathIndex.end())
+									{
+										ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath));
+									}
+									else
+									{
+										ZEN_TRACE_CPU("Copy");
+										if (IsFileWithRetry(TargetFilePath))
+										{
+											SetFileReadOnlyWithRetry(TargetFilePath, false);
+										}
+										else
+										{
+											CreateDirectories(TargetFilePath.parent_path());
+										}
+
+										ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath));
+										ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath);
+										const uint64_t		  RawSize = RemoteContent.RawSizes[RemotePathIndex];
+										std::atomic<uint64_t> WriteCount;
+										std::atomic<uint64_t> WriteByteCount;
+										CopyFile(FirstTargetFilePath, TargetFilePath, RawSize, WriteCount, WriteByteCount);
+										RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
+									}
+
+									OutLocalFolderState.Paths[RemotePathIndex]	  = TargetPath;
+									OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex];
+
+									OutLocalFolderState.Attributes[RemotePathIndex] =
+										RemoteContent.Attributes.empty()
+											? GetNativeFileAttributes(TargetFilePath)
+											: SetNativeFileAttributes(TargetFilePath,
+																	  RemoteContent.Platform,
+																	  RemoteContent.Attributes[RemotePathIndex]);
+									OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
+
+									TargetOffset++;
+									TargetsComplete++;
+								}
 							}
 						}
-					},
-					Work.DefaultErrorFunction());
+					});
 
 				TargetOffset += TargetCount;
 			}
@@ -5804,24 +8389,124 @@ namespace {
 			{
 				ZEN_TRACE_CPU("FinalizeTree_Wait");
 
-				Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-					ZEN_UNUSED(IsAborted, PendingWork);
-					std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size());
+				Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+					ZEN_UNUSED(PendingWork);
+					const uint64_t WorkTotal	= Targets.size() + RemoveLocalPathIndexes.size();
+					const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load();
+					std::string	   Details		= fmt::format("{}/{} files", WorkComplete, WorkTotal);
 					RebuildProgressBar.UpdateState({.Task			= "Rebuilding state ",
 													.Details		= Details,
-													.TotalCount		= gsl::narrow<uint64_t>(Targets.size()),
-													.RemainingCount = gsl::narrow<uint64_t>(Targets.size() - TargetsComplete.load())},
+													.TotalCount		= gsl::narrow<uint64_t>(WorkTotal),
+													.RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete),
+													.Status			= ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
 												   false);
 				});
 			}
 
-			if (AbortFlag)
+			RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+			RebuildProgressBar.Finish();
+		}
+	}
+
+	std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix)
+	{
+		ExtendableStringBuilder<512>					 SB;
+		std::vector<std::pair<std::string, std::string>> NameStringValuePairs;
+		for (CbFieldView Field : Object)
+		{
+			std::string_view Name = Field.GetName();
+			switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
 			{
-				return;
+				case CbFieldType::String:
+					NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())});
+					break;
+				case CbFieldType::IntegerPositive:
+					NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())});
+					break;
+				case CbFieldType::IntegerNegative:
+					NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())});
+					break;
+				case CbFieldType::Float32:
+					{
+						const float Value = Accessor.AsFloat32();
+						if (std::isfinite(Value))
+						{
+							NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)});
+						}
+						else
+						{
+							NameStringValuePairs.push_back({std::string(Name), "null"});
+						}
+					}
+					break;
+				case CbFieldType::Float64:
+					{
+						const double Value = Accessor.AsFloat64();
+						if (std::isfinite(Value))
+						{
+							NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)});
+						}
+						else
+						{
+							NameStringValuePairs.push_back({std::string(Name), "null"});
+						}
+					}
+					break;
+				case CbFieldType::BoolFalse:
+					NameStringValuePairs.push_back({std::string(Name), "false"});
+					break;
+				case CbFieldType::BoolTrue:
+					NameStringValuePairs.push_back({std::string(Name), "true"});
+					break;
+				case CbFieldType::Hash:
+					{
+						NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()});
+					}
+					break;
+				case CbFieldType::Uuid:
+					{
+						StringBuilder<Oid::StringLength + 1> Builder;
+						Accessor.AsUuid().ToString(Builder);
+						NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+					}
+					break;
+				case CbFieldType::DateTime:
+					{
+						ExtendableStringBuilder<64> Builder;
+						Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+						NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+					}
+					break;
+				case CbFieldType::TimeSpan:
+					{
+						ExtendableStringBuilder<64> Builder;
+						const TimeSpan				Span(Accessor.AsTimeSpanTicks());
+						if (Span.GetDays() == 0)
+						{
+							Builder << Span.ToString("%h:%m:%s.%n");
+						}
+						else
+						{
+							Builder << Span.ToString("%d.%h:%m:%s.%n");
+						}
+						NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+						break;
+					}
+				case CbFieldType::ObjectId:
+					NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()});
+					break;
 			}
-
-			RebuildProgressBar.Finish();
 		}
+		std::string::size_type LongestKey = 0;
+		for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
+		{
+			LongestKey = Max(KeyValue.first.length(), LongestKey);
+		}
+		for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
+		{
+			SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix));
+		}
+		return SB.ToString();
 	}
 
 	std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(BuildStorage&				Storage,
@@ -5833,17 +8518,13 @@ namespace {
 		std::vector<std::pair<Oid, std::string>> Result;
 		{
 			Stopwatch GetBuildTimer;
-
-			std::vector<std::pair<Oid, std::string>> AvailableParts;
-
-			CbObject BuildObject = Storage.GetBuild(BuildId);
-
+			CbObject  BuildObject = Storage.GetBuild(BuildId);
 			ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
 						NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
-						BuildObject["BuildName"sv].AsString(),
+						BuildObject["name"sv].AsString(),
 						NiceBytes(BuildObject.GetSize()));
 
-			ZEN_DEBUG("Build object: {}", BuildObject);
+			ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, "    "sv, "\n"sv));
 
 			CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
 			if (!PartsObject)
@@ -5853,6 +8534,8 @@ namespace {
 
 			OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize);
 
+			std::vector<std::pair<Oid, std::string>> AvailableParts;
+
 			for (CbFieldView PartView : PartsObject)
 			{
 				const std::string BuildPartName = std::string(PartView.GetName());
@@ -5914,7 +8597,7 @@ namespace {
 		return Result;
 	}
 
-	ChunkedFolderContent GetRemoteContent(BuildStorage&									  Storage,
+	ChunkedFolderContent GetRemoteContent(StorageInstance&								  Storage,
 										  const Oid&									  BuildId,
 										  const std::vector<std::pair<Oid, std::string>>& BuildParts,
 										  std::unique_ptr<ChunkingController>&			  OutChunkController,
@@ -5927,12 +8610,13 @@ namespace {
 		Stopwatch			   GetBuildPartTimer;
 		const Oid			   BuildPartId		 = BuildParts[0].first;
 		const std::string_view BuildPartName	 = BuildParts[0].second;
-		CbObject			   BuildPartManifest = Storage.GetBuildPart(BuildId, BuildPartId);
+		CbObject			   BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
 		ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
 					BuildPartId,
 					BuildPartName,
 					NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
 					NiceBytes(BuildPartManifest.GetSize()));
+		ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, "    "sv, "\n"sv));
 
 		{
 			CbObjectView	 Chunker	 = BuildPartManifest["chunker"sv].AsObjectView();
@@ -5941,7 +8625,7 @@ namespace {
 			OutChunkController			 = CreateChunkingController(ChunkerName, Parameters);
 		}
 
-		auto ParseBuildPartManifest = [](BuildStorage&						 Storage,
+		auto ParseBuildPartManifest = [](StorageInstance&					 Storage,
 										 const Oid&							 BuildId,
 										 const Oid&							 BuildPartId,
 										 CbObject							 BuildPartManifest,
@@ -5967,12 +8651,103 @@ namespace {
 
 			// TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them
 
-			Stopwatch GetBlockMetadataTimer;
-			OutBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockRawHashes);
-			ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
-						BuildPartId,
-						NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
-						OutBlockDescriptions.size());
+			{
+				Stopwatch GetBlockMetadataTimer;
+
+				std::vector<ChunkBlockDescription>			   UnorderedList;
+				tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup;
+				if (Storage.BuildCacheStorage)
+				{
+					std::vector<CbObject> CacheBlockMetadatas = Storage.BuildCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes);
+					UnorderedList.reserve(CacheBlockMetadatas.size());
+					for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < CacheBlockMetadatas.size();
+						 CacheBlockMetadataIndex++)
+					{
+						const CbObject&		  CacheBlockMetadata = CacheBlockMetadatas[CacheBlockMetadataIndex];
+						ChunkBlockDescription Description		 = ParseChunkBlockDescription(CacheBlockMetadata);
+						if (Description.BlockHash == IoHash::Zero)
+						{
+							ZEN_WARN("Unexpected/invalid block metadata received from remote cache, skipping block");
+						}
+						else
+						{
+							UnorderedList.emplace_back(std::move(Description));
+						}
+					}
+					for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++)
+					{
+						const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex];
+						BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex);
+					}
+				}
+
+				if (UnorderedList.size() < BlockRawHashes.size())
+				{
+					std::vector<IoHash> RemainingBlockHashes;
+					RemainingBlockHashes.reserve(BlockRawHashes.size() - UnorderedList.size());
+					for (const IoHash& BlockRawHash : BlockRawHashes)
+					{
+						if (!BlockDescriptionLookup.contains(BlockRawHash))
+						{
+							RemainingBlockHashes.push_back(BlockRawHash);
+						}
+					}
+					CbObject BlockMetadatas = Storage.BuildStorage->GetBlockMetadatas(BuildId, RemainingBlockHashes);
+					std::vector<ChunkBlockDescription> RemainingList;
+					{
+						CbArrayView			  BlocksArray = BlockMetadatas["blocks"sv].AsArrayView();
+						std::vector<IoHash>	  FoundBlockHashes;
+						std::vector<CbObject> FoundBlockMetadatas;
+						for (CbFieldView Block : BlocksArray)
+						{
+							ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView());
+
+							if (Description.BlockHash == IoHash::Zero)
+							{
+								ZEN_WARN("Unexpected/invalid block metadata received from remote store, skipping block");
+							}
+							else
+							{
+								if (Storage.BuildCacheStorage)
+								{
+									UniqueBuffer MetaBuffer = UniqueBuffer::Alloc(Block.GetSize());
+									Block.CopyTo(MetaBuffer.GetMutableView());
+									CbObject BlockMetadata(MetaBuffer.MoveToShared());
+
+									FoundBlockHashes.push_back(Description.BlockHash);
+									FoundBlockMetadatas.push_back(BlockMetadata);
+								}
+								RemainingList.emplace_back(std::move(Description));
+							}
+						}
+						if (Storage.BuildCacheStorage && !FoundBlockHashes.empty())
+						{
+							Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, FoundBlockHashes, FoundBlockMetadatas);
+						}
+					}
+
+					for (size_t DescriptionIndex = 0; DescriptionIndex < RemainingList.size(); DescriptionIndex++)
+					{
+						const ChunkBlockDescription& Description = RemainingList[DescriptionIndex];
+						BlockDescriptionLookup.insert_or_assign(Description.BlockHash, UnorderedList.size() + DescriptionIndex);
+					}
+					UnorderedList.insert(UnorderedList.end(), RemainingList.begin(), RemainingList.end());
+				}
+
+				OutBlockDescriptions.reserve(BlockDescriptionLookup.size());
+				for (const IoHash& BlockHash : BlockRawHashes)
+				{
+					if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end())
+					{
+						OutBlockDescriptions.push_back(std::move(UnorderedList[It->second]));
+					}
+				}
+
+				ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks",
+							BuildPartId,
+							NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
+							OutBlockDescriptions.size());
+			}
 
 			if (OutBlockDescriptions.size() != BlockRawHashes.size())
 			{
@@ -5985,7 +8760,8 @@ namespace {
 					ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription);
 					std::vector<ChunkBlockDescription> AugmentedBlockDescriptions;
 					AugmentedBlockDescriptions.reserve(BlockRawHashes.size());
-					std::vector<ChunkBlockDescription> FoundBlocks = Storage.FindBlocks(BuildId);
+					std::vector<ChunkBlockDescription> FoundBlocks =
+						ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, (uint64_t)-1));
 
 					for (const IoHash& BlockHash : BlockRawHashes)
 					{
@@ -6008,7 +8784,7 @@ namespace {
 						}
 						else
 						{
-							IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockHash);
+							IoBuffer BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockHash);
 							if (!BlockBuffer)
 							{
 								throw std::runtime_error(fmt::format("Block {} could not be found", BlockHash));
@@ -6074,7 +8850,7 @@ namespace {
 				const Oid&		   OverlayBuildPartId	= BuildParts[PartIndex].first;
 				const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
 				Stopwatch		   GetOverlayBuildPartTimer;
-				CbObject		   OverlayBuildPartManifest = Storage.GetBuildPart(BuildId, OverlayBuildPartId);
+				CbObject		   OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
 				ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
 							OverlayBuildPartId,
 							OverlayBuildPartName,
@@ -6137,257 +8913,303 @@ namespace {
 		return RemoteContent;
 	}
 
-	ChunkedFolderContent GetLocalContent(GetFolderContentStatistics&  LocalFolderScanStats,
-										 ChunkingStatistics&		  ChunkingStats,
-										 const std::filesystem::path& Path,
-										 ChunkingController&		  ChunkController)
+	ChunkedFolderContent GetLocalContent(GetFolderContentStatistics&			LocalFolderScanStats,
+										 ChunkingStatistics&					ChunkingStats,
+										 const std::filesystem::path&			Path,
+										 const std::filesystem::path&			StateFilePath,
+										 ChunkingController&					ChunkController,
+										 std::span<const std::filesystem::path> ReferencePaths,
+										 FolderContent&							OutLocalFolderContent)
 	{
+		FolderContent		 LocalFolderState;
 		ChunkedFolderContent LocalContent;
 
-		auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool {
-			for (const std::string_view& ExcludeFolder : ExcludeFolders)
+		Stopwatch  ReadStateTimer;
+		const bool HasLocalState = IsFile(StateFilePath) && ReadStateFile(StateFilePath, LocalFolderState, LocalContent);
+		if (HasLocalState)
+		{
+			ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
+		}
+		{
+			const uint32_t LocalPathCount  = gsl::narrow<uint32_t>(ReferencePaths.size());
+			const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size());
+
+			std::vector<std::filesystem::path> PathsToCheck;
+			PathsToCheck.reserve(LocalPathCount + RemotePathCount);
+
+			tsl::robin_set<std::string> FileSet;
+			FileSet.reserve(LocalPathCount + RemotePathCount);
+
+			for (const std::filesystem::path& LocalPath : LocalFolderState.Paths)
 			{
-				if (RelativePath.starts_with(ExcludeFolder))
-				{
-					if (RelativePath.length() == ExcludeFolder.length())
-					{
-						return false;
-					}
-					else if (RelativePath[ExcludeFolder.length()] == '/')
-					{
-						return false;
-					}
-				}
+				FileSet.insert(LocalPath.generic_string());
+				PathsToCheck.push_back(LocalPath);
 			}
-			return true;
-		};
 
-		auto IsAcceptedFile = [ExcludeExtensions =
-								   DefaultExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool {
-			for (const std::string_view& ExcludeExtension : ExcludeExtensions)
+			for (const std::filesystem::path& RemotePath : ReferencePaths)
 			{
-				if (RelativePath.ends_with(ExcludeExtension))
+				if (FileSet.insert(RemotePath.generic_string()).second)
 				{
-					return false;
+					PathsToCheck.push_back(RemotePath);
 				}
 			}
-			return true;
-		};
 
-		FolderContent CurrentLocalFolderContent = GetFolderContent(
-			LocalFolderScanStats,
-			Path,
-			std::move(IsAcceptedFolder),
-			std::move(IsAcceptedFile),
-			GetMediumWorkerPool(EWorkloadType::Burst),
-			UsePlainProgress ? 5000 : 200,
-			[&](bool, std::ptrdiff_t) { ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); },
-			AbortFlag);
-		if (AbortFlag)
-		{
-			return {};
+			ProgressBar ProgressBar(ProgressMode, "Check Files");
+			OutLocalFolderContent = GetValidFolderContent(
+				LocalFolderScanStats,
+				Path,
+				PathsToCheck,
+				[&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
+					std::string Details =
+						fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
+					ProgressBar.UpdateState({.Task			 = "Checking files   ",
+											 .Details		 = Details,
+											 .TotalCount	 = PathCount,
+											 .RemainingCount = PathCount - CompletedPathCount,
+											 .Status		 = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
+											false);
+				});
+			ProgressBar.Finish();
+			if (AbortFlag)
+			{
+				return {};
+			}
 		}
 
-		FolderContent LocalFolderState;
-
 		bool				  ScanContent = true;
 		std::vector<uint32_t> PathIndexesOufOfDate;
-		if (std::filesystem::is_regular_file(Path / ZenStateFilePath))
+		if (HasLocalState)
 		{
-			try
+			if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent))
 			{
-				Stopwatch ReadStateTimer;
-				CbObject  CurrentStateObject = LoadCompactBinaryObject(Path / ZenStateFilePath).Object;
-				if (CurrentStateObject)
-				{
-					Oid								  CurrentBuildId;
-					std::vector<Oid>				  SavedBuildPartIds;
-					std::vector<std::string>		  SavedBuildPartsNames;
-					std::vector<ChunkedFolderContent> SavedPartContents;
-					if (ReadStateObject(CurrentStateObject,
-										CurrentBuildId,
-										SavedBuildPartIds,
-										SavedBuildPartsNames,
-										SavedPartContents,
-										LocalFolderState))
-					{
-						if (!SavedPartContents.empty())
-						{
-							if (SavedPartContents.size() == 1)
-							{
-								LocalContent = std::move(SavedPartContents[0]);
-							}
-							else
-							{
-								LocalContent =
-									MergeChunkedFolderContents(SavedPartContents[0],
-															   std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1));
-							}
-
-							if (!LocalFolderState.AreKnownFilesEqual(CurrentLocalFolderContent))
-							{
-								const size_t					   LocaStatePathCount = LocalFolderState.Paths.size();
-								std::vector<std::filesystem::path> DeletedPaths;
-								FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, CurrentLocalFolderContent, DeletedPaths);
-								if (!DeletedPaths.empty())
-								{
-									LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
-								}
+				const size_t					   LocaStatePathCount = LocalFolderState.Paths.size();
+				std::vector<std::filesystem::path> DeletedPaths;
+				FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths);
+				if (!DeletedPaths.empty())
+				{
+					LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
+				}
 
-								ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
-											DeletedPaths.size(),
-											UpdatedContent.Paths.size(),
-											LocaStatePathCount);
-								if (UpdatedContent.Paths.size() > 0)
-								{
-									uint64_t ByteCountToScan = 0;
-									for (const uint64_t RawSize : UpdatedContent.RawSizes)
-									{
-										ByteCountToScan += RawSize;
-									}
-									ProgressBar	 ProgressBar(false);
-									FilteredRate FilteredBytesHashed;
-									FilteredBytesHashed.Start();
-									ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
-										ChunkingStats,
-										GetMediumWorkerPool(EWorkloadType::Burst),
-										Path,
-										UpdatedContent,
-										ChunkController,
-										UsePlainProgress ? 5000 : 200,
-										[&](bool, std::ptrdiff_t) {
-											FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
-											std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
-																			  ChunkingStats.FilesProcessed.load(),
-																			  UpdatedContent.Paths.size(),
-																			  NiceBytes(ChunkingStats.BytesHashed.load()),
-																			  NiceBytes(ByteCountToScan),
-																			  NiceNum(FilteredBytesHashed.GetCurrent()),
-																			  ChunkingStats.UniqueChunksFound.load(),
-																			  NiceBytes(ChunkingStats.UniqueBytesFound.load()));
-											ProgressBar.UpdateState({.Task			 = "Scanning files   ",
-																	 .Details		 = Details,
-																	 .TotalCount	 = ByteCountToScan,
-																	 .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()},
-																	false);
-										},
-										AbortFlag);
-									if (AbortFlag)
-									{
-										return {};
-									}
-									FilteredBytesHashed.Stop();
-									ProgressBar.Finish();
-									LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}});
-								}
-							}
-							else
-							{
-								// Remove files from LocalContent no longer in LocalFolderState
-								tsl::robin_set<std::string> LocalFolderPaths;
-								LocalFolderPaths.reserve(LocalFolderState.Paths.size());
-								for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths)
-								{
-									LocalFolderPaths.insert(LocalFolderPath.generic_string());
-								}
-								std::vector<std::filesystem::path> DeletedPaths;
-								for (const std::filesystem::path& LocalContentPath : LocalContent.Paths)
-								{
-									if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
-									{
-										DeletedPaths.push_back(LocalContentPath);
-									}
-								}
-								if (!DeletedPaths.empty())
-								{
-									LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
-								}
+				ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
+							DeletedPaths.size(),
+							UpdatedContent.Paths.size(),
+							LocaStatePathCount);
+				if (UpdatedContent.Paths.size() > 0)
+				{
+					uint64_t ByteCountToScan = 0;
+					for (const uint64_t RawSize : UpdatedContent.RawSizes)
+					{
+						ByteCountToScan += RawSize;
+					}
+					ProgressBar	 ProgressBar(ProgressMode, "Scan Files");
+					FilteredRate FilteredBytesHashed;
+					FilteredBytesHashed.Start();
+					ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
+						ChunkingStats,
+						GetIOWorkerPool(),
+						Path,
+						UpdatedContent,
+						ChunkController,
+						GetUpdateDelayMS(ProgressMode),
+						[&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
+							FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
+							std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
+															  ChunkingStats.FilesProcessed.load(),
+															  UpdatedContent.Paths.size(),
+															  NiceBytes(ChunkingStats.BytesHashed.load()),
+															  NiceBytes(ByteCountToScan),
+															  NiceNum(FilteredBytesHashed.GetCurrent()),
+															  ChunkingStats.UniqueChunksFound.load(),
+															  NiceBytes(ChunkingStats.UniqueBytesFound.load()));
+							ProgressBar.UpdateState({.Task			 = "Scanning files   ",
+													 .Details		 = Details,
+													 .TotalCount	 = ByteCountToScan,
+													 .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load(),
+													 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+													false);
+						},
+						AbortFlag,
+						PauseFlag);
 
-								ZEN_CONSOLE("Using cached local state");
-							}
-							ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
-							ScanContent = false;
-						}
+					FilteredBytesHashed.Stop();
+					ProgressBar.Finish();
+					if (AbortFlag)
+					{
+						return {};
 					}
+					LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}});
 				}
 			}
-			catch (const std::exception& Ex)
+			else
 			{
-				ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what());
+				// Remove files from LocalContent no longer in LocalFolderState
+				tsl::robin_set<std::string> LocalFolderPaths;
+				LocalFolderPaths.reserve(LocalFolderState.Paths.size());
+				for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths)
+				{
+					LocalFolderPaths.insert(LocalFolderPath.generic_string());
+				}
+				std::vector<std::filesystem::path> DeletedPaths;
+				for (const std::filesystem::path& LocalContentPath : LocalContent.Paths)
+				{
+					if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
+					{
+						DeletedPaths.push_back(LocalContentPath);
+					}
+				}
+				if (!DeletedPaths.empty())
+				{
+					LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths);
+				}
 			}
+			ScanContent = false;
 		}
 
 		if (ScanContent)
 		{
 			uint64_t ByteCountToScan = 0;
-			for (const uint64_t RawSize : CurrentLocalFolderContent.RawSizes)
+			for (const uint64_t RawSize : OutLocalFolderContent.RawSizes)
 			{
 				ByteCountToScan += RawSize;
 			}
-			ProgressBar	 ProgressBar(false);
+			ProgressBar	 ProgressBar(ProgressMode, "Scan Files");
 			FilteredRate FilteredBytesHashed;
 			FilteredBytesHashed.Start();
-			ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
+			LocalContent = ChunkFolderContent(
 				ChunkingStats,
-				GetMediumWorkerPool(EWorkloadType::Burst),
+				GetIOWorkerPool(),
 				Path,
-				CurrentLocalFolderContent,
+				OutLocalFolderContent,
 				ChunkController,
-				UsePlainProgress ? 5000 : 200,
-				[&](bool, std::ptrdiff_t) {
+				GetUpdateDelayMS(ProgressMode),
+				[&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
 					FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
 					std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
 													  ChunkingStats.FilesProcessed.load(),
-													  CurrentLocalFolderContent.Paths.size(),
+													  OutLocalFolderContent.Paths.size(),
 													  NiceBytes(ChunkingStats.BytesHashed.load()),
-													  ByteCountToScan,
+													  NiceBytes(ByteCountToScan),
 													  NiceNum(FilteredBytesHashed.GetCurrent()),
 													  ChunkingStats.UniqueChunksFound.load(),
 													  NiceBytes(ChunkingStats.UniqueBytesFound.load()));
 					ProgressBar.UpdateState({.Task			 = "Scanning files   ",
 											 .Details		 = Details,
 											 .TotalCount	 = ByteCountToScan,
-											 .RemainingCount = (ByteCountToScan - ChunkingStats.BytesHashed.load())},
+											 .RemainingCount = (ByteCountToScan - ChunkingStats.BytesHashed.load()),
+											 .Status		 = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
 											false);
 				},
-				AbortFlag);
+				AbortFlag,
+				PauseFlag);
 
+			FilteredBytesHashed.Stop();
+			ProgressBar.Finish();
 			if (AbortFlag)
 			{
 				return {};
 			}
-
-			FilteredBytesHashed.Stop();
-			ProgressBar.Finish();
 		}
 		return LocalContent;
 	}
 
-	void DownloadFolder(BuildStorage&				 Storage,
+	ChunkedFolderContent ScanAndChunkFolder(
+		GetFolderContentStatistics&																 GetFolderContentStats,
+		ChunkingStatistics&																		 ChunkingStats,
+		const std::filesystem::path&															 Path,
+		std::function<bool(const std::string_view& RelativePath)>&&								 IsAcceptedFolder,
+		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
+		ChunkingController&																		 ChunkController)
+	{
+		Stopwatch Timer;
+
+		ZEN_TRACE_CPU("ScanAndChunkFolder");
+
+		FolderContent Content = GetFolderContent(
+			GetFolderContentStats,
+			Path,
+			std::move(IsAcceptedFolder),
+			std::move(IsAcceptedFile),
+			GetIOWorkerPool(),
+			GetUpdateDelayMS(ProgressMode),
+			[](bool, std::ptrdiff_t) {},
+			AbortFlag);
+		if (AbortFlag)
+		{
+			return {};
+		}
+
+		FolderContent		 _;
+		ChunkedFolderContent Result = GetLocalContent(GetFolderContentStats,
+													  ChunkingStats,
+													  Path,
+													  ZenStateFilePath(Path / ZenFolderName),
+													  ChunkController,
+													  Content.Paths,
+													  _);
+
+		const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
+		const uint64_t ChunkedRawSize =
+			std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
+
+		ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+					Result.Paths.size(),
+					NiceBytes(TotalRawSize),
+					Result.ChunkedContent.ChunkHashes.size(),
+					NiceBytes(ChunkedRawSize),
+					Path,
+					NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+					NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+		return Result;
+	};
+
+	void DownloadFolder(StorageInstance&			 Storage,
 						const Oid&					 BuildId,
 						const std::vector<Oid>&		 BuildPartIds,
 						std::span<const std::string> BuildPartNames,
 						const std::filesystem::path& Path,
+						const std::filesystem::path& ZenFolderPath,
+						const std::filesystem::path	 SystemRootDir,
 						bool						 AllowMultiparts,
 						bool						 AllowPartialBlockRequests,
 						bool						 WipeTargetFolder,
-						bool						 PostDownloadVerify)
+						bool						 PostDownloadVerify,
+						bool						 PrimeCacheOnly,
+						bool						 EnableScavenging)
 	{
 		ZEN_TRACE_CPU("DownloadFolder");
 
+		ProgressBar::SetLogOperationName(ProgressMode, "Download Folder");
+
+		enum TaskSteps : uint32_t
+		{
+			CheckState,
+			CompareState,
+			Download,
+			Verify,
+			Cleanup,
+			StepCount
+		};
+
+		auto EndProgress =
+			MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
+
+		ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests)));
+
 		Stopwatch DownloadTimer;
 
-		const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName;
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount);
+
+		const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath);
 		CreateDirectories(ZenTempFolder);
 
-		CreateDirectories(Path / ZenTempBlockFolderName);
-		CreateDirectories(Path / ZenTempCacheFolderName);
-		CreateDirectories(Path / ZenTempDownloadFolderName);
+		CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath));
+		CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath));
+		CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath));
 
 		std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
 
 		std::vector<std::pair<Oid, std::string>> AllBuildParts =
-			ResolveBuildPartNames(Storage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+			ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
 
 		std::vector<ChunkedFolderContent> PartContents;
 
@@ -6396,29 +9218,41 @@ namespace {
 		std::vector<ChunkBlockDescription> BlockDescriptions;
 		std::vector<IoHash>				   LooseChunkHashes;
 
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount);
+
 		ChunkedFolderContent RemoteContent =
 			GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes);
 
-		const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
-		if (!ChunkController)
-		{
-			ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default");
-			ChunkController = CreateBasicChunkingController();
-		}
-
+		const std::uint64_t		   LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
 		GetFolderContentStatistics LocalFolderScanStats;
 		ChunkingStatistics		   ChunkingStats;
 		ChunkedFolderContent	   LocalContent;
-		if (std::filesystem::is_directory(Path))
+		FolderContent			   LocalFolderContent;
+		if (!PrimeCacheOnly)
 		{
-			if (!WipeTargetFolder)
+			if (IsDir(Path))
 			{
-				LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController);
+				if (!WipeTargetFolder)
+				{
+					if (!ChunkController)
+					{
+						ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default");
+						ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
+					}
+
+					LocalContent = GetLocalContent(LocalFolderScanStats,
+												   ChunkingStats,
+												   Path,
+												   ZenStateFilePath(ZenFolderPath),
+												   *ChunkController,
+												   RemoteContent.Paths,
+												   LocalFolderContent);
+				}
+			}
+			else
+			{
+				CreateDirectories(Path);
 			}
-		}
-		else
-		{
-			CreateDirectories(Path);
 		}
 		if (AbortFlag)
 		{
@@ -6471,19 +9305,42 @@ namespace {
 		{
 			ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
 						NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+
+			Stopwatch WriteStateTimer;
+			CbObject  StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path);
+			CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
+			TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
+			ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+
+			AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
 		}
 		else
 		{
-			ExtendableStringBuilder<128> SB;
+			ExtendableStringBuilder<128> BuildPartString;
 			for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
 			{
-				SB.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first));
+				BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first));
 			}
-			ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, SB.ToView());
+
+			uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));
+
+			ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
 			FolderContent LocalFolderState;
-			UpdateFolder(Storage,
+
+			DiskStatistics				 DiskStats;
+			CacheMappingStatistics		 CacheMappingStats;
+			DownloadStatistics			 DownloadStats;
+			WriteChunkStatistics		 WriteChunkStats;
+			RebuildFolderStateStatistics RebuildFolderStateStats;
+			VerifyFolderStatistics		 VerifyFolderStats;
+
+			ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);
+
+			UpdateFolder(SystemRootDir,
+						 Storage,
 						 BuildId,
 						 Path,
+						 ZenFolderPath,
 						 LargeAttachmentSize,
 						 PreferredMultipartChunkSize,
 						 LocalContent,
@@ -6492,42 +9349,122 @@ namespace {
 						 LooseChunkHashes,
 						 AllowPartialBlockRequests,
 						 WipeTargetFolder,
-						 LocalFolderState);
+						 PrimeCacheOnly,
+						 EnableScavenging,
+						 LocalFolderState,
+						 DiskStats,
+						 CacheMappingStats,
+						 DownloadStats,
+						 WriteChunkStats,
+						 RebuildFolderStateStats);
 
 			if (!AbortFlag)
 			{
-				VerifyFolder(RemoteContent, Path, PostDownloadVerify);
+				if (!PrimeCacheOnly)
+				{
+					ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount);
+
+					VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats);
+
+					Stopwatch WriteStateTimer;
+					CbObject  StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState, Path);
 
-				Stopwatch WriteStateTimer;
-				CbObject  StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState);
+					CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
+					TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
+					ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
 
-				CreateDirectories((Path / ZenStateFilePath).parent_path());
-				TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView());
-				ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+					AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
 
 #if 0
-				ExtendableStringBuilder<1024> SB;
-				CompactBinaryToJson(StateObject, SB);
-				WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+					ExtendableStringBuilder<1024> SB;
+					CompactBinaryToJson(StateObject, SB);
+					WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
 #endif	// 0
+				}
+				const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() +
+											   DownloadStats.DownloadedPartialBlockCount.load();
+				const uint64_t DownloadByteCount = DownloadStats.DownloadedChunkByteCount.load() +
+												   DownloadStats.DownloadedBlockByteCount.load() +
+												   DownloadStats.DownloadedPartialBlockByteCount.load();
+				const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();
+
+				ZEN_CONSOLE(
+					"Downloaded build {}, parts:{} in {}\n"
+					"    Download:   {} ({}) {}bits/s\n"
+					"    Write:      {} ({}) {}B/s\n"
+					"    Clean:      {}\n"
+					"    Finalize:   {}\n"
+					"    Verify:     {}",
+					BuildId,
+					BuildPartString.ToView(),
+					NiceTimeSpanMs(DownloadTimeMs),
+
+					DownloadCount,
+					NiceBytes(DownloadByteCount),
+					NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
+
+					DiskStats.WriteCount.load(),
+					NiceBytes(DiskStats.WriteByteCount.load()),
+					NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())),
 
-				ZEN_CONSOLE("Downloaded build in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+					NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
+
+					NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
+
+					NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+			}
+		}
+		if (PrimeCacheOnly)
+		{
+			if (Storage.BuildCacheStorage)
+			{
+				Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) {
+					if (Remaining == 0)
+					{
+						ZEN_CONSOLE("Build cache upload complete");
+					}
+					else
+					{
+						ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
+					}
+					return !AbortFlag;
+				});
 			}
 		}
+
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
+
 		if (CleanDirectory(ZenTempFolder, {}))
 		{
-			std::filesystem::remove(ZenTempFolder);
+			RemoveDirWithRetry(ZenTempFolder);
 		}
 	}
 
 	void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked)
 	{
+		ZEN_TRACE_CPU("DiffFolders");
+
+		ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders");
+
+		enum TaskSteps : uint32_t
+		{
+			CheckBase,
+			CheckCompare,
+			Diff,
+			Cleanup,
+			StepCount
+		};
+
+		auto EndProgress =
+			MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
+
 		ChunkedFolderContent BaseFolderContent;
 		ChunkedFolderContent CompareFolderContent;
 
 		{
-			std::unique_ptr<ChunkingController> ChunkController	  = CreateBasicChunkingController();
-			std::vector<std::string_view>		ExcludeExtensions = DefaultExcludeExtensions;
+			std::unique_ptr<ChunkingController> ChunkController =
+				CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
+			std::vector<std::string_view> ExcludeExtensions = DefaultExcludeExtensions;
 			if (OnlyChunked)
 			{
 				ExcludeExtensions.insert(ExcludeExtensions.end(),
@@ -6564,6 +9501,8 @@ namespace {
 				return true;
 			};
 
+			ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount);
+
 			GetFolderContentStatistics BaseGetFolderContentStats;
 			ChunkingStatistics		   BaseChunkingStats;
 			BaseFolderContent = ScanAndChunkFolder(BaseGetFolderContentStats,
@@ -6577,6 +9516,8 @@ namespace {
 				return;
 			}
 
+			ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount);
+
 			GetFolderContentStatistics CompareGetFolderContentStats;
 			ChunkingStatistics		   CompareChunkingStats;
 			CompareFolderContent = ScanAndChunkFolder(CompareGetFolderContentStats,
@@ -6592,6 +9533,8 @@ namespace {
 			}
 		}
 
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount);
+
 		std::vector<IoHash> AddedHashes;
 		std::vector<IoHash> RemovedHashes;
 		uint64_t			RemovedSize = 0;
@@ -6632,7 +9575,7 @@ namespace {
 
 		double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0;
 
-		ZEN_CONSOLE("{} ({}) files removed, {} ({}) files added, {} ({} {:.1f}%) files kept",
+		ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept",
 					RemovedHashes.size(),
 					NiceBytes(RemovedSize),
 					AddedHashes.size(),
@@ -6667,7 +9610,7 @@ namespace {
 		double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0;
 		double NewPercent	= CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0;
 
-		ZEN_CONSOLE("Found {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
+		ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
 					FoundChunkCount,
 					NiceBytes(FoundChunkSize),
 					FoundPercent,
@@ -6678,6 +9621,8 @@ namespace {
 					NewChunkCount,
 					NiceBytes(NewChunkSize),
 					NewPercent);
+
+		ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
 	}
 
 }  // namespace
@@ -6688,9 +9633,17 @@ BuildsCommand::BuildsCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
 
-	auto AddAuthOptions = [this](cxxopts::Options& Ops) {
-		Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value<std::string>(m_SystemRootDir), "<systemdir>");
+	auto AddSystemOptions = [this](cxxopts::Options& Ops) {
+		Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
+		Ops.add_option("",
+					   "",
+					   "use-sparse-files",
+					   "Enable use of sparse files when writing large files. Defaults to true.",
+					   cxxopts::value(m_UseSparseFiles),
+					   "<usesparsefiles>");
+	};
 
+	auto AddAuthOptions = [this](cxxopts::Options& Ops) {
 		// Direct access token (may expire)
 		Ops.add_option("auth-token",
 					   "",
@@ -6755,12 +9708,25 @@ BuildsCommand::BuildsCommand()
 					   "OAuth client secret",
 					   cxxopts::value<std::string>(m_OAuthClientSecret)->default_value(""),
 					   "");
+		Ops.add_option("auth",
+					   "",
+					   "oidctoken-exe-path",
+					   "Path to OidcToken executable",
+					   cxxopts::value<std::string>(m_OidcTokenAuthExecutablePath)->default_value(""),
+					   "");
 	};
 
 	auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) {
 		AddAuthOptions(Ops);
-
-		Ops.add_option("cloud build", "", "url", "Cloud Builds URL", cxxopts::value(m_BuildsUrl), "<url>");
+		Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
+		Ops.add_option("cloud build",
+					   "",
+					   "url",
+					   "Cloud Builds host url (legacy - use --override-host)",
+					   cxxopts::value(m_OverrideHost),
+					   "<url>");
+		Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>");
+		Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
 		Ops.add_option("cloud build",
 					   "",
 					   "assume-http2",
@@ -6770,6 +9736,12 @@ BuildsCommand::BuildsCommand()
 
 		Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>");
 		Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>");
+		Ops.add_option("cloud build",
+					   "",
+					   "allow-redirect",
+					   "Allow redirect of requests",
+					   cxxopts::value(m_AllowRedirect),
+					   "<allow-redirect>");
 	};
 
 	auto AddFileOptions = [this](cxxopts::Options& Ops) {
@@ -6782,25 +9754,104 @@ BuildsCommand::BuildsCommand()
 					   "<jsonmetadata>");
 	};
 
+	auto AddCacheOptions = [this](cxxopts::Options& Ops) {
+		Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
+	};
+
 	auto AddOutputOptions = [this](cxxopts::Options& Ops) {
-		Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<progress>");
+		Ops.add_option("output",
+					   "",
+					   "plain-progress",
+					   "Show progress using plain output",
+					   cxxopts::value(m_PlainProgress),
+					   "<plainprogress>");
+		Ops.add_option("output",
+					   "",
+					   "log-progress",
+					   "Write @progress style progress to output",
+					   cxxopts::value(m_LogProgress),
+					   "<logprogress>");
 		Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
 	};
 
-	m_Options.add_option("", "v", "verb", "Verb for build - list, upload, download, diff", cxxopts::value(m_Verb), "<verb>");
+	auto AddWorkerOptions = [this](cxxopts::Options& Ops) {
+		Ops.add_option("",
+					   "",
+					   "boost-workers",
+					   "Increase the number of worker threads - may cause computer to be less responsive",
+					   cxxopts::value(m_BoostWorkerThreads),
+					   "<boostworkers>");
+	};
+
+	auto AddZenFolderOptions = [this](cxxopts::Options& Ops) {
+		Ops.add_option("",
+					   "",
+					   "zen-folder-path",
+					   fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName),
+					   cxxopts::value(m_ZenFolderPath),
+					   "<boostworkers>");
+	};
+	m_Options.add_option("",
+						 "v",
+						 "verb",
+						 "Verb for build - list-namespaces, list, upload, download, diff, fetch-blob, validate-part",
+						 cxxopts::value(m_Verb),
+						 "<verb>");
 	m_Options.parse_positional({"verb"});
 	m_Options.positional_help("verb");
 
+	// list-namespaces
+	AddSystemOptions(m_ListNamespacesOptions);
+	AddCloudOptions(m_ListNamespacesOptions);
+	AddFileOptions(m_ListNamespacesOptions);
+	AddOutputOptions(m_ListNamespacesOptions);
+	AddZenFolderOptions(m_ListNamespacesOptions);
+	m_ListNamespacesOptions.add_options()("h,help", "Print help");
+	m_ListNamespacesOptions.add_option("",
+									   "",
+									   "recursive",
+									   "Enable fetch of buckets within namespaces also",
+									   cxxopts::value(m_ListNamespacesRecursive),
+									   "<recursive>");
+	m_ListNamespacesOptions.add_option("",
+									   "",
+									   "result-path",
+									   "Path to json or compactbinary to write query result to",
+									   cxxopts::value(m_ListResultPath),
+									   "<result-path>");
+	m_ListNamespacesOptions.parse_positional({"result-path"});
+	m_ListNamespacesOptions.positional_help("result-path");
+
 	// list
+	AddSystemOptions(m_ListOptions);
 	AddCloudOptions(m_ListOptions);
 	AddFileOptions(m_ListOptions);
 	AddOutputOptions(m_ListOptions);
+	AddZenFolderOptions(m_ListOptions);
 	m_ListOptions.add_options()("h,help", "Print help");
+	m_ListOptions.add_option("",
+							 "",
+							 "query-path",
+							 "Path to json or compactbinary file containing list query",
+							 cxxopts::value(m_ListQueryPath),
+							 "<query-path>");
+	m_ListOptions.add_option("",
+							 "",
+							 "result-path",
+							 "Path to json or compactbinary to write query result to",
+							 cxxopts::value(m_ListResultPath),
+							 "<result-path>");
+	m_ListOptions.parse_positional({"query-path", "result-path"});
+	m_ListOptions.positional_help("query-path result-path");
 
 	// upload
+	AddSystemOptions(m_UploadOptions);
 	AddCloudOptions(m_UploadOptions);
 	AddFileOptions(m_UploadOptions);
 	AddOutputOptions(m_UploadOptions);
+	AddCacheOptions(m_UploadOptions);
+	AddWorkerOptions(m_UploadOptions);
+	AddZenFolderOptions(m_UploadOptions);
 	m_UploadOptions.add_options()("h,help", "Print help");
 	m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
 	m_UploadOptions.add_option("",
@@ -6856,14 +9907,31 @@ BuildsCommand::BuildsCommand()
 							   "<manifestpath>");
 	m_UploadOptions
 		.add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>");
+	m_UploadOptions.add_option("",
+							   "",
+							   "find-max-block-count",
+							   "The maximum number of blocks we search for in the build context",
+							   cxxopts::value(m_FindBlockMaxCount),
+							   "<maxblockcount>");
 
 	m_UploadOptions.parse_positional({"local-path", "build-id"});
 	m_UploadOptions.positional_help("local-path build-id");
 
 	// download
+	AddSystemOptions(m_DownloadOptions);
 	AddCloudOptions(m_DownloadOptions);
 	AddFileOptions(m_DownloadOptions);
 	AddOutputOptions(m_DownloadOptions);
+	AddCacheOptions(m_DownloadOptions);
+	AddZenFolderOptions(m_DownloadOptions);
+	AddWorkerOptions(m_DownloadOptions);
+	m_DownloadOptions.add_option("cache",
+								 "",
+								 "cache-prime-only",
+								 "Only download blobs missing in cache and upload to cache",
+								 cxxopts::value(m_PrimeCacheOnly),
+								 "<cacheprimeonly>");
+
 	m_DownloadOptions.add_options()("h,help", "Print help");
 	m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
 	m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
@@ -6895,12 +9963,20 @@ BuildsCommand::BuildsCommand()
 								 "Allow request for partial chunk blocks. Defaults to true.",
 								 cxxopts::value(m_AllowPartialBlockRequests),
 								 "<allowpartialblockrequests>");
+
 	m_DownloadOptions
 		.add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>");
+	m_DownloadOptions.add_option("",
+								 "",
+								 "enable-scavenge",
+								 "Enable scavenging of data from previouse download locations",
+								 cxxopts::value(m_EnableScavenging),
+								 "<scavenge>");
 	m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"});
 	m_DownloadOptions.positional_help("local-path build-id build-part-name");
 
 	AddOutputOptions(m_DiffOptions);
+	AddWorkerOptions(m_DiffOptions);
 	m_DiffOptions.add_options()("h,help", "Print help");
 	m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
 	m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>");
@@ -6913,9 +9989,12 @@ BuildsCommand::BuildsCommand()
 	m_DiffOptions.parse_positional({"local-path", "compare-path"});
 	m_DiffOptions.positional_help("local-path compare-path");
 
+	AddSystemOptions(m_TestOptions);
 	AddCloudOptions(m_TestOptions);
 	AddFileOptions(m_TestOptions);
 	AddOutputOptions(m_TestOptions);
+	AddCacheOptions(m_TestOptions);
+	AddWorkerOptions(m_TestOptions);
 	m_TestOptions.add_options()("h,help", "Print help");
 	m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
 	m_TestOptions.add_option("",
@@ -6930,21 +10009,48 @@ BuildsCommand::BuildsCommand()
 							 "Allow request for partial chunk blocks. Defaults to true.",
 							 cxxopts::value(m_AllowPartialBlockRequests),
 							 "<allowpartialblockrequests>");
+	m_TestOptions.add_option("",
+							 "",
+							 "enable-scavenge",
+							 "Enable scavenging of data from previouse download locations",
+							 cxxopts::value(m_EnableScavenging),
+							 "<scavenge>");
 	m_TestOptions.parse_positional({"local-path"});
 	m_TestOptions.positional_help("local-path");
 
+	AddSystemOptions(m_FetchBlobOptions);
 	AddCloudOptions(m_FetchBlobOptions);
 	AddFileOptions(m_FetchBlobOptions);
 	AddOutputOptions(m_FetchBlobOptions);
+	AddCacheOptions(m_FetchBlobOptions);
+	AddZenFolderOptions(m_FetchBlobOptions);
 	m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
 	m_FetchBlobOptions
 		.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>");
 	m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"});
 	m_FetchBlobOptions.positional_help("build-id blob-hash");
 
+	auto AddZenProcessOptions = [this](cxxopts::Options& Ops) {
+		Ops.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
+	};
+	AddZenProcessOptions(m_PauseOptions);
+	m_PauseOptions.parse_positional({"process-id"});
+	m_PauseOptions.positional_help("process-id");
+
+	AddZenProcessOptions(m_ResumeOptions);
+	m_ResumeOptions.parse_positional({"process-id"});
+	m_ResumeOptions.positional_help("process-id");
+
+	AddZenProcessOptions(m_AbortOptions);
+	m_AbortOptions.parse_positional({"process-id"});
+	m_AbortOptions.positional_help("process-id");
+
+	AddSystemOptions(m_ValidateBuildPartOptions);
 	AddCloudOptions(m_ValidateBuildPartOptions);
 	AddFileOptions(m_ValidateBuildPartOptions);
 	AddOutputOptions(m_ValidateBuildPartOptions);
+	AddWorkerOptions(m_ValidateBuildPartOptions);
+	AddZenFolderOptions(m_ValidateBuildPartOptions);
 	m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
 	m_ValidateBuildPartOptions.add_option("",
 										  "",
@@ -6962,12 +10068,21 @@ BuildsCommand::BuildsCommand()
 	m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"});
 	m_ValidateBuildPartOptions.positional_help("build-id build-part-id");
 
+	AddSystemOptions(m_MultiTestDownloadOptions);
 	AddCloudOptions(m_MultiTestDownloadOptions);
 	AddFileOptions(m_MultiTestDownloadOptions);
 	AddOutputOptions(m_MultiTestDownloadOptions);
+	AddCacheOptions(m_MultiTestDownloadOptions);
+	AddWorkerOptions(m_MultiTestDownloadOptions);
 	m_MultiTestDownloadOptions
 		.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
 	m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>");
+	m_MultiTestDownloadOptions.add_option("",
+										  "",
+										  "enable-scavenge",
+										  "Enable scavenging of data from previouse download locations",
+										  cxxopts::value(m_EnableScavenging),
+										  "<scavenge>");
 	m_MultiTestDownloadOptions.parse_positional({"local-path"});
 	m_MultiTestDownloadOptions.positional_help("local-path");
 }
@@ -7004,29 +10119,75 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		return 0;
 	}
 
-	auto ParseStorageOptions = [&]() {
-		if (!m_BuildsUrl.empty())
+	std::filesystem::path SystemRootDir;
+	auto				  ParseSystemOptions = [&]() {
+		 if (m_SystemRootDir.empty())
+		 {
+			 m_SystemRootDir = PickDefaultSystemRootDirectory();
+		 }
+		 MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+	};
+	ParseSystemOptions();
+
+	auto ParseStorageOptions = [&](bool RequireNamespace, bool RequireBucket) {
+		m_Host		   = RemoveQuotes(m_Host);
+		m_OverrideHost = RemoveQuotes(m_OverrideHost);
+		m_Url		   = RemoveQuotes(m_Url);
+		m_Namespace	   = RemoveQuotes(m_Namespace);
+		m_Bucket	   = RemoveQuotes(m_Bucket);
+		if (!m_Url.empty())
 		{
-			if (!m_StoragePath.empty())
+			if (!m_Host.empty())
+			{
+				throw zen::OptionParseException(fmt::format("host is not compatible with the url option\n{}", SubOption->help()));
+			}
+			if (!m_Bucket.empty())
+			{
+				throw zen::OptionParseException(fmt::format("bucket is not compatible with the url option\n{}", SubOption->help()));
+			}
+			if (!m_BuildId.empty())
 			{
-				throw zen::OptionParseException(fmt::format("url is not compatible with the storage-path option\n{}", m_Options.help()));
+				throw zen::OptionParseException(fmt::format("buildid is not compatible with the url option\n{}", SubOption->help()));
 			}
-			if (m_Namespace.empty() || m_Bucket.empty())
+			if (!ParseCloudUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId))
+			{
+				throw zen::OptionParseException(fmt::format("url does not match the Cloud Artifact URL format\n{}", SubOption->help()));
+			}
+		}
+
+		if (!m_OverrideHost.empty() || !m_Host.empty())
+		{
+			if (!m_StoragePath.empty())
 			{
 				throw zen::OptionParseException(
-					fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help()));
+					fmt::format("host/url/override-host is not compatible with the storage-path option\n{}", SubOption->help()));
+			}
+			if (RequireNamespace && m_Namespace.empty())
+			{
+				throw zen::OptionParseException(fmt::format("namespace option is required for this storage option\n{}", SubOption->help()));
+			}
+			if (RequireBucket && m_Bucket.empty())
+			{
+				throw zen::OptionParseException(fmt::format("bucket option is required for this storage option\n{}", SubOption->help()));
 			}
 		}
+		else if (m_StoragePath.empty())
+		{
+			throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", SubOption->help()));
+		}
+		MakeSafeAbsolutePathÍnPlace(m_StoragePath);
 	};
 
 	std::unique_ptr<AuthMgr> Auth;
-	HttpClientSettings		 ClientSettings{.AssumeHttp2 = m_AssumeHttp2, .AllowResume = true, .RetryCount = 2};
+	HttpClientSettings		 ClientSettings{.LogCategory = "httpbuildsclient",
+											.AssumeHttp2 = m_AssumeHttp2,
+											.AllowResume = true,
+											.RetryCount	 = 2};
 
 	auto CreateAuthMgr = [&]() {
+		ZEN_ASSERT(!m_SystemRootDir.empty());
 		if (!Auth)
 		{
-			std::filesystem::path DataRoot = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : StringToPath(m_SystemRootDir);
-
 			if (m_EncryptionKey.empty())
 			{
 				m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456";
@@ -7039,7 +10200,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				ZEN_CONSOLE("Warning: Using default encryption initialization vector");
 			}
 
-			AuthConfig AuthMgrConfig = {.RootDirectory = DataRoot / "auth",
+			AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth",
 										.EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey),
 										.EncryptionIV  = AesIV128Bit::FromString(m_EncryptionIV)};
 			if (!AuthMgrConfig.EncryptionKey.IsValid())
@@ -7055,6 +10216,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	auto ParseAuthOptions = [&]() {
+		m_OpenIdProviderUrl = RemoveQuotes(m_OpenIdProviderUrl);
+		m_OpenIdClientId	= RemoveQuotes(m_OpenIdClientId);
+
+		m_AccessToken	= RemoveQuotes(m_AccessToken);
+		m_EncryptionKey = RemoveQuotes(m_EncryptionKey);
+		m_EncryptionIV	= RemoveQuotes(m_EncryptionIV);
+
+		m_OAuthUrl			= RemoveQuotes(m_OAuthUrl);
+		m_OAuthClientId		= RemoveQuotes(m_OAuthClientId);
+		m_OAuthClientSecret = RemoveQuotes(m_OAuthClientSecret);
+
+		m_OidcTokenAuthExecutablePath = RemoveQuotes(m_OidcTokenAuthExecutablePath);
+
 		if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty())
 		{
 			CreateAuthMgr();
@@ -7066,26 +10240,48 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			}
 		}
 
+		auto GetEnvAccessToken = [](const std::string& AccessTokenEnv) -> std::string {
+			if (!AccessTokenEnv.empty())
+			{
+				return GetEnvVariable(AccessTokenEnv);
+			}
+			return {};
+		};
+
+		auto FindOidcTokenExePath = [](const std::string& OidcTokenAuthExecutablePath) -> std::filesystem::path {
+			if (OidcTokenAuthExecutablePath.empty())
+			{
+				const std::string	  OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL;
+				std::filesystem::path OidcTokenPath		 = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred();
+				if (IsFile(OidcTokenPath))
+				{
+					return OidcTokenPath;
+				}
+			}
+			else
+			{
+				std::filesystem::path OidcTokenPath = std::filesystem::absolute(StringToPath(OidcTokenAuthExecutablePath)).make_preferred();
+				if (IsFile(OidcTokenPath))
+				{
+					return OidcTokenPath;
+				}
+			}
+			return {};
+		};
+
 		if (!m_AccessToken.empty())
 		{
 			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(m_AccessToken);
 		}
 		else if (!m_AccessTokenPath.empty())
 		{
+			MakeSafeAbsolutePathÍnPlace(m_AccessTokenPath);
 			std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath);
 			if (!ResolvedAccessToken.empty())
 			{
 				ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken);
 			}
 		}
-		else if (!m_AccessTokenEnv.empty())
-		{
-			std::string ResolvedAccessToken = GetEnvVariable(m_AccessTokenEnv);
-			if (!ResolvedAccessToken.empty())
-			{
-				ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken);
-			}
-		}
 		else if (!m_OAuthUrl.empty())
 		{
 			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOAuthClientCredentials(
@@ -7096,219 +10292,660 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			CreateAuthMgr();
 			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOpenIdProvider(*Auth, m_OpenIdProviderName);
 		}
-		else
+		else if (std::string ResolvedAccessToken = GetEnvAccessToken(m_AccessTokenEnv); !ResolvedAccessToken.empty())
 		{
-			CreateAuthMgr();
-			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth);
+			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken);
+		}
+		else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty())
+		{
+			const std::string& CloudHost	   = m_OverrideHost.empty() ? m_Host : m_OverrideHost;
+			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost);
 		}
 
-		if (!m_BuildsUrl.empty() && !ClientSettings.AccessTokenProvider)
+		if (!ClientSettings.AccessTokenProvider)
 		{
-			ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials.");
+			CreateAuthMgr();
+			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth);
 		}
 	};
 
 	auto ParseOutputOptions = [&]() {
-		IsVerbose		 = m_Verbose;
-		UsePlainProgress = IsVerbose || m_PlainProgress;
+		IsVerbose = m_Verbose;
+		if (m_LogProgress)
+		{
+			ProgressMode = ProgressBar::Mode::Log;
+		}
+		else if (m_Verbose || m_PlainProgress)
+		{
+			ProgressMode = ProgressBar::Mode::Plain;
+		}
+		else
+		{
+			ProgressMode = ProgressBar::Mode::Pretty;
+		}
 	};
 	ParseOutputOptions();
 
-	try
-	{
-		if (SubOption == &m_ListOptions)
+	auto CreateBuildStorage = [&](BuildStorage::Statistics&		 StorageStats,
+								  BuildStorageCache::Statistics& StorageCacheStats,
+								  const std::filesystem::path&	 TempPath,
+								  bool							 RequireNamespace,
+								  bool							 RequireBucket) -> StorageInstance {
+		ParseStorageOptions(RequireNamespace, RequireBucket);
+
+		m_ZenCacheHost = RemoveQuotes(m_ZenCacheHost);
+
+		StorageInstance Result;
+
+		std::string BuildStorageName = ZEN_CLOUD_STORAGE;
+		std::string BuildCacheName;
+		bool		CacheAssumeHttp2 = false;
+		std::string StorageDescription;
+		std::string CacheDescription;
+
+		if (!m_Host.empty() || !m_OverrideHost.empty())
 		{
-			ParseStorageOptions();
 			ParseAuthOptions();
+		}
 
-			HttpClient Http(m_BuildsUrl, ClientSettings);
+		std::string CloudHost;
 
-			CbObjectWriter QueryWriter;
-			QueryWriter.BeginObject("query");
+		auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair<bool, std::string> {
+			HttpClientSettings	 TestClientSettings{.LogCategory	= "httpcacheclient",
+													.ConnectTimeout = std::chrono::milliseconds{1000},
+													.Timeout		= std::chrono::milliseconds{2000},
+													.AssumeHttp2	= AssumeHttp2,
+													.AllowResume	= true,
+													.RetryCount		= 0};
+			HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
+			HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds");
+			if (TestResponse.IsSuccess())
 			{
-				//			QueryWriter.BeginObject("platform");
-				//			{
-				//				QueryWriter.AddString("$eq", "Windows");
-				//			}
-				//			QueryWriter.EndObject(); // changelist
+				return {true, ""};
 			}
-			QueryWriter.EndObject();  // query
+			return {false, TestResponse.ErrorMessage("")};
+		};
 
-			BuildStorage::Statistics	  StorageStats;
-			std::unique_ptr<BuildStorage> Storage;
-			if (!m_BuildsUrl.empty())
+		if (!m_Host.empty())
+		{
+			if (m_OverrideHost.empty() || m_ZenCacheHost.empty())
 			{
-				ZEN_CONSOLE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'",
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket);
-				Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{});
+				HttpClient			 DiscoveryHttpClient(m_Host, ClientSettings);
+				HttpClient::Response ServerInfoResponse =
+					DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON));
+				if (!ServerInfoResponse.IsSuccess())
+				{
+					throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'",
+														 m_Host,
+														 ServerInfoResponse.ErrorMessage("")));
+				}
+
+				std::string_view JsonResponse		= ServerInfoResponse.AsText();
+				CbObject		 ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject();
+
+				auto TestHostEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair<bool, std::string> {
+					HttpClientSettings TestClientSettings{.LogCategory	  = "httpbuildsclient",
+														  .ConnectTimeout = std::chrono::milliseconds{1000},
+														  .Timeout		  = std::chrono::milliseconds{2000},
+														  .AssumeHttp2	  = AssumeHttp2,
+														  .AllowResume	  = true,
+														  .RetryCount	  = 0};
+
+					HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
+					HttpClient::Response TestResponse = TestHttpClient.Get("/health/live");
+					if (TestResponse.IsSuccess())
+					{
+						return {true, ""};
+					}
+					return {false, TestResponse.ErrorMessage("")};
+				};
+
+				if (m_OverrideHost.empty())
+				{
+					CbArrayView	  ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView();
+					std::uint64_t ServerCount		   = ServerEndpointsArray.Num();
+					if (ServerCount == 0)
+					{
+						throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host));
+					}
+					for (CbFieldView ServerEndpointView : ServerEndpointsArray)
+					{
+						CbObjectView ServerEndpointObject = ServerEndpointView.AsObjectView();
+
+						std::string_view BaseUrl = ServerEndpointObject["baseUrl"sv].AsString();
+						if (!BaseUrl.empty())
+						{
+							const bool		 AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false);
+							std::string_view Name		 = ServerEndpointObject["name"sv].AsString();
+							if (auto TestResult = TestHostEndpoint(BaseUrl, AssumeHttp2); TestResult.first)
+							{
+								CloudHost		 = BaseUrl;
+								m_AssumeHttp2	 = AssumeHttp2;
+								BuildStorageName = Name;
+								break;
+							}
+							else
+							{
+								ZEN_DEBUG("Unable to reach host {}. Reason: {}", BaseUrl, TestResult.second);
+							}
+						}
+					}
+					if (CloudHost.empty())
+					{
+						throw std::runtime_error(
+							fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host));
+					}
+				}
+				else if (auto TestResult = TestHostEndpoint(m_OverrideHost, m_AssumeHttp2); TestResult.first)
+				{
+					CloudHost = m_OverrideHost;
+				}
+				else
+				{
+					throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.second));
+				}
+
+				if (m_ZenCacheHost.empty())
+				{
+					CbArrayView	  CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView();
+					std::uint64_t CacheCount		  = CacheEndpointsArray.Num();
+					for (CbFieldView CacheEndpointView : CacheEndpointsArray)
+					{
+						CbObjectView CacheEndpointObject = CacheEndpointView.AsObjectView();
+
+						std::string_view BaseUrl = CacheEndpointObject["baseUrl"sv].AsString();
+						if (!BaseUrl.empty())
+						{
+							const bool		 AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false);
+							std::string_view Name		 = CacheEndpointObject["name"sv].AsString();
+
+							if (auto TestResult = TestCacheEndpoint(BaseUrl, AssumeHttp2); TestResult.first)
+							{
+								m_ZenCacheHost	 = BaseUrl;
+								CacheAssumeHttp2 = AssumeHttp2;
+								BuildCacheName	 = Name;
+								break;
+							}
+						}
+					}
+					if (m_ZenCacheHost.empty())
+					{
+						ZenServerState State;
+						if (State.InitializeReadOnly())
+						{
+							State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+								if (m_ZenCacheHost.empty())
+								{
+									std::string ZenServerLocalHostUrl =
+										fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
+									if (auto TestResult = TestCacheEndpoint(ZenServerLocalHostUrl, false); TestResult.first)
+									{
+										m_ZenCacheHost	 = ZenServerLocalHostUrl;
+										CacheAssumeHttp2 = false;
+										BuildCacheName	 = "localhost";
+									}
+								}
+							});
+						}
+						if (m_ZenCacheHost.empty())
+						{
+							ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
+						}
+					}
+				}
+				else if (auto TestResult = TestCacheEndpoint(m_ZenCacheHost, false); TestResult.first)
+				{
+					std::string::size_type HostnameStart  = 0;
+					std::string::size_type HostnameLength = std::string::npos;
+					if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos)
+					{
+						HostnameStart = StartPos + 2;
+					}
+					if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos)
+					{
+						HostnameLength = EndPos - HostnameStart;
+					}
+					BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength);
+				}
+				else
+				{
+					ZEN_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second);
+				}
 			}
-			else if (!m_StoragePath.empty())
+		}
+		else
+		{
+			CloudHost = m_OverrideHost;
+		}
+
+		if (!CloudHost.empty())
+		{
+			Result.BuildStorageHttp = std::make_unique<HttpClient>(CloudHost, ClientSettings);
+			StorageDescription		= fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
+											  BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName),
+											  CloudHost,
+											  Result.BuildStorageHttp->GetSessionId(),
+											  m_Namespace,
+											  m_Bucket);
+			Result.BuildStorage		= CreateJupiterBuildStorage(Log(),
+															*Result.BuildStorageHttp,
+															StorageStats,
+															m_Namespace,
+															m_Bucket,
+															m_AllowRedirect,
+															TempPath / "storage");
+			Result.StorageName		= BuildStorageName;
+		}
+		else if (!m_StoragePath.empty())
+		{
+			StorageDescription	= fmt::format("folder {}", m_StoragePath);
+			Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
+			Result.StorageName	= fmt::format("Disk {}", m_StoragePath.stem());
+		}
+		else
+		{
+			throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", SubOption->help()));
+		}
+		if (!m_ZenCacheHost.empty())
+		{
+			Result.CacheHttp		 = std::make_unique<HttpClient>(m_ZenCacheHost,
+															HttpClientSettings{.LogCategory	   = "httpcacheclient",
+																					   .ConnectTimeout = std::chrono::milliseconds{3000},
+																					   .Timeout		   = std::chrono::milliseconds{30000},
+																					   .AssumeHttp2	   = CacheAssumeHttp2,
+																					   .AllowResume	   = true,
+																					   .RetryCount	   = 0});
+			Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp,
+																  StorageCacheStats,
+																  m_Namespace,
+																  m_Bucket,
+																  TempPath / "zencache",
+																  m_PrimeCacheOnly);
+			CacheDescription		 = fmt::format("Zen {}{}. SessionId: '{}'",
+										   BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName),
+										   m_ZenCacheHost,
+										   Result.CacheHttp->GetSessionId());
+
+			if (!m_Namespace.empty())
 			{
-				std::filesystem::path StoragePath = StringToPath(m_StoragePath);
-				ZEN_CONSOLE("Querying builds in folder '{}'.", StoragePath);
-				Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
+				CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
 			}
-			else
+			if (!m_Bucket.empty())
 			{
-				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
+				CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
 			}
+			Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName;
+		}
+		ZEN_CONSOLE("Remote: {}", StorageDescription);
+		if (!Result.CacheName.empty())
+		{
+			ZEN_CONSOLE("Cache : {}", CacheDescription);
+		}
+		return Result;
+	};
 
-			CbObject					  Response = Storage->ListBuilds(QueryWriter.Save());
-			ExtendableStringBuilder<1024> SB;
-			CompactBinaryToJson(Response.GetView(), SB);
-			ZEN_CONSOLE("{}", SB.ToView());
-			return 0;
+	auto ParsePath = [&]() {
+		if (m_Path.empty())
+		{
+			throw zen::OptionParseException(fmt::format("local-path is required\n{}", SubOption->help()));
 		}
+		MakeSafeAbsolutePathÍnPlace(m_Path);
+	};
 
-		if (SubOption == &m_UploadOptions)
+	auto ParseDiffPath = [&]() {
+		if (m_DiffPath.empty())
 		{
-			ParseStorageOptions();
-			ParseAuthOptions();
+			throw zen::OptionParseException(fmt::format("compare-path is required\n{}", SubOption->help()));
+		}
+		MakeSafeAbsolutePathÍnPlace(m_DiffPath);
+	};
 
-			HttpClient Http(m_BuildsUrl, ClientSettings);
+	auto ParseBlobHash = [&]() -> IoHash {
+		m_BlobHash = RemoveQuotes(m_BlobHash);
+		if (m_BlobHash.empty())
+		{
+			throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", SubOption->help()));
+		}
+
+		IoHash BlobHash;
+		if (!IoHash::TryParse(m_BlobHash, BlobHash))
+		{
+			throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", SubOption->help()));
+		}
 
-			if (m_Path.empty())
+		return BlobHash;
+	};
+
+	auto ParseBuildId = [&]() -> Oid {
+		m_BuildId = RemoveQuotes(m_BuildId);
+		if (m_BuildId.length() != Oid::StringLength)
+		{
+			throw zen::OptionParseException(fmt::format("Invalid build id\n{}", SubOption->help()));
+		}
+		else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero)
+		{
+			throw zen::OptionParseException(fmt::format("Invalid build id\n{}", SubOption->help()));
+		}
+		else
+		{
+			return BuildId;
+		}
+	};
+
+	auto ParseBuildPartId = [&]() -> Oid {
+		m_BuildPartId = RemoveQuotes(m_BuildPartId);
+		if (m_BuildPartId.length() != Oid::StringLength)
+		{
+			throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", SubOption->help()));
+		}
+		else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero)
+		{
+			throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", SubOption->help()));
+		}
+		else
+		{
+			return BuildPartId;
+		}
+	};
+
+	auto ParseBuildPartIds = [&]() -> std::vector<Oid> {
+		std::vector<Oid> BuildPartIds;
+		for (const std::string& BuildPartId : m_BuildPartIds)
+		{
+			BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId)));
+			if (BuildPartIds.back() == Oid::Zero)
 			{
-				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help()));
+				throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, SubOption->help()));
 			}
+		}
+		return BuildPartIds;
+	};
 
-			if (m_CreateBuild)
+	auto ParseBuildPartNames = [&]() -> std::vector<std::string> {
+		std::vector<std::string> BuildPartNames;
+		for (const std::string& BuildPartName : m_BuildPartNames)
+		{
+			BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName)));
+			if (BuildPartNames.back().empty())
 			{
-				if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty())
+				throw zen::OptionParseException(fmt::format("build-part-names '{}' is invalid\n{}", BuildPartName, SubOption->help()));
+			}
+		}
+		return BuildPartNames;
+	};
+
+	auto ParseBuildMetadata = [&]() -> CbObject {
+		if (m_CreateBuild)
+		{
+			if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty())
+			{
+				throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", SubOption->help()));
+			}
+			if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty())
+			{
+				throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", SubOption->help()));
+			}
+
+			if (!m_BuildMetadataPath.empty())
+			{
+				MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath);
+				IoBuffer		 MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten();
+				std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+				std::string		 JsonError;
+				CbObject		 MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+				if (!JsonError.empty())
 				{
-					throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help()));
+					throw std::runtime_error(
+						fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError));
 				}
-				if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty())
+				return MetaData;
+			}
+			m_BuildMetadata = RemoveQuotes(m_BuildMetadata);
+			if (!m_BuildMetadata.empty())
+			{
+				CbObjectWriter MetaDataWriter(1024);
+				ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) {
+					size_t SplitPos = Pair.find('=');
+					if (SplitPos == std::string::npos || SplitPos == 0)
+					{
+						throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair));
+					}
+					MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
+					return true;
+				});
+				return MetaDataWriter.Save();
+			}
+		}
+		else
+		{
+			if (!m_BuildMetadataPath.empty())
+			{
+				throw zen::OptionParseException(
+					fmt::format("metadata-path option is only valid if creating a build\n{}", SubOption->help()));
+			}
+			if (!m_BuildMetadata.empty())
+			{
+				throw zen::OptionParseException(fmt::format("metadata option is only valid if creating a build\n{}", SubOption->help()));
+			}
+		}
+		return {};
+	};
+
+	BoostWorkerThreads = m_BoostWorkerThreads;
+	UseSparseFiles	   = m_UseSparseFiles;
+
+	try
+	{
+		if (SubOption == &m_ListNamespacesOptions)
+		{
+			if (!m_ListResultPath.empty())
+			{
+				ZEN_CONSOLE("Running {}: {} (pid {})",
+							GetRunningExecutablePath(),
+							ZEN_CFG_VERSION_BUILD_STRING_FULL,
+							GetCurrentProcessId());
+			}
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
+															? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
+															: MakeSafeAbsolutePath(m_ZenFolderPath);
+			CreateDirectories(ZenFolderPath);
+			auto _ = MakeGuard([ZenFolderPath]() {
+				if (CleanDirectory(ZenFolderPath, {}))
 				{
-					throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help()));
+					std::error_code DummyEc;
+					RemoveDir(ZenFolderPath, DummyEc);
 				}
+			});
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(ZenFolderPath),
+														 /*RequriesNamespace*/ false,
+														 /*RequireBucket*/ false);
+
+			CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive);
+			ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None);
+			if (m_ListResultPath.empty())
+			{
+				ExtendableStringBuilder<1024> SB;
+				CompactBinaryToJson(Response.GetView(), SB);
+				ZEN_CONSOLE("{}", SB.ToView());
 			}
 			else
 			{
-				if (!m_BuildMetadataPath.empty())
+				std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath);
+				if (ToLower(ListResultPath.extension().string()) == ".cbo")
 				{
-					throw zen::OptionParseException(
-						fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help()));
+					MemoryView ResponseView = Response.GetView();
+					WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
 				}
-				if (!m_BuildMetadata.empty())
+				else
 				{
-					throw zen::OptionParseException(
-						fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help()));
+					ExtendableStringBuilder<1024> SB;
+					CompactBinaryToJson(Response.GetView(), SB);
+					WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
 				}
 			}
 
-			std::filesystem::path Path = StringToPath(m_Path);
+			return 0;
+		}
 
-			if (m_BuildPartName.empty())
-			{
-				m_BuildPartName = Path.filename().string();
-			}
+		if (SubOption == &m_ListOptions)
+		{
+			MakeSafeAbsolutePathÍnPlace(m_ListQueryPath);
+			MakeSafeAbsolutePathÍnPlace(m_ListResultPath);
 
-			const bool GeneratedBuildId = m_BuildId.empty();
-			if (GeneratedBuildId)
+			if (!m_ListResultPath.empty())
 			{
-				m_BuildId = Oid::NewOid().ToString();
+				ZEN_CONSOLE("Running {}: {} (pid {})",
+							GetRunningExecutablePath(),
+							ZEN_CFG_VERSION_BUILD_STRING_FULL,
+							GetCurrentProcessId());
 			}
-			else if (m_BuildId.length() != Oid::StringLength)
+			CbObject QueryObject;
+			if (m_ListQueryPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
+				CbObjectWriter QueryWriter;
+				QueryWriter.BeginObject("query");
+				QueryWriter.EndObject();  // query
+				QueryObject = QueryWriter.Save();
 			}
-			else if (Oid::FromHexString(m_BuildId) == Oid::Zero)
+			else
 			{
-				throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
+				if (ToLower(m_ListQueryPath.extension().string()) == ".cbo")
+				{
+					QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath));
+				}
+				else
+				{
+					IoBuffer		 MetaDataJson = ReadFile(m_ListQueryPath).Flatten();
+					std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+					std::string		 JsonError;
+					QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+					if (!JsonError.empty())
+					{
+						throw std::runtime_error(
+							fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError));
+					}
+				}
 			}
 
-			const bool GeneratedBuildPartId = m_BuildPartId.empty();
-			if (GeneratedBuildPartId)
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			if (m_ZenFolderPath.empty())
 			{
-				m_BuildPartId = Oid::NewOid().ToString();
+				m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
 			}
-			else if (m_BuildPartId.length() != Oid::StringLength)
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+			CreateDirectories(m_ZenFolderPath);
+			auto _ = MakeGuard([this]() {
+				if (CleanDirectory(m_ZenFolderPath, {}))
+				{
+					std::error_code DummyEc;
+					RemoveDir(m_ZenFolderPath, DummyEc);
+				}
+			});
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(m_ZenFolderPath),
+														 /*RequriesNamespace*/ true,
+														 /*RequireBucket*/ false);
+
+			CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject);
+			ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None);
+			if (m_ListResultPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
+				ExtendableStringBuilder<1024> SB;
+				CompactBinaryToJson(Response.GetView(), SB);
+				ZEN_CONSOLE("{}", SB.ToView());
 			}
-			else if (Oid::FromHexString(m_BuildPartId) == Oid::Zero)
+			else
 			{
-				throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help()));
+				if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
+				{
+					MemoryView ResponseView = Response.GetView();
+					WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
+				}
+				else
+				{
+					ExtendableStringBuilder<1024> SB;
+					CompactBinaryToJson(Response.GetView(), SB);
+					WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+				}
 			}
 
+			return 0;
+		}
+
+		if (SubOption == &m_UploadOptions)
+		{
+			ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+
+			ZenState InstanceState;
+
+			ParsePath();
+
 			BuildStorage::Statistics	  StorageStats;
-			const Oid					  BuildId	  = Oid::FromHexString(m_BuildId);
-			const Oid					  BuildPartId = Oid::FromHexString(m_BuildPartId);
-			std::unique_ptr<BuildStorage> Storage;
-			std::string					  StorageName;
-			if (!m_BuildsUrl.empty())
-			{
-				ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'",
-							m_BuildPartName,
-							Path,
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket,
-							GeneratedBuildId ? "Generated " : "",
-							BuildId);
-				Storage		= CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
-				StorageName = "Cloud DDC";
-			}
-			else if (!m_StoragePath.empty())
-			{
-				std::filesystem::path StoragePath = StringToPath(m_StoragePath);
-				ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'",
-							m_BuildPartName,
-							Path,
-							StoragePath,
-							GeneratedBuildId ? "Generated " : "",
-							BuildId);
-				Storage = CreateFileBuildStorage(StoragePath, StorageStats, m_WriteMetadataAsJson, DefaultLatency, DefaultDelayPerKBSec);
-				StorageName = fmt::format("Disk {}", StoragePath.stem());
-			}
-			else
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			if (m_ZenFolderPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
+				m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
 			}
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
 
-			CbObject MetaData;
-			if (m_CreateBuild)
-			{
-				if (!m_BuildMetadataPath.empty())
+			CreateDirectories(m_ZenFolderPath);
+			auto _ = MakeGuard([this]() {
+				if (CleanDirectory(m_ZenFolderPath, {}))
 				{
-					std::filesystem::path MetadataPath(m_BuildMetadataPath);
-					IoBuffer			  MetaDataJson = ReadFile(MetadataPath).Flatten();
-					std::string_view	  Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
-					std::string			  JsonError;
-					MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
-					if (!JsonError.empty())
-					{
-						throw std::runtime_error(
-							fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError));
-					}
-				}
-				if (!m_BuildMetadata.empty())
-				{
-					CbObjectWriter MetaDataWriter(1024);
-					ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) {
-						size_t SplitPos = Pair.find('=');
-						if (SplitPos == std::string::npos || SplitPos == 0)
-						{
-							throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair));
-						}
-						MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
-						return true;
-					});
-					MetaData = MetaDataWriter.Save();
+					std::error_code DummyEc;
+					RemoveDir(m_ZenFolderPath, DummyEc);
 				}
+			});
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(m_ZenFolderPath),
+														 /*RequriesNamespace*/ true,
+														 /*RequireBucket*/ true);
+
+			if (m_BuildPartName.empty())
+			{
+				m_BuildPartName = m_Path.filename().string();
+			}
+
+			const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
+			if (m_BuildId.empty())
+			{
+				m_BuildId = BuildId.ToString();
+			}
+			const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId();
+			if (m_BuildPartId.empty())
+			{
+				m_BuildPartId = BuildPartId.ToString();
 			}
 
-			UploadFolder(*Storage,
+			CbObject MetaData = ParseBuildMetadata();
+
+			const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath);
+
+			m_ManifestPath = RemoveQuotes(m_ManifestPath);
+
+			UploadFolder(Storage,
 						 BuildId,
 						 BuildPartId,
 						 m_BuildPartName,
-						 Path,
+						 m_Path,
+						 TempDir,
 						 m_ManifestPath,
+						 m_FindBlockMaxCount,
 						 m_BlockReuseMinPercentLimit,
 						 m_AllowMultiparts,
 						 MetaData,
@@ -7325,7 +10962,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					"Requests:           {}\n"
 					"Avg Request Time:   {}\n"
 					"Avg I/O Time:       {}",
-					StorageName,
+					Storage.StorageName,
 					NiceBytes(StorageStats.TotalBytesRead.load()),
 					NiceBytes(StorageStats.TotalBytesWritten.load()),
 					StorageStats.TotalRequestCount.load(),
@@ -7341,185 +10978,210 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 		if (SubOption == &m_DownloadOptions)
 		{
-			ParseStorageOptions();
-			ParseAuthOptions();
+			ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
 
-			HttpClient Http(m_BuildsUrl, ClientSettings);
+			ZenState InstanceState;
 
-			if (m_Path.empty())
-			{
-				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
-			}
-			if (m_BuildId.empty())
-			{
-				throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));
-			}
-			Oid BuildId = Oid::TryFromHexString(m_BuildId);
-			if (BuildId == Oid::Zero)
+			ParsePath();
+
+			if (m_ZenFolderPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help()));
+				m_ZenFolderPath = m_Path / ZenFolderName;
 			}
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
 
-			if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(m_ZenFolderPath),
+														 /*RequriesNamespace*/ true,
+														 /*RequireBucket*/ true);
+
+			const Oid BuildId = ParseBuildId();
+
+			if (m_PostDownloadVerify && m_PrimeCacheOnly)
 			{
-				throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
+				throw zen::OptionParseException(
+					fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", SubOption->help()));
 			}
 
-			std::vector<Oid> BuildPartIds;
-			for (const std::string& BuildPartId : m_BuildPartIds)
+			if (m_Clean && m_PrimeCacheOnly)
 			{
-				BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId));
-				if (BuildPartIds.back() == Oid::Zero)
-				{
-					throw zen::OptionParseException(
-						fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help()));
-				}
+				ZEN_WARN("ignoring 'clean' option when 'cache-prime-only' is enabled");
 			}
 
-			std::filesystem::path Path = StringToPath(m_Path);
-
-			BuildStorage::Statistics	  StorageStats;
-			std::unique_ptr<BuildStorage> Storage;
-			std::string					  StorageName;
-			if (!m_BuildsUrl.empty())
-			{
-				ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'",
-							BuildId,
-							Path,
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket,
-							BuildId);
-				Storage		= CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
-				StorageName = "Cloud DDC";
-			}
-			else if (!m_StoragePath.empty())
-			{
-				std::filesystem::path StoragePath = StringToPath(m_StoragePath);
-				ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, Path, StoragePath, BuildId);
-				Storage		= CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
-				StorageName = fmt::format("Disk {}", StoragePath.stem());
-			}
-			else
+			if (m_AllowPartialBlockRequests && m_PrimeCacheOnly)
 			{
-				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
+				ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled");
 			}
 
-			DownloadFolder(*Storage,
+			std::vector<Oid>		 BuildPartIds	= ParseBuildPartIds();
+			std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+
+			DownloadFolder(Storage,
 						   BuildId,
 						   BuildPartIds,
-						   m_BuildPartNames,
-						   Path,
+						   BuildPartNames,
+						   m_Path,
+						   m_ZenFolderPath,
+						   m_SystemRootDir,
 						   m_AllowMultiparts,
-						   m_AllowPartialBlockRequests,
+						   m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
 						   m_Clean,
-						   m_PostDownloadVerify);
-
-			if (false)
-			{
-				ZEN_CONSOLE(
-					"{}:\n"
-					"Read:               {}\n"
-					"Write:              {}\n"
-					"Requests:           {}\n"
-					"Avg Request Time:   {}\n"
-					"Avg I/O Time:       {}",
-					StorageName,
-					NiceBytes(StorageStats.TotalBytesRead.load()),
-					NiceBytes(StorageStats.TotalBytesWritten.load()),
-					StorageStats.TotalRequestCount.load(),
-					StorageStats.TotalExecutionTimeUs.load() > 0
-						? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
-						: 0,
-					StorageStats.TotalRequestCount.load() > 0
-						? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
-						: 0);
-			}
+						   m_PostDownloadVerify,
+						   m_PrimeCacheOnly,
+						   m_EnableScavenging);
 
 			return AbortFlag ? 11 : 0;
 		}
 		if (SubOption == &m_DiffOptions)
 		{
-			if (m_Path.empty())
+			ParsePath();
+			ParseDiffPath();
+
+			DiffFolders(m_Path, m_DiffPath, m_OnlyChunked);
+			return AbortFlag ? 11 : 0;
+		}
+
+		if (SubOption == &m_FetchBlobOptions)
+		{
+			ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			if (m_ZenFolderPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
+				m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
 			}
-			if (m_DiffPath.empty())
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+			CreateDirectories(m_ZenFolderPath);
+			auto _ = MakeGuard([this]() {
+				if (CleanDirectory(m_ZenFolderPath, {}))
+				{
+					std::error_code DummyEc;
+					RemoveDir(m_ZenFolderPath, DummyEc);
+				}
+			});
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(m_ZenFolderPath),
+														 /*RequriesNamespace*/ true,
+														 /*RequireBucket*/ true);
+
+			IoHash BlobHash = ParseBlobHash();
+
+			const Oid BuildId = Oid::FromHexString(m_BuildId);
+
+			uint64_t CompressedSize;
+			uint64_t DecompressedSize;
+			ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
+			if (AbortFlag)
 			{
-				throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help()));
+				return 11;
 			}
-			std::filesystem::path Path = StringToPath(m_Path);
-			DiffFolders(Path, m_DiffPath, m_OnlyChunked);
-			return AbortFlag ? 11 : 0;
+			ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
+						BlobHash,
+						CompressedSize,
+						DecompressedSize);
+			return 0;
 		}
 
-		if (SubOption == &m_MultiTestDownloadOptions)
+		if (SubOption == &m_ValidateBuildPartOptions)
 		{
-			if (m_Path.empty())
+			ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+
+			ZenState InstanceState;
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			if (m_ZenFolderPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
+				m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
 			}
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
 
-			ParseStorageOptions();
-			ParseAuthOptions();
+			CreateDirectories(m_ZenFolderPath);
+			auto _ = MakeGuard([this]() {
+				if (CleanDirectory(m_ZenFolderPath, {}))
+				{
+					std::error_code DummyEc;
+					RemoveDir(m_ZenFolderPath, DummyEc);
+				}
+			});
 
-			HttpClient Http(m_BuildsUrl, ClientSettings);
-			//			m_StoragePath = "D:\\buildstorage";
-			//			m_Path		  = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient";
-			//			std::vector<std::string> BuildIdStrings{"07d3942f0e7f4ca1b13b0587",
-			//													"07d394eed89d769f2254e75d",
-			//													"07d3953f22fa3f8000fa6f0a",
-			//													"07d3959df47ed1f42ddbe44c",
-			//													"07d395fa7803d50804f14417",
-			//													"07d3964f919d577a321a1fdd",
-			//													"07d396a6ce875004e16b9528"};
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(m_ZenFolderPath),
+														 /*RequriesNamespace*/ true,
+														 /*RequireBucket*/ true);
 
-			std::filesystem::path Path = StringToPath(m_Path);
+			Oid BuildId = ParseBuildId();
 
-			BuildStorage::Statistics	  StorageStats;
-			std::unique_ptr<BuildStorage> Storage;
-			std::string					  StorageName;
-			if (!m_BuildsUrl.empty())
-			{
-				ZEN_CONSOLE("Downloading {} to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
-							FormatArray<std::string>(m_BuildIds, " "sv),
-							Path,
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket);
-				Storage		= CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
-				StorageName = "Cloud DDC";
-			}
-			else if (!m_StoragePath.empty())
-			{
-				std::filesystem::path StoragePath = StringToPath(m_StoragePath);
-				ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray<std::string>(m_BuildIds, " "sv), Path, StoragePath);
-				Storage		= CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
-				StorageName = fmt::format("Disk {}", StoragePath.stem());
+			if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
+			{
+				throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", SubOption->help()));
 			}
-			else
+
+			const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();
+
+			ValidateStatistics ValidateStats;
+			DownloadStatistics DownloadStats;
+			ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats);
+
+			return AbortFlag ? 13 : 0;
+		}
+
+		if (SubOption == &m_MultiTestDownloadOptions)
+		{
+			m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+			CreateDirectories(m_SystemRootDir);
+			CleanDirectory(m_SystemRootDir, {});
+			auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
+
+			ParsePath();
+
+			if (m_ZenFolderPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
+				m_ZenFolderPath = m_Path / ZenFolderName;
 			}
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
 
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(m_ZenFolderPath),
+														 /*RequriesNamespace*/ true,
+														 /*RequireBucket*/ true);
+
+			Stopwatch Timer;
 			for (const std::string& BuildIdString : m_BuildIds)
 			{
-				Oid BuildId = Oid::FromHexString(BuildIdString);
+				Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString));
 				if (BuildId == Oid::Zero)
 				{
-					throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help()));
+					throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, SubOption->help()));
 				}
-				DownloadFolder(*Storage,
+				DownloadFolder(Storage,
 							   BuildId,
 							   {},
 							   {},
-							   Path,
+							   m_Path,
+							   m_ZenFolderPath,
+							   m_SystemRootDir,
 							   m_AllowMultiparts,
 							   m_AllowPartialBlockRequests,
 							   BuildIdString == m_BuildIds.front(),
-							   true);
+							   true,
+							   false,
+							   m_EnableScavenging);
 				if (AbortFlag)
 				{
 					ZEN_CONSOLE("Download cancelled");
@@ -7527,76 +11189,110 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				}
 				ZEN_CONSOLE("\n");
 			}
+			ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 			return 0;
 		}
 
-		if (SubOption == &m_TestOptions)
-		{
-			ParseStorageOptions();
-			ParseAuthOptions();
-
-			HttpClient Http(m_BuildsUrl, ClientSettings);
-
-			if (m_Path.empty())
+		auto ParseZenProcessId = [&]() {
+			if (m_ZenProcessId == -1)
 			{
-				throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
+				const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath();
+				ProcessHandle				RunningProcess;
+				std::error_code				Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false);
+				if (Ec)
+				{
+					throw std::runtime_error(
+						fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message()));
+				}
+				if (!RunningProcess.IsValid())
+				{
+					throw zen::OptionParseException(
+						fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath));
+				}
+				m_ZenProcessId = RunningProcess.Pid();
 			}
+		};
 
-			std::filesystem::path Path = StringToPath(m_Path);
+		if (SubOption == &m_PauseOptions)
+		{
+			ParseZenProcessId();
+			ZenState RunningState(m_ZenProcessId);
+			RunningState.StateData().Pause.store(true);
+			return 0;
+		}
 
-			m_BuildId		= Oid::NewOid().ToString();
-			m_BuildPartName = Path.filename().string();
-			m_BuildPartId	= Oid::NewOid().ToString();
-			m_CreateBuild	= true;
+		if (SubOption == &m_ResumeOptions)
+		{
+			ParseZenProcessId();
+			ZenState RunningState(m_ZenProcessId);
+			RunningState.StateData().Pause.store(false);
+			return 0;
+		}
 
-			BuildStorage::Statistics	  StorageStats;
-			const Oid					  BuildId	  = Oid::FromHexString(m_BuildId);
-			const Oid					  BuildPartId = Oid::FromHexString(m_BuildPartId);
-			std::unique_ptr<BuildStorage> Storage;
-			std::string					  StorageName;
+		if (SubOption == &m_AbortOptions)
+		{
+			ParseZenProcessId();
+			ZenState RunningState(m_ZenProcessId);
+			RunningState.StateData().Abort.store(true);
+			return 0;
+		}
 
-			std::filesystem::path StoragePath = StringToPath(m_StoragePath);
+		if (SubOption == &m_TestOptions)
+		{
+			m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+			CreateDirectories(m_SystemRootDir);
+			CleanDirectory(m_SystemRootDir, {});
+			auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
 
-			if (m_BuildsUrl.empty() && StoragePath.empty())
+			ParsePath();
+
+			if (m_OverrideHost.empty() && m_StoragePath.empty())
 			{
-				m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").generic_string();
-				CreateDirectories(StoragePath);
-				CleanDirectory(StoragePath, {});
+				m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
+				CreateDirectories(m_StoragePath);
+				CleanDirectory(m_StoragePath, {});
+				m_StoragePath = m_StoragePath.generic_string();
 			}
-			auto _ = MakeGuard([&]() {
-				if (m_BuildsUrl.empty() && StoragePath.empty())
+
+			auto __ = MakeGuard([&]() {
+				if (m_OverrideHost.empty() && m_StoragePath.empty())
 				{
-					DeleteDirectories(StoragePath);
+					DeleteDirectories(m_StoragePath);
 				}
 			});
 
-			if (!m_BuildsUrl.empty())
-			{
-				ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'",
-							m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName,
-							Path,
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket,
-							BuildId);
-				Storage		= CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
-				StorageName = "Cloud DDC";
-			}
-			else if (!StoragePath.empty())
-			{
-				ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'",
-							m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName,
-							Path,
-							StoragePath,
-							BuildId);
-				Storage		= CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
-				StorageName = fmt::format("Disk {}", StoragePath.stem());
-			}
-			else
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			const std::filesystem::path DownloadPath  = m_Path.parent_path() / (m_BuildPartName + "_test");
+			const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
+
+			auto ___ = MakeGuard([DownloadPath, DownloadPath2]() {
+				CleanDirectory(DownloadPath, true);
+				DeleteDirectories(DownloadPath);
+				CleanDirectory(DownloadPath2, true);
+				DeleteDirectories(DownloadPath2);
+			});
+
+			if (m_ZenFolderPath.empty())
 			{
-				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
+				m_ZenFolderPath = m_Path / ZenFolderName;
 			}
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats,
+														 StorageCacheStats,
+														 ZenTempFolderPath(m_ZenFolderPath),
+														 /*RequriesNamespace*/ true,
+														 /*RequireBucket*/ true);
+
+			m_BuildId		= Oid::NewOid().ToString();
+			m_BuildPartName = m_Path.filename().string();
+			m_BuildPartId	= Oid::NewOid().ToString();
+			m_CreateBuild	= true;
+
+			const Oid BuildId	  = Oid::FromHexString(m_BuildId);
+			const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
 
 			auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
 				CbObjectWriter BuildMetaDataWriter;
@@ -7616,15 +11312,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			{
 				ExtendableStringBuilder<256> SB;
 				CompactBinaryToJson(MetaData, SB);
-				ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView());
+				ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
 			}
 
-			UploadFolder(*Storage,
+			const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
+			//			std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp";
+
+			UploadFolder(Storage,
 						 BuildId,
 						 BuildPartId,
 						 m_BuildPartName,
-						 Path,
+						 m_Path,
+						 UploadTempDir,
 						 {},
+						 m_FindBlockMaxCount,
 						 m_BlockReuseMinPercentLimit,
 						 m_AllowMultiparts,
 						 MetaData,
@@ -7637,9 +11338,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				return 11;
 			}
 
-			const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download");
 			ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
-			DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true);
+			DownloadFolder(Storage,
+						   BuildId,
+						   {BuildPartId},
+						   {},
+						   DownloadPath,
+						   DownloadPath / ZenFolderName,
+						   m_SystemRootDir,
+						   m_AllowMultiparts,
+						   m_AllowPartialBlockRequests,
+						   true,
+						   true,
+						   false,
+						   m_EnableScavenging);
 			if (AbortFlag)
 			{
 				ZEN_CONSOLE("Download failed.");
@@ -7651,7 +11363,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						BuildPartId,
 						m_BuildPartName,
 						DownloadPath);
-			DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true);
+			DownloadFolder(Storage,
+						   BuildId,
+						   {BuildPartId},
+						   {},
+						   DownloadPath,
+						   DownloadPath / ZenFolderName,
+						   m_SystemRootDir,
+						   m_AllowMultiparts,
+						   m_AllowPartialBlockRequests,
+						   false,
+						   true,
+						   false,
+						   m_EnableScavenging);
 			if (AbortFlag)
 			{
 				ZEN_CONSOLE("Re-download failed. (identical target)");
@@ -7685,7 +11409,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					return true;
 				};
 
-				ParallellWork Work(AbortFlag);
+				ParallelWork Work(AbortFlag, PauseFlag);
 
 				uint32_t Randomizer = 0;
 				auto	 FileSizeIt = DownloadContent.FileSizes.begin();
@@ -7699,39 +11423,43 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 							case 0:
 								{
 									uint64_t SourceSize = *FileSizeIt;
-									if (SourceSize > 0)
+									if (SourceSize > 256)
 									{
 										Work.ScheduleWork(
-											GetMediumWorkerPool(EWorkloadType::Burst),
-											[SourceSize, FilePath](std::atomic<bool>&) {
+											GetIOWorkerPool(),
+											[SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
 												if (!AbortFlag)
 												{
-													IoBuffer Scrambled(SourceSize);
+													bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false);
 													{
-														IoBuffer Source = IoBufferBuilder::MakeFromFile(FilePath);
-														Scrambled.GetMutableView().CopyFrom(
-															Source.GetView().Mid(SourceSize / 3, SourceSize / 3));
-														Scrambled.GetMutableView()
-															.Mid(SourceSize / 3)
-															.CopyFrom(Source.GetView().Mid(0, SourceSize / 3));
-														Scrambled.GetMutableView()
-															.Mid((SourceSize / 3) * 2)
-															.CopyFrom(Source.GetView().Mid(SourceSize / 2, SourceSize / 3));
+														BasicFile Source(FilePath, BasicFile::Mode::kWrite);
+														uint64_t  RangeSize = Min(SourceSize / 3, 512u * 1024u);
+														IoBuffer  TempBuffer1(RangeSize);
+														IoBuffer  TempBuffer2(RangeSize);
+														IoBuffer  TempBuffer3(RangeSize);
+														Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0);
+														Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2);
+														Source.Read(TempBuffer3.GetMutableView().GetData(),
+																	RangeSize,
+																	SourceSize - RangeSize);
+														Source.Write(TempBuffer1, SourceSize / 2);
+														Source.Write(TempBuffer2, SourceSize - RangeSize);
+														Source.Write(TempBuffer3, SourceSize - 0);
 													}
-													bool IsReadOnly = SetFileReadOnly(FilePath, false);
-													WriteFile(FilePath, Scrambled);
 													if (IsReadOnly)
 													{
 														SetFileReadOnly(FilePath, true);
 													}
 												}
-											},
-											Work.DefaultErrorFunction());
+											});
 									}
 								}
 								break;
 							case 1:
-								std::filesystem::remove(FilePath);
+								{
+									(void)SetFileReadOnlyWithRetry(FilePath, false);
+									RemoveFileWithRetry(FilePath);
+								}
 								break;
 							default:
 								break;
@@ -7739,8 +11467,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					}
 					FileSizeIt++;
 				}
-				Work.Wait(5000, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-					ZEN_UNUSED(IsAborted);
+				Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+					ZEN_UNUSED(IsAborted, IsPaused);
 					ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
 				});
 				ZEN_ASSERT(!AbortFlag.load());
@@ -7753,7 +11481,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						BuildPartId,
 						m_BuildPartName,
 						DownloadPath);
-			DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true);
+			DownloadFolder(Storage,
+						   BuildId,
+						   {BuildPartId},
+						   {},
+						   DownloadPath,
+						   DownloadPath / ZenFolderName,
+						   m_SystemRootDir,
+						   m_AllowMultiparts,
+						   m_AllowPartialBlockRequests,
+						   false,
+						   true,
+						   false,
+						   m_EnableScavenging);
 			if (AbortFlag)
 			{
 				ZEN_CONSOLE("Re-download failed. (scrambled target)");
@@ -7772,12 +11512,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
 			}
 
-			UploadFolder(*Storage,
+			UploadFolder(Storage,
 						 BuildId2,
 						 BuildPartId2,
 						 m_BuildPartName,
 						 DownloadPath,
+						 UploadTempDir,
 						 {},
+						 m_FindBlockMaxCount,
 						 m_BlockReuseMinPercentLimit,
 						 m_AllowMultiparts,
 						 MetaData2,
@@ -7791,7 +11533,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			}
 
 			ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
-			DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true);
+			DownloadFolder(Storage,
+						   BuildId,
+						   {BuildPartId},
+						   {},
+						   DownloadPath,
+						   DownloadPath / ZenFolderName,
+						   m_SystemRootDir,
+						   m_AllowMultiparts,
+						   m_AllowPartialBlockRequests,
+						   false,
+						   true,
+						   false,
+						   m_EnableScavenging);
 			if (AbortFlag)
 			{
 				ZEN_CONSOLE("Re-download failed.");
@@ -7799,15 +11553,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			}
 
 			ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
-			DownloadFolder(*Storage,
+			DownloadFolder(Storage,
 						   BuildId2,
 						   {BuildPartId2},
 						   {},
 						   DownloadPath,
+						   DownloadPath / ZenFolderName,
+						   m_SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   false,
-						   true);
+						   true,
+						   false,
+						   m_EnableScavenging);
 			if (AbortFlag)
 			{
 				ZEN_CONSOLE("Re-download failed.");
@@ -7815,151 +11573,64 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			}
 
 			ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
-			DownloadFolder(*Storage,
+			DownloadFolder(Storage,
 						   BuildId2,
 						   {BuildPartId2},
 						   {},
 						   DownloadPath,
+						   DownloadPath / ZenFolderName,
+						   m_SystemRootDir,
 						   m_AllowMultiparts,
 						   m_AllowPartialBlockRequests,
 						   false,
-						   true);
+						   true,
+						   false,
+						   m_EnableScavenging);
 			if (AbortFlag)
 			{
 				ZEN_CONSOLE("Re-download failed.");
 				return 11;
 			}
 
-			return 0;
-		}
-
-		if (SubOption == &m_FetchBlobOptions)
-		{
-			ParseStorageOptions();
-			ParseAuthOptions();
-
-			HttpClient Http(m_BuildsUrl, ClientSettings);
-
-			if (m_BlobHash.empty())
-			{
-				throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help()));
-			}
-
-			IoHash BlobHash;
-			if (!IoHash::TryParse(m_BlobHash, BlobHash))
-			{
-				throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help()));
-			}
-
-			if (m_BuildsUrl.empty() && m_StoragePath.empty())
-			{
-				throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help()));
-			}
-
-			BuildStorage::Statistics	  StorageStats;
-			const Oid					  BuildId = Oid::FromHexString(m_BuildId);
-			std::unique_ptr<BuildStorage> Storage;
-			std::string					  StorageName;
-
-			std::filesystem::path Path = StringToPath(m_Path);
-
-			if (!m_BuildsUrl.empty())
-			{
-				ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'",
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket,
-							BuildId);
-				Storage		= CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
-				StorageName = "Cloud DDC";
-			}
-			else if (!m_StoragePath.empty())
-			{
-				std::filesystem::path StoragePath = StringToPath(m_StoragePath);
-				ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId);
-				Storage		= CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
-				StorageName = fmt::format("Disk {}", StoragePath.stem());
-			}
-			else
-			{
-				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
-			}
-
-			uint64_t CompressedSize;
-			uint64_t DecompressedSize;
-			ValidateBlob(*Storage, BuildId, BlobHash, CompressedSize, DecompressedSize);
+			ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
+			DownloadFolder(Storage,
+						   BuildId,
+						   {BuildPartId},
+						   {},
+						   DownloadPath2,
+						   DownloadPath2 / ZenFolderName,
+						   m_SystemRootDir,
+						   m_AllowMultiparts,
+						   m_AllowPartialBlockRequests,
+						   false,
+						   true,
+						   false,
+						   m_EnableScavenging);
 			if (AbortFlag)
 			{
+				ZEN_CONSOLE("Re-download failed.");
 				return 11;
 			}
-			ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
-						BlobHash,
-						CompressedSize,
-						DecompressedSize);
+
 			return 0;
 		}
-
-		if (SubOption == &m_ValidateBuildPartOptions)
+	}
+	catch (const std::system_error& SysErr)
+	{
+		if (IsOOD(SysErr))
 		{
-			ParseStorageOptions();
-			ParseAuthOptions();
-
-			HttpClient Http(m_BuildsUrl, ClientSettings);
-
-			if (m_BuildsUrl.empty() && m_StoragePath.empty())
-			{
-				throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help()));
-			}
-
-			if (m_BuildId.empty())
-			{
-				throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));
-			}
-			Oid BuildId = Oid::TryFromHexString(m_BuildId);
-			if (BuildId == Oid::Zero)
-			{
-				throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help()));
-			}
-
-			if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
-			{
-				throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
-			}
-
-			BuildStorage::Statistics	  StorageStats;
-			std::unique_ptr<BuildStorage> Storage;
-			std::string					  StorageName;
-
-			std::filesystem::path Path = StringToPath(m_Path);
-
-			if (!m_BuildsUrl.empty())
-			{
-				ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'",
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket,
-							BuildId);
-				Storage		= CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
-				StorageName = "Cloud DDC";
-			}
-			else if (!m_StoragePath.empty())
-			{
-				std::filesystem::path StoragePath = StringToPath(m_StoragePath);
-				ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId);
-				Storage		= CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
-				StorageName = fmt::format("Disk {}", StoragePath.stem());
-			}
-			else
-			{
-				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
-			}
-			Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId);
-
-			ValidateBuildPart(*Storage, BuildId, BuildPartId, m_BuildPartName);
-
-			return AbortFlag ? 13 : 0;
+			ZEN_CONSOLE("Operation failed due to out of disk space: {}", SysErr.what());
+			return 3;
+		}
+		else if (IsOOM(SysErr))
+		{
+			ZEN_CONSOLE("Operation failed due to out of memory: {}", SysErr.what());
+			return 3;
+		}
+		else
+		{
+			ZEN_ERROR("{}", SysErr.what());
+			return 3;
 		}
 	}
 	catch (const std::exception& Ex)
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 60953efad..b3466c0d3 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -25,37 +25,49 @@ public:
 private:
 	cxxopts::Options m_Options{Name, Description};
 
-	std::string m_SystemRootDir;
+	std::filesystem::path m_SystemRootDir;
 
-	bool m_PlainProgress = false;
-	bool m_Verbose		 = false;
+	bool m_PlainProgress	  = false;
+	bool m_LogProgress		  = false;
+	bool m_Verbose			  = false;
+	bool m_BoostWorkerThreads = false;
+	bool m_UseSparseFiles	  = true;
+
+	std::filesystem::path m_ZenFolderPath;
 
 	// cloud builds
-	std::string m_BuildsUrl;
-	bool		m_AssumeHttp2 = false;
+	std::string m_OverrideHost;
+	std::string m_Host;
+	std::string m_Url;
+	bool		m_AssumeHttp2	= false;
+	bool		m_AllowRedirect = false;
 	std::string m_Namespace;
 	std::string m_Bucket;
 
 	// file storage
-	std::string m_StoragePath;
-	bool		m_WriteMetadataAsJson = false;
-
-	std::string m_BuildId;
-	bool		m_CreateBuild = false;
-	std::string m_BuildMetadataPath;
-	std::string m_BuildMetadata;
-	std::string m_BuildPartName;  // Defaults to name of leaf folder in m_Path
-	std::string m_BuildPartId;	  // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName
-	bool		m_Clean						= false;
-	uint8_t		m_BlockReuseMinPercentLimit = 85;
-	bool		m_AllowMultiparts			= true;
-	bool		m_AllowPartialBlockRequests = true;
-	std::string m_ManifestPath;
+	std::filesystem::path m_StoragePath;
+	bool				  m_WriteMetadataAsJson = false;
+
+	// cache
+	std::string m_ZenCacheHost;
+	bool		m_PrimeCacheOnly = false;
+
+	std::string			  m_BuildId;
+	bool				  m_CreateBuild = false;
+	std::filesystem::path m_BuildMetadataPath;
+	std::string			  m_BuildMetadata;
+	std::string			  m_BuildPartName;	// Defaults to name of leaf folder in m_Path
+	std::string			  m_BuildPartId;  // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName
+	bool				  m_Clean					  = false;
+	uint8_t				  m_BlockReuseMinPercentLimit = 85;
+	bool				  m_AllowMultiparts			  = true;
+	bool				  m_AllowPartialBlockRequests = true;
+	std::string			  m_ManifestPath;  // Not a std::filesystem::path since it can be relative to m_Path
 
 	// Direct access token (may expire)
-	std::string m_AccessToken;
-	std::string m_AccessTokenEnv;
-	std::string m_AccessTokenPath;
+	std::string			  m_AccessToken;
+	std::string			  m_AccessTokenEnv;
+	std::filesystem::path m_AccessTokenPath;
 
 	// Auth manager token encryption
 	std::string m_EncryptionKey;  // 256 bit AES encryption key
@@ -72,42 +84,61 @@ private:
 	std::string m_OAuthClientId;
 	std::string m_OAuthClientSecret;
 
+	std::string m_OidcTokenAuthExecutablePath;
+
 	std::string m_Verb;	 // list, upload, download
 
-	cxxopts::Options m_ListOptions{"list", "List available builds"};
+	cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"};
+	bool			 m_ListNamespacesRecursive = false;
 
-	std::string m_Path;
+	cxxopts::Options	  m_ListOptions{"list", "List available builds"};
+	std::filesystem::path m_ListQueryPath;
+	std::filesystem::path m_ListResultPath;
+
+	std::filesystem::path m_Path;
 
 	cxxopts::Options m_UploadOptions{"upload", "Upload a folder"};
-	bool			 m_PostUploadVerify = false;
+	uint64_t		 m_FindBlockMaxCount = 10000;
+	bool			 m_PostUploadVerify	 = false;
 
 	cxxopts::Options		 m_DownloadOptions{"download", "Download a folder"};
 	std::vector<std::string> m_BuildPartNames;
 	std::vector<std::string> m_BuildPartIds;
 	bool					 m_PostDownloadVerify = false;
+	bool					 m_EnableScavenging	  = true;
 
-	cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"};
-	std::string		 m_DiffPath;
-	bool			 m_OnlyChunked = false;
-
-	cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"};
-
-	cxxopts::Options		 m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
-	std::vector<std::string> m_BuildIds;
+	cxxopts::Options	  m_DiffOptions{"diff", "Compare two local folders"};
+	std::filesystem::path m_DiffPath;
+	bool				  m_OnlyChunked = false;
 
 	cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"};
 	std::string		 m_BlobHash;
 
+	cxxopts::Options m_PauseOptions{"pause", "Pause an ongoing zen builds process"};
+	cxxopts::Options m_ResumeOptions{"resume", "Resume a paused zen builds process"};
+	cxxopts::Options m_AbortOptions{"abort", "Abort an ongoing zen builds process"};
+
+	int m_ZenProcessId = -1;
+
 	cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"};
 
-	cxxopts::Options* m_SubCommands[8] = {&m_ListOptions,
-										  &m_UploadOptions,
-										  &m_DownloadOptions,
-										  &m_DiffOptions,
-										  &m_TestOptions,
-										  &m_FetchBlobOptions,
-										  &m_ValidateBuildPartOptions,
-										  &m_MultiTestDownloadOptions};
+	cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"};
+
+	cxxopts::Options		 m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
+	std::vector<std::string> m_BuildIds;
+
+	cxxopts::Options* m_SubCommands[12] = {&m_ListNamespacesOptions,
+										   &m_ListOptions,
+										   &m_UploadOptions,
+										   &m_DownloadOptions,
+										   &m_PauseOptions,
+										   &m_ResumeOptions,
+										   &m_AbortOptions,
+										   &m_DiffOptions,
+										   &m_FetchBlobOptions,
+										   &m_ValidateBuildPartOptions,
+										   &m_TestOptions,
+										   &m_MultiTestDownloadOptions};
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp
index 6ec6a80db..4412eaf34 100644
--- a/src/zen/cmds/cache_cmd.cpp
+++ b/src/zen/cmds/cache_cmd.cpp
@@ -625,22 +625,20 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	HttpClient Http(m_HostName);
 
-	std::filesystem::path TargetPath;
 	if (!m_OutputPath.empty())
 	{
-		TargetPath = std::filesystem::path(m_OutputPath);
-		if (std::filesystem::is_directory(TargetPath))
+		if (IsDir(m_OutputPath))
 		{
-			TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
+			m_OutputPath = m_OutputPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
 		}
 		else
 		{
-			CreateDirectories(TargetPath.parent_path());
+			CreateDirectories(m_OutputPath.parent_path());
 		}
 	}
-	if (TargetPath.empty())
+	if (m_OutputPath.empty())
 	{
-		TargetPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
+		m_OutputPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
 	}
 
 	std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey);
@@ -670,17 +668,17 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			}
 			else
 			{
-				WriteFile(TargetPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length()));
-				ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), TargetPath, ToString(ChunkData.GetContentType()));
+				WriteFile(m_OutputPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length()));
+				ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), m_OutputPath, ToString(ChunkData.GetContentType()));
 			}
 		}
 		else
 		{
-			if (!MoveToFile(TargetPath, ChunkData))
+			if (!MoveToFile(m_OutputPath, ChunkData))
 			{
-				WriteFile(TargetPath, ChunkData);
+				WriteFile(m_OutputPath, ChunkData);
 			}
-			ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), TargetPath, ToString(ChunkData.GetContentType()));
+			ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), m_OutputPath, ToString(ChunkData.GetContentType()));
 		}
 	}
 	else
diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h
index 73702cada..b8a319359 100644
--- a/src/zen/cmds/cache_cmd.h
+++ b/src/zen/cmds/cache_cmd.h
@@ -108,15 +108,15 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{Name, Description};
-	std::string		 m_HostName;
-	std::string		 m_Namespace;
-	std::string		 m_Bucket;
-	std::string		 m_ValueKey;
-	std::string		 m_AttachmentHash;
-	std::string		 m_OutputPath;
-	bool			 m_AsText	  = false;
-	bool			 m_Decompress = true;
+	cxxopts::Options	  m_Options{Name, Description};
+	std::string			  m_HostName;
+	std::string			  m_Namespace;
+	std::string			  m_Bucket;
+	std::string			  m_ValueKey;
+	std::string			  m_AttachmentHash;
+	std::filesystem::path m_OutputPath;
+	bool				  m_AsText	   = false;
+	bool				  m_Decompress = true;
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp
index cc6ddd505..e86b6964c 100644
--- a/src/zen/cmds/copy_cmd.cpp
+++ b/src/zen/cmds/copy_cmd.cpp
@@ -42,11 +42,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	if (m_CopyTarget.empty())
 		throw std::runtime_error("No target specified");
 
-	std::filesystem::path FromPath;
-	std::filesystem::path ToPath;
-
-	FromPath = m_CopySource;
-	ToPath	 = m_CopyTarget;
+	std::filesystem::path FromPath = m_CopySource;
+	std::filesystem::path ToPath   = m_CopyTarget;
 
 	std::error_code		  Ec;
 	std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec);
@@ -64,8 +61,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 	}
 
-	const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource);
-	const bool IsDirCopy  = std::filesystem::is_directory(m_CopySource);
+	const bool IsFileCopy = IsFile(m_CopySource);
+	const bool IsDirCopy  = IsDir(m_CopySource);
 
 	if (!IsFileCopy && !IsDirCopy)
 	{
@@ -79,20 +76,14 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	if (IsDirCopy)
 	{
-		if (std::filesystem::exists(ToPath))
+		if (IsFile(ToPath))
 		{
-			const bool IsTargetDir = std::filesystem::is_directory(ToPath);
-			if (!IsTargetDir)
-			{
-				if (std::filesystem::is_regular_file(ToPath))
-				{
-					throw std::runtime_error("Attempted copy of directory into file");
-				}
-			}
+			throw std::runtime_error("Attempted copy of directory into file");
 		}
-		else
+
+		if (!IsDir(ToPath))
 		{
-			std::filesystem::create_directories(ToPath);
+			CreateDirectories(ToPath);
 		}
 
 		std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h
index 876aff3f5..e9735c159 100644
--- a/src/zen/cmds/copy_cmd.h
+++ b/src/zen/cmds/copy_cmd.h
@@ -19,11 +19,11 @@ public:
 	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
 
 private:
-	cxxopts::Options m_Options{"copy", "Copy files efficiently"};
-	std::string		 m_CopySource;
-	std::string		 m_CopyTarget;
-	bool			 m_NoClone	 = false;
-	bool			 m_MustClone = false;
+	cxxopts::Options	  m_Options{"copy", "Copy files efficiently"};
+	std::filesystem::path m_CopySource;
+	std::filesystem::path m_CopyTarget;
+	bool				  m_NoClone	  = false;
+	bool				  m_MustClone = false;
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h
index c4f0068e4..2721be2b9 100644
--- a/src/zen/cmds/dedup_cmd.h
+++ b/src/zen/cmds/dedup_cmd.h
@@ -21,8 +21,8 @@ public:
 private:
 	cxxopts::Options		 m_Options{"dedup", "Deduplicate files"};
 	std::vector<std::string> m_Positional;
-	std::string				 m_DedupSource;
-	std::string				 m_DedupTarget;
+	std::filesystem::path	 m_DedupSource;
+	std::filesystem::path	 m_DedupTarget;
 	size_t					 m_SizeThreshold = 1024 * 1024;
 };
 
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 6bc499f03..58af0577e 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -41,7 +41,7 @@ namespace {
 
 	std::string ReadJupiterAccessTokenFromFile(const std::filesystem::path& Path)
 	{
-		if (!std::filesystem::is_regular_file(Path))
+		if (!IsFile(Path))
 		{
 			throw std::runtime_error(fmt::format("the file '{}' does not exist", Path));
 		}
@@ -61,6 +61,75 @@ namespace {
 		return AuthToken;
 	}
 
+	std::filesystem::path FindOidcTokenExePath(std::string_view OidcTokenAuthExecutablePath)
+	{
+		if (OidcTokenAuthExecutablePath.empty())
+		{
+			const std::string	  OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL;
+			std::filesystem::path OidcTokenPath		 = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred();
+			if (IsFile(OidcTokenPath))
+			{
+				return OidcTokenPath;
+			}
+			OidcTokenPath = (std::filesystem::current_path() / OidcExecutableName).make_preferred();
+			if (IsFile(OidcTokenPath))
+			{
+				return OidcTokenPath;
+			}
+		}
+		else
+		{
+			std::filesystem::path OidcTokenPath = std::filesystem::absolute(StringToPath(OidcTokenAuthExecutablePath)).make_preferred();
+			if (IsFile(OidcTokenPath))
+			{
+				return OidcTokenPath;
+			}
+		}
+		return {};
+	};
+
+	void WriteAuthOptions(CbObjectWriter&  Writer,
+						  std::string_view JupiterOpenIdProvider,
+						  std::string_view JupiterAccessToken,
+						  std::string_view JupiterAccessTokenEnv,
+						  std::string_view JupiterAccessTokenPath,
+						  std::string_view OidcTokenAuthExecutablePath)
+	{
+		if (!JupiterOpenIdProvider.empty())
+		{
+			Writer.AddString("openid-provider"sv, JupiterOpenIdProvider);
+		}
+		if (!JupiterAccessToken.empty())
+		{
+			Writer.AddString("access-token"sv, JupiterAccessToken);
+		}
+		if (!JupiterAccessTokenPath.empty())
+		{
+			std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(JupiterAccessTokenPath);
+			if (!ResolvedCloudAccessToken.empty())
+			{
+				Writer.AddString("access-token"sv, ResolvedCloudAccessToken);
+			}
+		}
+		if (!JupiterAccessTokenEnv.empty())
+		{
+			std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(JupiterAccessTokenEnv);
+
+			if (!ResolvedCloudAccessTokenEnv.empty())
+			{
+				Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv);
+			}
+			else
+			{
+				Writer.AddString("access-token-env"sv, JupiterAccessTokenEnv);
+			}
+		}
+		if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(OidcTokenAuthExecutablePath); !OidcTokenExePath.empty())
+		{
+			Writer.AddString("oidc-exe-path"sv, OidcTokenExePath.generic_string());
+		}
+	}
+
 	IoBuffer MakeCbObjectPayload(std::function<void(CbObjectWriter& Writer)> WriteCB)
 	{
 		CbObjectWriter Writer;
@@ -99,7 +168,7 @@ namespace {
 					throw std::runtime_error(fmt::format("invalid job id returned, received '{}'", JobIdText));
 				}
 
-				ProgressBar ProgressBar(PlainProgress);
+				ProgressBar ProgressBar(PlainProgress ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty, ""sv);
 
 				auto OuputMessages = [&](CbObjectView StatusObject) {
 					CbArrayView Messages = StatusObject["Messages"sv].AsArrayView();
@@ -863,6 +932,12 @@ ExportOplogCommand::ExportOplogCommand()
 						 "<filepath>");
 	m_Options.add_option("",
 						 "",
+						 "oidctoken-exe-path",
+						 "Path to OidcToken executable",
+						 cxxopts::value<std::string>(m_OidcTokenAuthExecutablePath)->default_value(""),
+						 "");
+	m_Options.add_option("",
+						 "",
 						 "assume-http2",
 						 "Assume that the cloud/builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake",
 						 cxxopts::value(m_JupiterAssumeHttp2),
@@ -1160,35 +1235,12 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					{
 						Writer.AddString("basekey"sv, m_BaseCloudKey);
 					}
-					if (!m_JupiterOpenIdProvider.empty())
-					{
-						Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider);
-					}
-					if (!m_JupiterAccessToken.empty())
-					{
-						Writer.AddString("access-token"sv, m_JupiterAccessToken);
-					}
-					if (!m_JupiterAccessTokenPath.empty())
-					{
-						std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath);
-						if (!ResolvedCloudAccessToken.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessToken);
-						}
-					}
-					if (!m_JupiterAccessTokenEnv.empty())
-					{
-						std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv);
-
-						if (!ResolvedCloudAccessTokenEnv.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv);
-						}
-						else
-						{
-							Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv);
-						}
-					}
+					WriteAuthOptions(Writer,
+									 m_JupiterOpenIdProvider,
+									 m_JupiterAccessToken,
+									 m_JupiterAccessTokenEnv,
+									 m_JupiterAccessTokenPath,
+									 m_OidcTokenAuthExecutablePath);
 					if (m_JupiterAssumeHttp2)
 					{
 						Writer.AddBool("assumehttp2"sv, true);
@@ -1219,35 +1271,12 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					Writer.AddString("namespace"sv, m_JupiterNamespace);
 					Writer.AddString("bucket"sv, m_JupiterBucket);
 					Writer.AddString("buildsid"sv, m_BuildsId);
-					if (!m_JupiterOpenIdProvider.empty())
-					{
-						Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider);
-					}
-					if (!m_JupiterAccessToken.empty())
-					{
-						Writer.AddString("access-token"sv, m_JupiterAccessToken);
-					}
-					if (!m_JupiterAccessTokenPath.empty())
-					{
-						std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath);
-						if (!ResolvedCloudAccessToken.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessToken);
-						}
-					}
-					if (!m_JupiterAccessTokenEnv.empty())
-					{
-						std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv);
-
-						if (!ResolvedCloudAccessTokenEnv.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv);
-						}
-						else
-						{
-							Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv);
-						}
-					}
+					WriteAuthOptions(Writer,
+									 m_JupiterOpenIdProvider,
+									 m_JupiterAccessToken,
+									 m_JupiterAccessTokenEnv,
+									 m_JupiterAccessTokenPath,
+									 m_OidcTokenAuthExecutablePath);
 					if (m_JupiterAssumeHttp2)
 					{
 						Writer.AddBool("assumehttp2"sv, true);
@@ -1383,6 +1412,12 @@ ImportOplogCommand::ImportOplogCommand()
 						 "<filepath>");
 	m_Options.add_option("",
 						 "",
+						 "oidctoken-exe-path",
+						 "Path to OidcToken executable",
+						 cxxopts::value<std::string>(m_OidcTokenAuthExecutablePath)->default_value(""),
+						 "");
+	m_Options.add_option("",
+						 "",
 						 "assume-http2",
 						 "Assume that the cloud/builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake",
 						 cxxopts::value(m_JupiterAssumeHttp2),
@@ -1449,12 +1484,6 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 		return 1;
 	}
 
-	m_OplogName = ResolveOplog(Http, m_ProjectName, m_OplogName);
-	if (m_OplogName.empty())
-	{
-		return 1;
-	}
-
 	size_t TargetCount = 0;
 	TargetCount += m_CloudUrl.empty() ? 0 : 1;
 	TargetCount += m_BuildsUrl.empty() ? 0 : 1;
@@ -1586,35 +1615,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					Writer.AddString("namespace"sv, m_JupiterNamespace);
 					Writer.AddString("bucket"sv, m_JupiterBucket);
 					Writer.AddString("key"sv, m_CloudKey);
-					if (!m_JupiterOpenIdProvider.empty())
-					{
-						Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider);
-					}
-					if (!m_JupiterAccessToken.empty())
-					{
-						Writer.AddString("access-token"sv, m_JupiterAccessToken);
-					}
-					if (!m_JupiterAccessTokenPath.empty())
-					{
-						std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath);
-						if (!ResolvedCloudAccessToken.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessToken);
-						}
-					}
-					if (!m_JupiterAccessTokenEnv.empty())
-					{
-						std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv);
-
-						if (!ResolvedCloudAccessTokenEnv.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv);
-						}
-						else
-						{
-							Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv);
-						}
-					}
+					WriteAuthOptions(Writer,
+									 m_JupiterOpenIdProvider,
+									 m_JupiterAccessToken,
+									 m_JupiterAccessTokenEnv,
+									 m_JupiterAccessTokenPath,
+									 m_OidcTokenAuthExecutablePath);
 					if (m_JupiterAssumeHttp2)
 					{
 						Writer.AddBool("assumehttp2"sv, true);
@@ -1631,35 +1637,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					Writer.AddString("namespace"sv, m_JupiterNamespace);
 					Writer.AddString("bucket"sv, m_JupiterBucket);
 					Writer.AddString("buildsid"sv, m_BuildsId);
-					if (!m_JupiterOpenIdProvider.empty())
-					{
-						Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider);
-					}
-					if (!m_JupiterAccessToken.empty())
-					{
-						Writer.AddString("access-token"sv, m_JupiterAccessToken);
-					}
-					if (!m_JupiterAccessTokenPath.empty())
-					{
-						std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath);
-						if (!ResolvedCloudAccessToken.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessToken);
-						}
-					}
-					if (!m_JupiterAccessTokenEnv.empty())
-					{
-						std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv);
-
-						if (!ResolvedCloudAccessTokenEnv.empty())
-						{
-							Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv);
-						}
-						else
-						{
-							Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv);
-						}
-					}
+					WriteAuthOptions(Writer,
+									 m_JupiterOpenIdProvider,
+									 m_JupiterAccessToken,
+									 m_JupiterAccessTokenEnv,
+									 m_JupiterAccessTokenPath,
+									 m_OidcTokenAuthExecutablePath);
 					if (m_JupiterAssumeHttp2)
 					{
 						Writer.AddBool("assumehttp2"sv, true);
@@ -2028,6 +2011,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 
 	std::filesystem::path TmpPath = RootPath / ".tmp";
 	CreateDirectories(TmpPath);
+	auto _ = MakeGuard([&TmpPath]() { DeleteDirectories(TmpPath); });
 
 	std::atomic_int64_t FileCount		= 0;
 	int					OplogEntryCount = 0;
@@ -2040,6 +2024,8 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 	std::unordered_set<std::u8string> FileNames;
 	std::atomic<uint64_t>			  WrittenByteCount = 0;
 
+	std::atomic<bool> AbortFlag(false);
+
 	Stopwatch WriteStopWatch;
 
 	auto EmitFilesForDataArray = [&](CbArrayView DataArray) {
@@ -2068,38 +2054,50 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				EmitCount++;
 				WorkRemaining.AddCount(1);
 				WorkerPool.ScheduleWork(
-					[this, &RootPath, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining, &WrittenByteCount]() {
+					[this, &RootPath, &AbortFlag, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining, &WrittenByteCount]() {
 						auto _ = MakeGuard([&WorkRemaining]() { WorkRemaining.CountDown(); });
-						if (HttpClient::Response ChunkResponse =
-								Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath))
+						if (!AbortFlag)
 						{
-							auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer {
-								IoHash	 RawHash;
-								uint64_t RawSize;
-								if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize))
-								{
-									return Compressed.Decompress().AsIoBuffer();
-								};
-								return std::move(Buffer);
-							};
-
-							IoBuffer ChunkData =
-								m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload;
-
 							std::filesystem::path TargetPath = RootPath / FileName;
-							if (!MoveToFile(TargetPath, ChunkData))
+							try
 							{
-								WriteFile(TargetPath, ChunkData);
+								if (HttpClient::Response ChunkResponse =
+										Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath))
+								{
+									auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer {
+										IoHash	 RawHash;
+										uint64_t RawSize;
+										if (CompressedBuffer Compressed =
+												CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize))
+										{
+											return Compressed.Decompress().AsIoBuffer();
+										};
+										return std::move(Buffer);
+									};
+
+									IoBuffer ChunkData =
+										m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload;
+
+									if (!MoveToFile(TargetPath, ChunkData))
+									{
+										WriteFile(TargetPath, ChunkData);
+									}
+									WrittenByteCount.fetch_add(ChunkData.GetSize());
+									++FileCount;
+								}
+								else
+								{
+									throw std::runtime_error(fmt::format("Unable to fetch '{}' (chunk {}). Reason: '{}'",
+																		 FileName,
+																		 ChunkId,
+																		 ChunkResponse.ErrorMessage(""sv)));
+								}
+							}
+							catch (const std::exception& Ex)
+							{
+								AbortFlag.store(true);
+								ZEN_CONSOLE("Failed writing file to '{}'. Reason: '{}'", TargetPath, Ex.what());
 							}
-							WrittenByteCount.fetch_add(ChunkData.GetSize());
-							++FileCount;
-						}
-						else
-						{
-							ZEN_CONSOLE("Unable to fetch '{}' (chunk {}). Reason: '{}'",
-										FileName,
-										ChunkId,
-										ChunkResponse.ErrorMessage(""sv));
 						}
 					});
 			}
@@ -2114,33 +2112,36 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 		{
 			std::unique_ptr<ProgressBar> EmitProgressBar;
 			{
-				ProgressBar ParseProgressBar(false);
+				ProgressBar ParseProgressBar(ProgressBar::Mode::Pretty, "");
 				CbArrayView Entries	  = ResponseObject["entries"sv].AsArrayView();
 				uint64_t	Remaining = Entries.Num();
 				for (auto EntryIter : Entries)
 				{
-					CbObjectView Entry = EntryIter.AsObjectView();
-					ParseProgressBar.UpdateState(
-						{.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining},
-						false);
-					Remaining--;
-					if (!m_KeyFilter.empty())
+					if (!AbortFlag)
 					{
-						if (Entry["key"].AsString().find(m_KeyFilter) == std::string_view::npos)
+						CbObjectView Entry = EntryIter.AsObjectView();
+						ParseProgressBar.UpdateState(
+							{.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining},
+							false);
+						Remaining--;
+						if (!m_KeyFilter.empty())
 						{
-							continue;
+							if (Entry["key"].AsString().find(m_KeyFilter) == std::string_view::npos)
+							{
+								continue;
+							}
+						}
+						if (!EmitProgressBar)
+						{
+							EmitProgressBar = std::make_unique<ProgressBar>(ProgressBar::Mode::Pretty, ""sv);
+							WriteStopWatch.Reset();
 						}
-					}
-					if (!EmitProgressBar)
-					{
-						EmitProgressBar = std::make_unique<ProgressBar>(false);
-						WriteStopWatch.Reset();
-					}
 
-					EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView());
-					EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView());
+						EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView());
+						EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView());
 
-					++OplogEntryCount;
+						++OplogEntryCount;
+					}
 				}
 				ParseProgressBar.Finish();
 			}
@@ -2172,6 +2173,12 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 			{
 				WorkRemaining.Wait();
 			}
+
+			if (AbortFlag)
+			{
+				// Error has already been reported by async code
+				return 1;
+			}
 		}
 		else
 		{
@@ -2185,8 +2192,6 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 		return 1;
 	}
 
-	std::filesystem::remove_all(TmpPath);
-
 	ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount);
 
 	return 0;
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index e66e98414..0d24d8529 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -109,6 +109,7 @@ private:
 	std::string m_JupiterAccessToken;
 	std::string m_JupiterAccessTokenEnv;
 	std::string m_JupiterAccessTokenPath;
+	std::string m_OidcTokenAuthExecutablePath;
 	bool		m_JupiterAssumeHttp2	   = false;
 	bool		m_JupiterDisableTempBlocks = false;
 
@@ -165,6 +166,7 @@ private:
 	std::string m_JupiterAccessToken;
 	std::string m_JupiterAccessTokenEnv;
 	std::string m_JupiterAccessTokenPath;
+	std::string m_OidcTokenAuthExecutablePath;
 	bool		m_JupiterAssumeHttp2 = false;
 
 	std::string m_CloudUrl;
diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp
index 5b88a1f73..4fc38d92a 100644
--- a/src/zen/cmds/rpcreplay_cmd.cpp
+++ b/src/zen/cmds/rpcreplay_cmd.cpp
@@ -196,7 +196,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		throw zen::OptionParseException("Rpc replay command requires a path");
 	}
 
-	if (!std::filesystem::exists(m_RecordingPath) || !std::filesystem::is_directory(m_RecordingPath))
+	if (!IsDir(m_RecordingPath))
 	{
 		throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath));
 	}
diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp
index a99ba9704..309b8996a 100644
--- a/src/zen/cmds/run_cmd.cpp
+++ b/src/zen/cmds/run_cmd.cpp
@@ -100,7 +100,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 		else
 		{
-			CleanDirectory(BaseDirectory);
+			CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false);
 		}
 	}
 
diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp
index f87725e36..64039e4c9 100644
--- a/src/zen/cmds/serve_cmd.cpp
+++ b/src/zen/cmds/serve_cmd.cpp
@@ -67,7 +67,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		throw zen::OptionParseException("command requires a root path");
 	}
 
-	if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath))
+	if (!IsDir(m_RootPath))
 	{
 		throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath));
 	}
diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp
index 4d1534e05..2b507e43d 100644
--- a/src/zen/cmds/status_cmd.cpp
+++ b/src/zen/cmds/status_cmd.cpp
@@ -32,17 +32,16 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	uint16_t EffectivePort = 0;
 	if (!m_DataDir.empty())
 	{
-		std::filesystem::path DataDir = StringToPath(m_DataDir);
-		if (!std::filesystem::is_regular_file(DataDir / ".lock"))
+		if (!IsFile(m_DataDir / ".lock"))
 		{
-			ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir);
+			ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
 			return 1;
 		}
-		LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock")));
+		LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
 		std::string	 Reason;
 		if (!ValidateLockFileInfo(Info, Reason))
 		{
-			ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason);
+			ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
 			return 1;
 		}
 		EffectivePort = Info.EffectiveListenPort;
diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h
index 00ad0e758..46bda9ee6 100644
--- a/src/zen/cmds/status_cmd.h
+++ b/src/zen/cmds/status_cmd.h
@@ -20,9 +20,9 @@ public:
 private:
 	int GetLockFileEffectivePort() const;
 
-	cxxopts::Options m_Options{"status", "Show zen status"};
-	uint16_t		 m_Port = 0;
-	std::string		 m_DataDir;
+	cxxopts::Options	  m_Options{"status", "Show zen status"};
+	uint16_t			  m_Port = 0;
+	std::filesystem::path m_DataDir;
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index 44a41146c..d0763701e 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -77,15 +77,13 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 	}
 
-	std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir);
-
-	if (ProgramBaseDir.empty())
+	if (m_ProgramBaseDir.empty())
 	{
 		std::filesystem::path ExePath = zen::GetRunningExecutablePath();
-		ProgramBaseDir				  = ExePath.parent_path();
+		m_ProgramBaseDir			  = ExePath.parent_path();
 	}
 	ZenServerEnvironment ServerEnvironment;
-	ServerEnvironment.Initialize(ProgramBaseDir);
+	ServerEnvironment.Initialize(m_ProgramBaseDir);
 	ZenServerInstance Server(ServerEnvironment);
 	std::string		  ServerArguments = GlobalOptions.PassthroughCommandLine;
 	if ((m_Port != 0) && (ServerArguments.find("--port"sv) == std::string::npos))
@@ -155,20 +153,18 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	Instance.Sweep();
 	ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port);
 
-	std::filesystem::path DataDir = StringToPath(m_DataDir);
-
-	if (!DataDir.empty())
+	if (!m_DataDir.empty())
 	{
-		if (!std::filesystem::is_regular_file(DataDir / ".lock"))
+		if (!IsFile(m_DataDir / ".lock"))
 		{
-			ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir);
+			ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
 			return 1;
 		}
-		LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock")));
+		LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
 		std::string	 Reason;
 		if (!ValidateLockFileInfo(Info, Reason))
 		{
-			ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason);
+			ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
 			return 1;
 		}
 		Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
@@ -218,27 +214,24 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	Instance.Initialize();
 	ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port);
 
-	std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir);
-	if (ProgramBaseDir.empty())
+	if (m_ProgramBaseDir.empty())
 	{
 		std::filesystem::path ExePath = GetRunningExecutablePath();
-		ProgramBaseDir				  = ExePath.parent_path();
+		m_ProgramBaseDir			  = ExePath.parent_path();
 	}
 
-	std::filesystem::path DataDir = StringToPath(m_DataDir);
-
-	if (!DataDir.empty())
+	if (!m_DataDir.empty())
 	{
-		if (!std::filesystem::is_regular_file(DataDir / ".lock"))
+		if (!IsFile(m_DataDir / ".lock"))
 		{
-			ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir);
+			ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
 			return 1;
 		}
-		LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock")));
+		LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
 		std::string	 Reason;
 		if (!ValidateLockFileInfo(Info, Reason))
 		{
-			ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason);
+			ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
 			return 1;
 		}
 		Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
@@ -251,7 +244,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		try
 		{
 			ZenServerEnvironment ServerEnvironment;
-			ServerEnvironment.Initialize(ProgramBaseDir);
+			ServerEnvironment.Initialize(m_ProgramBaseDir);
 			ZenServerInstance Server(ServerEnvironment);
 			Server.AttachToRunningServer(EntryPort);
 
@@ -316,9 +309,9 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	if (m_ForceTerminate)
 	{
 		// Try to find the running executable by path name
-		std::filesystem::path ServerExePath = ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
+		std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
 		ProcessHandle		  RunningProcess;
-		if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec)
+		if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec, /*IncludeSelf*/ false)
 		{
 			ZEN_WARN("attempting hard terminate of zen process with pid ({})", RunningProcess.Pid());
 			try
diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h
index 32d8ddab3..c9af16749 100644
--- a/src/zen/cmds/up_cmd.h
+++ b/src/zen/cmds/up_cmd.h
@@ -18,11 +18,11 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{"up", "Bring up zen service"};
-	uint16_t		 m_Port		   = 0;
-	bool			 m_ShowConsole = false;
-	bool			 m_ShowLog	   = false;
-	std::string		 m_ProgramBaseDir;
+	cxxopts::Options	  m_Options{"up", "Bring up zen service"};
+	uint16_t			  m_Port		= 0;
+	bool				  m_ShowConsole = false;
+	bool				  m_ShowLog		= false;
+	std::filesystem::path m_ProgramBaseDir;
 };
 
 class AttachCommand : public ZenCmdBase
@@ -35,10 +35,10 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"};
-	uint16_t		 m_Port		= 0;
-	int				 m_OwnerPid = 0;
-	std::string		 m_DataDir;
+	cxxopts::Options	  m_Options{"attach", "Add a sponsor process to a running zen service"};
+	uint16_t			  m_Port	 = 0;
+	int					  m_OwnerPid = 0;
+	std::filesystem::path m_DataDir;
 };
 
 class DownCommand : public ZenCmdBase
@@ -51,11 +51,11 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{"down", "Bring down zen service"};
-	uint16_t		 m_Port			  = 0;
-	bool			 m_ForceTerminate = false;
-	std::string		 m_ProgramBaseDir;
-	std::string		 m_DataDir;
+	cxxopts::Options	  m_Options{"down", "Bring down zen service"};
+	uint16_t			  m_Port		   = 0;
+	bool				  m_ForceTerminate = false;
+	std::filesystem::path m_ProgramBaseDir;
+	std::filesystem::path m_DataDir;
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp
index 41042533d..7dfa125e4 100644
--- a/src/zen/cmds/version_cmd.cpp
+++ b/src/zen/cmds/version_cmd.cpp
@@ -2,10 +2,12 @@
 
 #include "version_cmd.h"
 
+#include <zencore/basicfile.h>
 #include <zencore/config.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zenhttp/httpclient.h>
 #include <zenhttp/httpcommon.h>
 #include <zenutil/zenserverprocess.h>
 
@@ -17,11 +19,14 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
+using namespace std::literals;
+
 VersionCommand::VersionCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
 	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), "[hosturl]");
 	m_Options.add_option("", "d", "detailed", "Detailed Version", cxxopts::value(m_DetailedVersion), "[detailedversion]");
+	m_Options.add_option("", "o", "output-path", "Path for output", cxxopts::value(m_OutputPath), "[outputpath]");
 	m_Options.parse_positional({"hosturl"});
 }
 
@@ -51,28 +56,40 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	}
 	else
 	{
-		const std::string UrlBase = fmt::format("{}/health", m_HostName);
-		cpr::Session	  Session;
-		std::string		  VersionRequest = fmt::format("{}/version{}", UrlBase, m_DetailedVersion ? "?detailed=true" : "");
-		Session.SetUrl(VersionRequest);
-		cpr::Response Response = Session.Get();
-		if (!zen::IsHttpSuccessCode(Response.status_code))
+		if (!m_OutputPath.empty())
 		{
-			if (Response.status_code)
-			{
-				ZEN_ERROR("{} failed: {}: {} ({})", VersionRequest, Response.status_code, Response.reason, Response.text);
-			}
-			else
-			{
-				ZEN_ERROR("{} failed: {}", VersionRequest, Response.error.message);
-			}
+			ZEN_CONSOLE("Querying host {}", m_HostName);
+		}
 
+		HttpClient Client(m_HostName, HttpClientSettings{.Timeout = std::chrono::milliseconds(5000)});
+
+		HttpClient::KeyValueMap Parameters;
+		if (m_DetailedVersion)
+		{
+			Parameters.Entries.insert_or_assign("detailed", "true");
+		}
+		const std::string_view VersionRequest("/health/version"sv);
+		HttpClient::Response   Response = Client.Get(VersionRequest, {}, Parameters);
+		if (!Response.IsSuccess())
+		{
+			ZEN_ERROR("{} failed: {}", VersionRequest, Response.ErrorMessage(""sv));
 			return 1;
 		}
-		Version = Response.text;
+		Version = Response.AsText();
 	}
 
-	ZEN_CONSOLE("{}", Version);
+	if (m_OutputPath.empty())
+	{
+		ZEN_CONSOLE("{}", Version);
+	}
+	else
+	{
+		ZEN_CONSOLE("Writing version '{}' to '{}'", Version, m_OutputPath);
+
+		BasicFile OutputFile(m_OutputPath, BasicFile::Mode::kTruncate);
+		OutputFile.Write(Version.data(), Version.length(), 0);
+		OutputFile.Close();
+	}
 
 	return 0;
 }
diff --git a/src/zen/cmds/version_cmd.h b/src/zen/cmds/version_cmd.h
index f8d16fb96..7a910e463 100644
--- a/src/zen/cmds/version_cmd.h
+++ b/src/zen/cmds/version_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
 class VersionCommand : public ZenCmdBase
 {
 public:
+	static constexpr char Name[]		= "version";
+	static constexpr char Description[] = "Get zen service version";
+
 	VersionCommand();
 	~VersionCommand();
 
@@ -16,9 +19,10 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{"version", "Get zen service version"};
-	std::string		 m_HostName;
-	bool			 m_DetailedVersion = false;
+	cxxopts::Options	  m_Options{Name, Description};
+	std::string			  m_HostName;
+	bool				  m_DetailedVersion = false;
+	std::filesystem::path m_OutputPath;
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp
new file mode 100644
index 000000000..fcc18df2b
--- /dev/null
+++ b/src/zen/cmds/wipe_cmd.cpp
@@ -0,0 +1,606 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "wipe_cmd.h"
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zenutil/parallelwork.h>
+#include <zenutil/workerpools.h>
+
+#include <signal.h>
+
+#include <iostream>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#else
+#	include <fcntl.h>
+#	include <sys/file.h>
+#	include <sys/stat.h>
+#	include <unistd.h>
+#endif
+
+namespace zen {
+
+namespace {
+	static std::atomic<bool> AbortFlag			= false;
+	static std::atomic<bool> PauseFlag			= false;
+	static bool				 IsVerbose			= false;
+	static bool				 Quiet				= false;
+	static ProgressBar::Mode ProgressMode		= ProgressBar::Mode::Pretty;
+	const bool				 SingleThreaded		= false;
+	bool					 BoostWorkerThreads = true;
+
+	uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode)
+	{
+		switch (InMode)
+		{
+			case ProgressBar::Mode::Plain:
+				return 5000;
+			case ProgressBar::Mode::Pretty:
+				return 200;
+			case ProgressBar::Mode::Log:
+				return 2000;
+			default:
+				ZEN_ASSERT(false);
+				return 0;
+		}
+	}
+
+	WorkerThreadPool& GetIOWorkerPool()
+	{
+		return SingleThreaded		? GetSyncWorkerPool()
+			   : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst)
+									: GetMediumWorkerPool(EWorkloadType::Burst);
+	}
+
+#define ZEN_CONSOLE_VERBOSE(fmtstr, ...)                                   \
+	if (IsVerbose)                                                         \
+	{                                                                      \
+		ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \
+	}
+
+	static void SignalCallbackHandler(int SigNum)
+	{
+		if (SigNum == SIGINT)
+		{
+			PauseFlag = false;
+			AbortFlag = true;
+		}
+#if ZEN_PLATFORM_WINDOWS
+		if (SigNum == SIGBREAK)
+		{
+			PauseFlag = false;
+			AbortFlag = true;
+		}
+#endif	// ZEN_PLATFORM_WINDOWS
+	}
+
+	bool IsReadOnly(uint32_t Attributes)
+	{
+#if ZEN_PLATFORM_WINDOWS
+		return IsFileAttributeReadOnly(Attributes);
+#else
+		return IsFileModeReadOnly(Attributes);
+#endif
+	}
+
+	bool IsFileWithRetry(const std::filesystem::path& Path)
+	{
+		std::error_code Ec;
+		bool			Result = IsFile(Path, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			Ec.clear();
+			Result = IsFile(Path, Ec);
+		}
+		if (Ec)
+		{
+			zen::ThrowSystemError(Ec.value(), Ec.message());
+		}
+		return Result;
+	}
+
+	bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
+	{
+		std::error_code Ec;
+		bool			Result = SetFileReadOnly(Path, ReadOnly, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			if (!IsFileWithRetry(Path))
+			{
+				return false;
+			}
+			Ec.clear();
+			Result = SetFileReadOnly(Path, ReadOnly, Ec);
+		}
+		if (Ec)
+		{
+			zen::ThrowSystemError(Ec.value(), Ec.message());
+		}
+		return Result;
+	}
+
+	void RemoveFileWithRetry(const std::filesystem::path& Path)
+	{
+		std::error_code Ec;
+		RemoveFile(Path, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			if (!IsFileWithRetry(Path))
+			{
+				return;
+			}
+			Ec.clear();
+			RemoveFile(Path, Ec);
+		}
+		if (Ec)
+		{
+			zen::ThrowSystemError(Ec.value(), Ec.message());
+		}
+	}
+
+	void RemoveDirWithRetry(const std::filesystem::path& Path)
+	{
+		std::error_code Ec;
+		RemoveDir(Path, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			if (!IsDir(Path))
+			{
+				return;
+			}
+			Ec.clear();
+			RemoveDir(Path, Ec);
+		}
+		if (Ec)
+		{
+			zen::ThrowSystemError(Ec.value(), Ec.message());
+		}
+	}
+
+	bool CleanDirectory(const std::filesystem::path&	  Path,
+						std::span<const std::string_view> ExcludeDirectories,
+						bool							  RemoveReadonly,
+						bool							  Dryrun)
+	{
+		ZEN_TRACE_CPU("CleanDirectory");
+		Stopwatch Timer;
+
+		ProgressBar Progress(ProgressMode, "Clean Folder");
+
+		std::atomic<bool>	  CleanWipe			  = true;
+		std::atomic<uint64_t> DiscoveredItemCount = 0;
+		std::atomic<uint64_t> DeletedItemCount	  = 0;
+		std::atomic<uint64_t> DeletedByteCount	  = 0;
+		std::atomic<uint64_t> FailedDeleteCount	  = 0;
+
+		std::vector<std::filesystem::path>			   SubdirectoriesToDelete;
+		tsl::robin_map<IoHash, size_t, IoHash::Hasher> SubdirectoriesToDeleteLookup;
+		tsl::robin_set<IoHash, IoHash::Hasher>		   SubdirectoriesToKeep;
+		RwLock										   SubdirectoriesLock;
+
+		auto AddFoundDirectory = [&](std::filesystem::path Directory, bool Keep) -> bool {
+			bool Added = false;
+			if (Keep)
+			{
+				bool IsLeaf = true;
+				while (Directory != Path)
+				{
+					const std::string		   DirectoryString	 = Directory.generic_string();
+					IoHash					   DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length());
+					RwLock::ExclusiveLockScope _(SubdirectoriesLock);
+					if (auto It = SubdirectoriesToKeep.find(DirectoryNameHash); It == SubdirectoriesToKeep.end())
+					{
+						SubdirectoriesToKeep.insert(DirectoryNameHash);
+						if (IsLeaf)
+						{
+							Added = true;
+						}
+					}
+					else
+					{
+						break;
+					}
+					Directory = Directory.parent_path();
+					IsLeaf	  = false;
+				}
+			}
+			else
+			{
+				bool IsLeaf = true;
+				while (Directory != Path)
+				{
+					const std::string		   DirectoryString	 = Directory.generic_string();
+					IoHash					   DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length());
+					RwLock::ExclusiveLockScope _(SubdirectoriesLock);
+					if (SubdirectoriesToKeep.contains(DirectoryNameHash))
+					{
+						break;
+					}
+					if (auto It = SubdirectoriesToDeleteLookup.find(DirectoryNameHash); It == SubdirectoriesToDeleteLookup.end())
+					{
+						SubdirectoriesToDeleteLookup.insert({DirectoryNameHash, SubdirectoriesToDelete.size()});
+						SubdirectoriesToDelete.push_back(Directory);
+						if (IsLeaf)
+						{
+							Added = true;
+						}
+					}
+					else
+					{
+						break;
+					}
+					Directory = Directory.parent_path();
+					IsLeaf	  = false;
+				}
+			}
+			return Added;
+		};
+
+		ParallelWork Work(AbortFlag, PauseFlag);
+
+		struct AsyncVisitor : public GetDirectoryContentVisitor
+		{
+			AsyncVisitor(const std::filesystem::path&							 InPath,
+						 std::atomic<bool>&										 InCleanWipe,
+						 std::atomic<uint64_t>&									 InDiscoveredItemCount,
+						 std::atomic<uint64_t>&									 InDeletedItemCount,
+						 std::atomic<uint64_t>&									 InDeletedByteCount,
+						 std::atomic<uint64_t>&									 InFailedDeleteCount,
+						 std::span<const std::string_view>						 InExcludeDirectories,
+						 bool													 InRemoveReadonly,
+						 bool													 InDryrun,
+						 const std::function<bool(std::filesystem::path, bool)>& InAddFoundDirectoryFunc)
+			: Path(InPath)
+			, CleanWipe(InCleanWipe)
+			, DiscoveredItemCount(InDiscoveredItemCount)
+			, DeletedItemCount(InDeletedItemCount)
+			, DeletedByteCount(InDeletedByteCount)
+			, FailedDeleteCount(InFailedDeleteCount)
+			, ExcludeDirectories(InExcludeDirectories)
+			, RemoveReadonly(InRemoveReadonly)
+			, Dryrun(InDryrun)
+			, AddFoundDirectoryFunc(InAddFoundDirectoryFunc)
+			{
+			}
+			virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+			{
+				ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
+				if (!AbortFlag)
+				{
+					if (!RelativeRoot.empty())
+					{
+						DiscoveredItemCount++;
+					}
+					if (Content.FileNames.empty())
+					{
+						const std::filesystem::path ParentPath	  = Path / RelativeRoot;
+						bool						KeepDirectory = RelativeRoot.empty();
+
+						bool Added = AddFoundDirectoryFunc(ParentPath, KeepDirectory);
+						if (Added)
+						{
+							ZEN_CONSOLE_VERBOSE("{} directory {}", KeepDirectory ? "Keeping" : "Removing", ParentPath);
+						}
+					}
+					else
+					{
+						DiscoveredItemCount += Content.FileNames.size();
+
+						const std::string RelativeRootString = RelativeRoot.generic_string();
+						bool			  RemoveContent		 = true;
+						for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+						{
+							if (RelativeRootString.starts_with(ExcludeDirectory))
+							{
+								if (RelativeRootString.length() > ExcludeDirectory.length())
+								{
+									const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
+									if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
+										MaybePathDelimiter == std::filesystem::path::preferred_separator)
+									{
+										RemoveContent = false;
+										break;
+									}
+								}
+								else
+								{
+									RemoveContent = false;
+									break;
+								}
+							}
+						}
+
+						const std::filesystem::path ParentPath	  = Path / RelativeRoot;
+						bool						KeepDirectory = RelativeRoot.empty();
+
+						if (RemoveContent)
+						{
+							ZEN_TRACE_CPU("DeleteFiles");
+							uint64_t RemovedCount = 0;
+							for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+							{
+								const std::filesystem::path& FileName = Content.FileNames[FileIndex];
+								const std::filesystem::path	 FilePath = (ParentPath / FileName).make_preferred();
+								try
+								{
+									const uint32_t Attributes = Content.FileAttributes[FileIndex];
+									const bool	   IsReadonly = IsReadOnly(Attributes);
+									bool		   RemoveFile = false;
+									if (IsReadonly)
+									{
+										if (RemoveReadonly)
+										{
+											if (!Dryrun)
+											{
+												SetFileReadOnlyWithRetry(FilePath, false);
+											}
+											RemoveFile = true;
+										}
+									}
+									else
+									{
+										RemoveFile = true;
+									}
+
+									if (RemoveFile)
+									{
+										if (!Dryrun)
+										{
+											RemoveFileWithRetry(FilePath);
+										}
+										DeletedItemCount++;
+										DeletedByteCount += Content.FileSizes[FileIndex];
+										RemovedCount++;
+										ZEN_CONSOLE_VERBOSE("Removed file {}", FilePath);
+									}
+									else
+									{
+										ZEN_CONSOLE_VERBOSE("Skipped readonly file {}", FilePath);
+										KeepDirectory = true;
+									}
+								}
+								catch (const std::exception& Ex)
+								{
+									ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what());
+									FailedDeleteCount++;
+									CleanWipe	  = false;
+									KeepDirectory = true;
+								}
+							}
+							ZEN_CONSOLE_VERBOSE("Removed {} files in {}", RemovedCount, ParentPath);
+						}
+						else
+						{
+							ZEN_CONSOLE_VERBOSE("Skipped removal of {} files in {}", Content.FileNames.size(), ParentPath);
+						}
+						bool Added = AddFoundDirectoryFunc(ParentPath, KeepDirectory);
+						if (Added)
+						{
+							ZEN_CONSOLE_VERBOSE("{} directory {}", KeepDirectory ? "Keeping" : "Removing", ParentPath);
+						}
+					}
+				}
+			}
+			const std::filesystem::path&					 Path;
+			std::atomic<bool>&								 CleanWipe;
+			std::atomic<uint64_t>&							 DiscoveredItemCount;
+			std::atomic<uint64_t>&							 DeletedItemCount;
+			std::atomic<uint64_t>&							 DeletedByteCount;
+			std::atomic<uint64_t>&							 FailedDeleteCount;
+			std::span<const std::string_view>				 ExcludeDirectories;
+			const bool										 RemoveReadonly;
+			const bool										 Dryrun;
+			std::function<bool(std::filesystem::path, bool)> AddFoundDirectoryFunc;
+		} Visitor(Path,
+				  CleanWipe,
+				  DiscoveredItemCount,
+				  DeletedItemCount,
+				  DeletedByteCount,
+				  FailedDeleteCount,
+				  ExcludeDirectories,
+				  RemoveReadonly,
+				  Dryrun,
+				  AddFoundDirectory);
+
+		uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+		GetDirectoryContent(Path,
+							DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive |
+								DirectoryContentFlags::IncludeFileSizes | DirectoryContentFlags::IncludeAttributes,
+							Visitor,
+							GetIOWorkerPool(),
+							Work.PendingWork());
+
+		Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) {
+			ZEN_UNUSED(PendingWork);
+			if (Quiet)
+			{
+				return;
+			}
+			LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+			uint64_t Deleted	  = DeletedItemCount.load();
+			uint64_t DeletedBytes = DeletedByteCount.load();
+			uint64_t Discovered	  = DiscoveredItemCount.load();
+			Progress.UpdateState({.Task			  = "Removing files  ",
+								  .Details		  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+								  .TotalCount	  = Discovered,
+								  .RemainingCount = Discovered - Deleted,
+								  .Status		  = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+								 false);
+		});
+
+		std::vector<std::filesystem::path> DirectoriesToDelete;
+		DirectoriesToDelete.reserve(SubdirectoriesToDelete.size());
+		for (auto It : SubdirectoriesToDeleteLookup)
+		{
+			const IoHash& DirHash = It.first;
+			if (auto KeepIt = SubdirectoriesToKeep.find(DirHash); KeepIt == SubdirectoriesToKeep.end())
+			{
+				DirectoriesToDelete.emplace_back(std::move(SubdirectoriesToDelete[It.second]));
+			}
+		}
+
+		std::sort(DirectoriesToDelete.begin(),
+				  DirectoriesToDelete.end(),
+				  [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) {
+					  return Lhs.string().length() > Rhs.string().length();
+				  });
+
+		for (size_t SubDirectoryIndex = 0; SubDirectoryIndex < DirectoriesToDelete.size(); SubDirectoryIndex++)
+		{
+			ZEN_TRACE_CPU("DeleteDirs");
+			const std::filesystem::path& DirectoryToDelete = DirectoriesToDelete[SubDirectoryIndex];
+			try
+			{
+				if (!Dryrun)
+				{
+					RemoveDirWithRetry(DirectoryToDelete);
+				}
+				ZEN_CONSOLE_VERBOSE("Removed directory {}", DirectoryToDelete);
+				DeletedItemCount++;
+			}
+			catch (const std::exception& Ex)
+			{
+				if (!Quiet)
+				{
+					ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
+				}
+				CleanWipe = false;
+				FailedDeleteCount++;
+			}
+
+			uint64_t NowMs = Timer.GetElapsedTimeMs();
+			if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode))
+			{
+				LastUpdateTimeMs = NowMs;
+
+				uint64_t Deleted	  = DeletedItemCount.load();
+				uint64_t DeletedBytes = DeletedByteCount.load();
+				uint64_t Discovered	  = DiscoveredItemCount.load();
+				Progress.UpdateState({.Task		  = "Removing folders",
+									  .Details	  = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+									  .TotalCount = DirectoriesToDelete.size(),
+									  .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex},
+									 false);
+			}
+		}
+
+		Progress.Finish();
+
+		uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
+		if (!Quiet)
+		{
+			ZEN_CONSOLE("Wiped folder '{}' {} ({}) ({} failed) in {}",
+						Path,
+						DeletedItemCount.load(),
+						NiceBytes(DeletedByteCount.load()),
+						FailedDeleteCount.load(),
+						NiceTimeSpanMs(ElapsedTimeMs));
+		}
+		if (FailedDeleteCount.load() > 0)
+		{
+			throw std::runtime_error(fmt::format("Failed to delete {} files/directories in '{}'", FailedDeleteCount.load(), Path));
+		}
+		return CleanWipe;
+	}
+}  // namespace
+
+WipeCommand::WipeCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "d", "directory", "Directory to wipe", cxxopts::value(m_Directory), "<directory>");
+	m_Options.add_option("", "r", "keep-readonly", "Leave read-only files", cxxopts::value(m_KeepReadOnlyFiles), "<keepreadonly>");
+	m_Options.add_option("", "q", "quiet", "Reduce output to console", cxxopts::value(m_Quiet), "<quiet>");
+	m_Options.add_option("", "y", "yes", "Don't query for confirmation", cxxopts::value(m_Yes), "<yes>");
+	m_Options.add_option("", "", "dryrun", "Do a dry run without deleting anything", cxxopts::value(m_Dryrun), "<dryrun>");
+	m_Options.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<progress>");
+	m_Options.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
+	m_Options.add_option("",
+						 "",
+						 "boost-workers",
+						 "Increase the number of worker threads - may cause computer to be less responsive",
+						 cxxopts::value(m_BoostWorkerThreads),
+						 "<boostworkers>");
+
+	m_Options.parse_positional({"directory"});
+}
+
+WipeCommand::~WipeCommand() = default;
+
+int
+WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	signal(SIGINT, SignalCallbackHandler);
+#if ZEN_PLATFORM_WINDOWS
+	signal(SIGBREAK, SignalCallbackHandler);
+#endif	// ZEN_PLATFORM_WINDOWS
+
+	if (!ZenCmdBase::ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	Quiet			   = m_Quiet;
+	IsVerbose		   = m_Verbose;
+	ProgressMode	   = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty;
+	BoostWorkerThreads = m_BoostWorkerThreads;
+
+	MakeSafeAbsolutePathÍnPlace(m_Directory);
+
+	if (!IsDir(m_Directory))
+	{
+		return 0;
+	}
+
+	while (!m_Yes)
+	{
+		const std::string Prompt = fmt::format("Do you want to wipe directory '{}'? (yes/no) ", m_Directory);
+		printf("%s", Prompt.c_str());
+		std::string Reponse;
+		std::getline(std::cin, Reponse);
+		Reponse = ToLower(Reponse);
+		if (Reponse == "y" || Reponse == "yes")
+		{
+			m_Yes = true;
+		}
+		else if (Reponse == "n" || Reponse == "no")
+		{
+			return 0;
+		}
+	}
+
+	try
+	{
+		CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun);
+	}
+	catch (std::exception& Ex)
+	{
+		if (!m_Quiet)
+		{
+			ZEN_ERROR("{}", Ex.what());
+		}
+		return 3;
+	}
+
+	return 0;
+}
+
+}  // namespace zen
diff --git a/src/zen/cmds/wipe_cmd.h b/src/zen/cmds/wipe_cmd.h
new file mode 100644
index 000000000..0e910bb81
--- /dev/null
+++ b/src/zen/cmds/wipe_cmd.h
@@ -0,0 +1,36 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+/** Wipe directories
+ */
+class WipeCommand : public ZenCmdBase
+{
+public:
+	static constexpr char Name[]		= "wipe";
+	static constexpr char Description[] = "Wipe the contents of a directory";
+
+	WipeCommand();
+	~WipeCommand();
+
+	virtual cxxopts::Options& Options() override { return m_Options; }
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
+
+private:
+	cxxopts::Options	  m_Options{Name, Description};
+	std::filesystem::path m_Directory;
+	bool				  m_KeepReadOnlyFiles  = true;
+	bool				  m_Quiet			   = false;
+	bool				  m_Yes				   = false;
+	bool				  m_PlainProgress	   = false;
+	bool				  m_Verbose			   = false;
+	bool				  m_Dryrun			   = false;
+	bool				  m_BoostWorkerThreads = false;
+};
+
+}  // namespace zen
diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp
index 5f3f8f7ca..773734f12 100644
--- a/src/zen/cmds/workspaces_cmd.cpp
+++ b/src/zen/cmds/workspaces_cmd.cpp
@@ -139,18 +139,16 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	m_HostName = ResolveTargetHostSpec(m_HostName);
 
-	std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir);
-
-	if (SystemRootDir.empty())
+	if (m_SystemRootDir.empty())
 	{
-		SystemRootDir = PickDefaultSystemRootDirectory();
-		if (SystemRootDir.empty())
+		m_SystemRootDir = PickDefaultSystemRootDirectory();
+		if (m_SystemRootDir.empty())
 		{
 			throw zen::OptionParseException("unable to resolve system root directory");
 		}
 	}
 
-	std::filesystem::path StatePath = SystemRootDir / "workspaces";
+	std::filesystem::path StatePath = m_SystemRootDir / "workspaces";
 
 	if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
 	{
@@ -392,18 +390,20 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
 
 	m_HostName = ResolveTargetHostSpec(m_HostName);
 
-	std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir);
-
-	if (SystemRootDir.empty())
+	if (m_SystemRootDir.empty())
 	{
-		SystemRootDir = PickDefaultSystemRootDirectory();
-		if (SystemRootDir.empty())
+		m_SystemRootDir = PickDefaultSystemRootDirectory();
+		if (m_SystemRootDir.empty())
 		{
 			throw zen::OptionParseException("unable to resolve system root directory");
 		}
 	}
+	else
+	{
+		MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+	}
 
-	std::filesystem::path StatePath = SystemRootDir / "workspaces";
+	std::filesystem::path StatePath = m_SystemRootDir / "workspaces";
 
 	if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
 	{
@@ -412,8 +412,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
 
 	if (SubOption == &m_CreateOptions)
 	{
-		std::filesystem::path WorkspaceRoot = StringToPath(m_WorkspaceRoot);
-		if (WorkspaceRoot.empty())
+		if (m_WorkspaceRoot.empty())
 		{
 			if (m_WorkspaceId.empty())
 			{
@@ -432,15 +431,15 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
 				ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId);
 				return 0;
 			}
-			WorkspaceRoot = WorkspaceConfig.RootPath;
+			m_WorkspaceRoot = WorkspaceConfig.RootPath;
 		}
 		else
 		{
-			RemoveTrailingPathSeparator(WorkspaceRoot);
+			RemoveTrailingPathSeparator(m_WorkspaceRoot);
 			if (m_WorkspaceId.empty())
 			{
-				m_WorkspaceId = Workspaces::PathToId(WorkspaceRoot).ToString();
-				ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, WorkspaceRoot);
+				m_WorkspaceId = Workspaces::PathToId(m_WorkspaceRoot).ToString();
+				ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, m_WorkspaceRoot);
 			}
 			else
 			{
@@ -449,25 +448,23 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
 					throw zen::OptionParseException(fmt::format("workspace id '{}' is invalid", m_WorkspaceId));
 				}
 			}
-			if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = WorkspaceRoot}))
+			if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = m_WorkspaceRoot}))
 			{
-				ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, WorkspaceRoot);
+				ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, m_WorkspaceRoot);
 			}
 			else
 			{
-				ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, WorkspaceRoot);
+				ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, m_WorkspaceRoot);
 			}
 		}
 
-		std::filesystem::path SharePath = StringToPath(m_SharePath);
-
-		RemoveLeadingPathSeparator(SharePath);
-		RemoveTrailingPathSeparator(SharePath);
+		RemoveLeadingPathSeparator(m_SharePath);
+		RemoveTrailingPathSeparator(m_SharePath);
 
 		if (m_ShareId.empty())
 		{
-			m_ShareId = Workspaces::PathToId(SharePath).ToString();
-			ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, SharePath);
+			m_ShareId = Workspaces::PathToId(m_SharePath).ToString();
+			ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, m_SharePath);
 		}
 
 		if (Oid::TryFromHexString(m_ShareId) == Oid::Zero)
@@ -476,8 +473,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
 		}
 
 		if (Workspaces::AddWorkspaceShare(Log(),
-										  WorkspaceRoot,
-										  {.Id = Oid::FromHexString(m_ShareId), .SharePath = SharePath, .Alias = m_Alias}))
+										  m_WorkspaceRoot,
+										  {.Id = Oid::FromHexString(m_ShareId), .SharePath = m_SharePath, .Alias = m_Alias}))
 		{
 			if (!m_HostName.empty())
 			{
diff --git a/src/zen/cmds/workspaces_cmd.h b/src/zen/cmds/workspaces_cmd.h
index 86452e25e..d85d8f7d8 100644
--- a/src/zen/cmds/workspaces_cmd.h
+++ b/src/zen/cmds/workspaces_cmd.h
@@ -21,9 +21,9 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{Name, Description};
-	std::string		 m_HostName;
-	std::string		 m_SystemRootDir;
+	cxxopts::Options	  m_Options{Name, Description};
+	std::string			  m_HostName;
+	std::filesystem::path m_SystemRootDir;
 
 	std::string m_Verb;	 // create, info, remove
 
@@ -53,17 +53,17 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{Name, Description};
-	std::string		 m_HostName;
-	std::string		 m_SystemRootDir;
-	std::string		 m_WorkspaceId;
-	std::string		 m_WorkspaceRoot;
-	std::string		 m_Verb;  // create, info, remove
-	std::string		 m_ShareId;
-	std::string		 m_Alias;
-
-	cxxopts::Options m_CreateOptions{"create", "Create a workspace share"};
-	std::string		 m_SharePath;
+	cxxopts::Options	  m_Options{Name, Description};
+	std::string			  m_HostName;
+	std::filesystem::path m_SystemRootDir;
+	std::string			  m_WorkspaceId;
+	std::filesystem::path m_WorkspaceRoot;
+	std::string			  m_Verb;  // create, info, remove
+	std::string			  m_ShareId;
+	std::string			  m_Alias;
+
+	cxxopts::Options	  m_CreateOptions{"create", "Create a workspace share"};
+	std::filesystem::path m_SharePath;
 
 	bool m_Refresh = false;
 
diff --git a/src/zen/xmake.lua b/src/zen/xmake.lua
index a755df374..dadda32dd 100644
--- a/src/zen/xmake.lua
+++ b/src/zen/xmake.lua
@@ -18,13 +18,14 @@ target("zen")
         add_ldflags("/subsystem:console,5.02")
         add_ldflags("/LTCG")
         add_links("crypt32", "wldap32", "Ws2_32", "Shlwapi")
+        add_links("dbghelp", "winhttp", "version")    -- for Sentry
     end
 
     if is_plat("macosx") then
         add_ldflags("-framework CoreFoundation")
+        add_ldflags("-framework Foundation")
         add_ldflags("-framework Security")
         add_ldflags("-framework SystemConfiguration")
-        add_syslinks("bsm")
     end
 
     add_packages("vcpkg::cpr", "vcpkg::cxxopts", "vcpkg::mimalloc", "vcpkg::fmt")
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index a5fd693f2..15988080a 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -24,6 +24,7 @@
 #include "cmds/up_cmd.h"
 #include "cmds/version_cmd.h"
 #include "cmds/vfs_cmd.h"
+#include "cmds/wipe_cmd.h"
 #include "cmds/workspaces_cmd.h"
 
 #include <zencore/callstack.h>
@@ -32,10 +33,12 @@
 #include <zencore/logging.h>
 #include <zencore/process.h>
 #include <zencore/scopeguard.h>
+#include <zencore/sentryintegration.h>
 #include <zencore/string.h>
 #include <zencore/trace.h>
 #include <zencore/windows.h>
 #include <zenhttp/httpcommon.h>
+#include <zenutil/environmentoptions.h>
 #include <zenutil/logging.h>
 #include <zenutil/zenserverprocess.h>
 
@@ -59,7 +62,8 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 #include <zencore/memory/newdelete.h>
 
-#ifndef ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+#	include <sys/ioctl.h>
 #	include <unistd.h>
 #endif
 
@@ -268,25 +272,121 @@ ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec)
 	return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy);
 }
 
+#if ZEN_PLATFORM_WINDOWS
+static HANDLE
+GetConsoleHandle()
+{
+	static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
+	return hStdOut;
+}
+#endif
+
 static bool
-IsStdoutTty()
+CheckStdoutTty()
 {
 #if ZEN_PLATFORM_WINDOWS
-	static HANDLE hStdOut	= ::GetStdHandle(STD_OUTPUT_HANDLE);
-	DWORD		  dwMode	= 0;
-	static bool	  IsConsole = ::GetConsoleMode(hStdOut, &dwMode);
+	HANDLE		hStdOut	  = GetConsoleHandle();
+	DWORD		dwMode	  = 0;
+	static bool IsConsole = ::GetConsoleMode(hStdOut, &dwMode);
 	return IsConsole;
 #else
 	return isatty(fileno(stdout));
 #endif
 }
 
-ProgressBar::ProgressBar(bool PlainProgress, bool ShowDetails)
-: m_StdoutIsTty(IsStdoutTty())
-, m_PlainProgress(PlainProgress || !m_StdoutIsTty)
-, m_ShowDetails(ShowDetails)
-, m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000)
+static bool
+IsStdoutTty()
+{
+	static bool StdoutIsTty = CheckStdoutTty();
+	return StdoutIsTty;
+}
+
+static void
+OutputToConsoleRaw(const char* String, size_t Length)
+{
+#if ZEN_PLATFORM_WINDOWS
+	HANDLE hStdOut = GetConsoleHandle();
+#endif
+
+#if ZEN_PLATFORM_WINDOWS
+	if (IsStdoutTty())
+	{
+		WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0);
+	}
+	else
+	{
+		::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0);
+	}
+#else
+	fwrite(String, 1, Length, stdout);
+#endif
+}
+
+static void
+OutputToConsoleRaw(const std::string& String)
 {
+	OutputToConsoleRaw(String.c_str(), String.length());
+}
+
+static void
+OutputToConsoleRaw(const StringBuilderBase& SB)
+{
+	OutputToConsoleRaw(SB.c_str(), SB.Size());
+}
+
+static uint32_t
+GetConsoleColumns()
+{
+#if ZEN_PLATFORM_WINDOWS
+	HANDLE					   hStdOut = GetConsoleHandle();
+	CONSOLE_SCREEN_BUFFER_INFO csbi;
+	if (GetConsoleScreenBufferInfo(hStdOut, &csbi) == TRUE)
+	{
+		return (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1);
+	}
+#else
+	struct winsize w;
+	if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0)
+	{
+		return (uint32_t)w.ws_col;
+	}
+#endif
+	return 1024;
+}
+
+void
+ProgressBar::SetLogOperationName(Mode InMode, std::string_view Name)
+{
+	if (InMode == Mode::Log)
+	{
+		std::string String = fmt::format("@progress {}\n", Name);
+		OutputToConsoleRaw(String);
+	}
+}
+
+void
+ProgressBar::SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount)
+{
+	if (InMode == Mode::Log)
+	{
+		const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u;
+
+		std::string String = fmt::format("@progress {}%\n", PercentDone);
+		OutputToConsoleRaw(String);
+	}
+}
+
+ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask)
+: m_Mode((!IsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode)
+, m_LastUpdateMS((uint64_t)-1)
+, m_PausedMS(0)
+, m_SubTask(InSubTask)
+{
+	if (!m_SubTask.empty() && InMode == Mode::Log)
+	{
+		std::string String = fmt::format("@progress push {}\n", m_SubTask);
+		OutputToConsoleRaw(String);
+	}
 }
 
 ProgressBar::~ProgressBar()
@@ -294,6 +394,11 @@ ProgressBar::~ProgressBar()
 	try
 	{
 		ForceLinebreak();
+		if (!m_SubTask.empty() && m_Mode == Mode::Log)
+		{
+			const std::string String("@progress pop\n");
+			OutputToConsoleRaw(String);
+		}
 	}
 	catch (const std::exception& Ex)
 	{
@@ -311,63 +416,153 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
 	}
 
 	uint64_t ElapsedTimeMS = m_SW.GetElapsedTimeMs();
-	if (!DoLinebreak && (NewState.Task == m_State.Task) && ((m_LastUpdateMS + 200) > ElapsedTimeMS))
+	if (m_LastUpdateMS != (uint64_t)-1)
 	{
-		return;
+		if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) &&
+			((m_LastUpdateMS + 200) > ElapsedTimeMS))
+		{
+			return;
+		}
+		if (m_State.Status == State::EStatus::Paused)
+		{
+			uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS;
+			m_PausedMS += ElapsedSinceLast;
+		}
 	}
 
 	m_LastUpdateMS = ElapsedTimeMS;
 
-	size_t PercentDone =
+	std::string Task = NewState.Task;
+	switch (NewState.Status)
+	{
+		case State::EStatus::Aborted:
+			Task = "Aborting";
+			break;
+		case State::EStatus::Paused:
+			Task = "Paused";
+			break;
+		default:
+			break;
+	}
+	if (NewState.Task.length() > Task.length())
+	{
+		Task += std::string(NewState.Task.length() - Task.length(), ' ');
+	}
+
+	const size_t PercentDone =
 		NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u;
 
-	if (m_PlainProgress)
+	if (m_Mode == Mode::Plain)
 	{
-		std::string Details = (m_ShowDetails && !NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : "";
-		ZEN_CONSOLE("{} {}% ({}){}", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details);
+		const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : "";
+		const std::string Output  = fmt::format("{} {}% ({}){}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details);
+		OutputToConsoleRaw(Output);
 	}
-	else
+	else if (m_Mode == Mode::Pretty)
 	{
 		size_t ProgressBarSize = 20;
 
-		size_t		ProgressBarCount = (ProgressBarSize * PercentDone) / 100;
-		uint64_t	Completed		 = NewState.TotalCount - NewState.RemainingCount;
-		uint64_t	ETAMS			 = (Completed > 0) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0;
-		std::string ETA				 = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : "";
-
-		std::string			   Output	   = fmt::format("\r{} {:#3}%: |{}{}|: {}{}{}",
-										 NewState.Task,
-										 PercentDone,
-										 std::string(ProgressBarCount, '#'),
-										 std::string(ProgressBarSize - ProgressBarCount, ' '),
-										 NiceTimeSpanMs(ElapsedTimeMS),
-										 ETA,
-										 NewState.Details.empty() ? "" : fmt::format(". {}", NewState.Details));
-		std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0;
+		size_t	 ProgressBarCount = (ProgressBarSize * PercentDone) / 100;
+		uint64_t Completed		  = NewState.TotalCount - NewState.RemainingCount;
+		uint64_t ETAElapsedMS	  = ElapsedTimeMS -= m_PausedMS;
+		uint64_t ETAMS =
+			(NewState.Status == State::EStatus::Running) && (PercentDone > 5) ? (ETAElapsedMS * NewState.RemainingCount) / Completed : 0;
 
-		ExtendableStringBuilder<128> LineToPrint;
-		LineToPrint << Output << std::string(EraseLength, ' ');
-		if (DoLinebreak)
-			LineToPrint << "\n";
+		uint32_t ConsoleColumns = GetConsoleColumns();
 
-#if ZEN_PLATFORM_WINDOWS
-		static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
+		const std::string PercentString = fmt::format("{:#3}%", PercentDone);
+
+		const std::string ProgressBarString =
+			fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' '));
+
+		const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS));
+
+		const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : "";
+
+		const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : "";
+
+		ExtendableStringBuilder<256> OutputBuilder;
+
+		OutputBuilder << "\r" << Task << PercentString;
+		if (OutputBuilder.Size() + 1 < ConsoleColumns)
+		{
+			size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1);
+			bool   ElapsedFits	  = RemainingSpace >= ElapsedString.length();
+			RemainingSpace -= ElapsedString.length();
+			bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length();
+			RemainingSpace -= ETAString.length();
+			bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length();
+			RemainingSpace -= DetailsString.length();
+			bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length();
+			RemainingSpace -= ProgressBarString.length();
+
+			if (ProgressBarFits)
+			{
+				OutputBuilder << ProgressBarString;
+			}
+			if (ElapsedFits)
+			{
+				OutputBuilder << ElapsedString;
+			}
+			if (ETAFits)
+			{
+				OutputBuilder << ETAString;
+			}
+			if (DetailsFits)
+			{
+				OutputBuilder << DetailsString;
+			}
+		}
+
+		std::string_view	   Output	   = OutputBuilder.ToView();
+		std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0;
 
-		if (m_StdoutIsTty)
+		ExtendableStringBuilder<256> LineToPrint;
+
+		if (Output.length() + EraseLength >= ConsoleColumns)
 		{
-			WriteConsoleA(hStdOut, LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0);
+			if (m_LastOutputLength > 0)
+			{
+				LineToPrint << "\n";
+			}
+			LineToPrint << Output.substr(1);
+			DoLinebreak = true;
 		}
 		else
 		{
-			::WriteFile(hStdOut, (LPCVOID)LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0);
+			LineToPrint << Output << std::string(EraseLength, ' ');
+		}
+
+		if (DoLinebreak)
+		{
+			LineToPrint << "\n";
 		}
-#else
-		fwrite(LineToPrint.c_str(), 1, LineToPrint.Size(), stdout);
-#endif
+
+		OutputToConsoleRaw(LineToPrint);
 
 		m_LastOutputLength = DoLinebreak ? 0 : Output.length();
 		m_State			   = NewState;
 	}
+	else if (m_Mode == Mode::Log)
+	{
+		if (m_State.Task != NewState.Task ||
+			m_State.Details != NewState.Details)  // TODO: Should we output just because details change? Will this spam the log collector?
+		{
+			const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : "";
+			const std::string Message = fmt::format("@progress {} ({}){}\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), Details);
+			OutputToConsoleRaw(Message);
+		}
+
+		const size_t OldPercentDone =
+			m_State.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u;
+
+		if (OldPercentDone != PercentDone)
+		{
+			const std::string Progress = fmt::format("@progress {}%\n", PercentDone);
+			OutputToConsoleRaw(Progress);
+		}
+		m_State = NewState;
+	}
 }
 
 void
@@ -383,7 +578,7 @@ ProgressBar::ForceLinebreak()
 void
 ProgressBar::Finish()
 {
-	if (m_LastOutputLength > 0)
+	if (m_LastOutputLength > 0 || m_State.RemainingCount > 0)
 	{
 		State NewState			= m_State;
 		NewState.RemainingCount = 0;
@@ -416,6 +611,8 @@ ProgressBar::HasActiveTask() const
 int
 main(int argc, char** argv)
 {
+	zen::SetCurrentThreadName("main");
+
 	std::vector<std::string> Args;
 #if ZEN_PLATFORM_WINDOWS
 	LPWSTR		RawCommandLine = GetCommandLine();
@@ -425,7 +622,11 @@ main(int argc, char** argv)
 	Args.reserve(argc);
 	for (int I = 0; I < argc; I++)
 	{
-		Args.push_back(std::string(argv[I]));
+		std::string Arg(argv[I]);
+		if ((!Arg.empty()) && (Arg != " "))
+		{
+			Args.emplace_back(std::move(Arg));
+		}
 	}
 #endif
 	std::vector<char*> RawArgs = zen::StripCommandlineQuotes(Args);
@@ -482,6 +683,7 @@ main(int argc, char** argv)
 	UpCommand				 UpCmd;
 	VersionCommand			 VersionCmd;
 	VfsCommand				 VfsCmd;
+	WipeCommand				 WipeCmd;
 	WorkspaceCommand		 WorkspaceCmd;
 	WorkspaceShareCommand	 WorkspaceShareCmd;
 	ServiceCommand			 ServiceCmd;
@@ -536,9 +738,10 @@ main(int argc, char** argv)
 		{"top",									&TopCmd,					"Monitor zen server activity"},
 		{"trace",								&TraceCmd,					"Control zen realtime tracing"},
 		{"up",									&UpCmd,						"Bring zen server up"},
-		{"version",								&VersionCmd,				"Get zen server version"},
+		{VersionCommand::Name,					&VersionCmd,				VersionCommand::Description},
 		{"vfs",									&VfsCmd,					"Manage virtual file system"},
 		{"flush",								&FlushCmd,					"Flush storage"},
+		{WipeCommand::Name,						&WipeCmd,					WipeCommand::Description},
 		{WorkspaceCommand::Name,				&WorkspaceCmd,				WorkspaceCommand::Description},
 		{WorkspaceShareCommand::Name,			&WorkspaceShareCmd,			WorkspaceShareCommand::Description},
 		{ServiceCommand::Name,					&ServiceCmd,				ServiceCommand::Description},
@@ -672,6 +875,25 @@ main(int argc, char** argv)
 		.add_option("ue-trace", "", "tracefile", "Path to write a trace to", cxxopts::value<std::string>(TraceFile)->default_value(""), "");
 #endif	// ZEN_WITH_TRACE
 
+#if ZEN_USE_SENTRY
+
+	SentryIntegration::Config SentryConfig;
+
+	bool NoSentry = false;
+
+	Options
+		.add_option("sentry", "", "no-sentry", "Disable Sentry crash handler", cxxopts::value<bool>(NoSentry)->default_value("false"), "");
+	Options.add_option("sentry",
+					   "",
+					   "sentry-allow-personal-info",
+					   "Allow personally identifiable information in sentry crash reports",
+					   cxxopts::value<bool>(SentryConfig.AllowPII)->default_value("false"),
+					   "");
+	Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(SentryConfig.Dsn), "");
+	Options.add_option("sentry", "", "sentry-environment", "Sentry environment", cxxopts::value<std::string>(SentryConfig.Environment), "");
+	Options.add_options()("sentry-debug", "Enable debug mode for Sentry", cxxopts::value<bool>(SentryConfig.Debug)->default_value("false"));
+#endif
+
 	Options.parse_positional({"command"});
 
 	const bool IsNullInvoke = (argc == 1);	// If no arguments are passed we want to print usage information
@@ -713,6 +935,53 @@ main(int argc, char** argv)
 			exit(0);
 		}
 
+#if ZEN_USE_SENTRY
+
+		{
+			EnvironmentOptions EnvOptions;
+
+			EnvOptions.AddOption("UE_ZEN_SENTRY_DSN"sv, SentryConfig.Dsn, "sentry-dsn"sv);
+			EnvOptions.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+			EnvOptions.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, SentryConfig.Environment, "sentry-environment"sv);
+
+			bool EnvEnableSentry = !NoSentry;
+			EnvOptions.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv);
+
+			EnvOptions.AddOption("UE_ZEN_SENTRY_DEBUG"sv, SentryConfig.Debug, "sentry-debug"sv);
+
+			EnvOptions.Parse(ParseResult);
+
+			if (EnvEnableSentry != !NoSentry)
+			{
+				NoSentry = !EnvEnableSentry;
+			}
+		}
+
+		SentryIntegration Sentry;
+
+		if (NoSentry == false)
+		{
+			std::string SentryDatabasePath = (std::filesystem::temp_directory_path() / ".zen-sentry-native").string();
+
+			ExtendableStringBuilder<512> SB;
+			for (int i = 0; i < argc; ++i)
+			{
+				if (i)
+				{
+					SB.Append(' ');
+				}
+
+				SB.Append(argv[i]);
+			}
+
+			SentryConfig.DatabasePath = SentryDatabasePath;
+
+			Sentry.Initialize(SentryConfig, SB.ToString());
+
+			SentryIntegration::ClearCaches();
+		}
+#endif
+
 		zen::LoggingOptions LogOptions;
 		LogOptions.IsDebug	  = GlobalOptions.IsDebug;
 		LogOptions.IsVerbose  = GlobalOptions.IsVerbose;
diff --git a/src/zen/zen.h b/src/zen/zen.h
index 6765101db..995bd13b6 100644
--- a/src/zen/zen.h
+++ b/src/zen/zen.h
@@ -5,10 +5,7 @@
 #include <zencore/except.h>
 #include <zencore/timer.h>
 #include <zencore/zencore.h>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <cxxopts.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
+#include <zenutil/commandlineoptions.h>
 
 namespace cpr {
 class Response;
@@ -82,9 +79,39 @@ public:
 		std::string Details;
 		uint64_t	TotalCount	   = 0;
 		uint64_t	RemainingCount = 0;
+		enum class EStatus
+		{
+			Running,
+			Aborted,
+			Paused
+		};
+		EStatus Status = EStatus::Running;
+
+		static EStatus CalculateStatus(bool IsAborted, bool IsPaused)
+		{
+			if (IsAborted)
+			{
+				return EStatus::Aborted;
+			}
+			if (IsPaused)
+			{
+				return EStatus::Paused;
+			}
+			return EStatus::Running;
+		}
 	};
 
-	explicit ProgressBar(bool PlainProgress, bool ShowDetails = true);
+	enum class Mode
+	{
+		Plain,
+		Pretty,
+		Log
+	};
+
+	static void SetLogOperationName(Mode InMode, std::string_view Name);
+	static void SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount);
+
+	explicit ProgressBar(Mode InMode, std::string_view InSubTask);
 	~ProgressBar();
 
 	void UpdateState(const State& NewState, bool DoLinebreak);
@@ -94,13 +121,13 @@ public:
 	bool HasActiveTask() const;
 
 private:
-	const bool m_StdoutIsTty = true;
-	const bool m_PlainProgress;
-	const bool m_ShowDetails;
-	Stopwatch  m_SW;
-	uint64_t   m_LastUpdateMS;
-	State	   m_State;
-	size_t	   m_LastOutputLength = 0;
+	const Mode		  m_Mode;
+	Stopwatch		  m_SW;
+	uint64_t		  m_LastUpdateMS;
+	uint64_t		  m_PausedMS;
+	State			  m_State;
+	const std::string m_SubTask;
+	size_t			  m_LastOutputLength = 0;
 };
 
 }  // namespace zen
diff --git a/src/zencore-test/zencore-test.cpp b/src/zencore-test/zencore-test.cpp
index 1cc5e1a87..0f64f4a5f 100644
--- a/src/zencore-test/zencore-test.cpp
+++ b/src/zencore-test/zencore-test.cpp
@@ -5,6 +5,7 @@
 
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 #include <zencore/zencore.h>
 
 #include <zencore/memory/newdelete.h>
@@ -25,6 +26,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 	zen::IgnoreChildSignals();
 #	endif
 
+	zen::TraceInit("zencore-test");
 	zen::logging::InitializeLogging();
 	zen::MaximizeOpenFileCount();
 
diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index 95876cff4..6989da67e 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -181,58 +181,18 @@ BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount)
 void
 BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
 {
-	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
-
-	while (BytesToRead)
+	const uint64_t	MaxChunkSize = 2u * 1024 * 1024 * 1024;
+	std::error_code Ec;
+	ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec);
+	if (Ec)
 	{
-		const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize);
-		int32_t		   Error			   = 0;
-		size_t		   BytesRead		   = 0;
-
-#if ZEN_PLATFORM_WINDOWS
-		OVERLAPPED Ovl{};
-
-		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
-		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
-
-		DWORD dwNumberOfBytesRead = 0;
-		BOOL  Success			  = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl);
-		if (Success)
-		{
-			BytesRead = size_t(dwNumberOfBytesRead);
-		}
-		else
-		{
-			Error = zen::GetLastError();
-		}
-#else
-		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
-		int		Fd		   = int(uintptr_t(m_FileHandle));
-		ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
-		if (ReadResult != -1)
-		{
-			BytesRead = size_t(ReadResult);
-		}
-		else
-		{
-			Error = zen::GetLastError();
-		}
-#endif
-
-		if (Error || (BytesRead != NumberOfBytesToRead))
-		{
-			std::error_code DummyEc;
-			throw std::system_error(std::error_code(Error, std::system_category()),
-									fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
-												FileOffset,
-												NumberOfBytesToRead,
-												PathFromHandle(m_FileHandle, DummyEc),
-												FileSizeFromHandle(m_FileHandle)));
-		}
-
-		BytesToRead -= NumberOfBytesToRead;
-		FileOffset += NumberOfBytesToRead;
-		Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+		std::error_code DummyEc;
+		throw std::system_error(Ec,
+								fmt::format("BasicFile::Read: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
+											FileOffset,
+											BytesToRead,
+											PathFromHandle(m_FileHandle, DummyEc),
+											FileSizeFromHandle(m_FileHandle)));
 	}
 }
 
@@ -323,43 +283,9 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
 void
 BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
 {
-	ZEN_ASSERT(m_FileHandle != nullptr);
+	const uint64_t MaxChunkSize = 2u * 1024 * 1024;
 
-	Ec.clear();
-
-	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
-
-	while (Size)
-	{
-		const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize);
-
-#if ZEN_PLATFORM_WINDOWS
-		OVERLAPPED Ovl{};
-
-		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
-		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
-
-		DWORD dwNumberOfBytesWritten = 0;
-
-		BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl);
-#else
-		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
-		int	 Fd			  = int(uintptr_t(m_FileHandle));
-		int	 BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset);
-		bool Success	  = (BytesWritten > 0);
-#endif
-
-		if (!Success)
-		{
-			Ec = MakeErrorCodeFromLastError();
-
-			return;
-		}
-
-		Size -= NumberOfBytesToWrite;
-		FileOffset += NumberOfBytesToWrite;
-		Data = reinterpret_cast<const uint8_t*>(Data) + NumberOfBytesToWrite;
-	}
+	WriteFile(m_FileHandle, Data, Size, FileOffset, MaxChunkSize, Ec);
 }
 
 void
@@ -405,59 +331,20 @@ BasicFile::Flush()
 uint64_t
 BasicFile::FileSize() const
 {
-#if ZEN_PLATFORM_WINDOWS
-	ULARGE_INTEGER liFileSize;
-	liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
-	if (liFileSize.LowPart == INVALID_FILE_SIZE)
-	{
-		int Error = zen::GetLastError();
-		if (Error)
-		{
-			std::error_code Dummy;
-			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
-		}
-	}
-	return uint64_t(liFileSize.QuadPart);
-#else
-	int Fd = int(uintptr_t(m_FileHandle));
-	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
-	struct stat Stat;
-	if (fstat(Fd, &Stat) == -1)
+	std::error_code Ec;
+	uint64_t		FileSize = FileSizeFromHandle(m_FileHandle, Ec);
+	if (Ec)
 	{
 		std::error_code Dummy;
-		ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
+		ThrowSystemError(Ec.value(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
 	}
-	return uint64_t(Stat.st_size);
-#endif
+	return FileSize;
 }
 
 uint64_t
 BasicFile::FileSize(std::error_code& Ec) const
 {
-#if ZEN_PLATFORM_WINDOWS
-	ULARGE_INTEGER liFileSize;
-	liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
-	if (liFileSize.LowPart == INVALID_FILE_SIZE)
-	{
-		int Error = zen::GetLastError();
-		if (Error)
-		{
-			Ec = MakeErrorCode(Error);
-			return 0;
-		}
-	}
-	return uint64_t(liFileSize.QuadPart);
-#else
-	int Fd = int(uintptr_t(m_FileHandle));
-	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
-	struct stat Stat;
-	if (fstat(Fd, &Stat) == -1)
-	{
-		Ec = MakeErrorCodeFromLastError();
-		return 0;
-	}
-	return uint64_t(Stat.st_size);
-#endif
+	return FileSizeFromHandle(m_FileHandle, Ec);
 }
 
 void
@@ -590,7 +477,7 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::
 	// deleting the temporary file
 	BasicFile::Close();
 
-	std::filesystem::rename(m_TempPath, FinalFileName, Ec);
+	RenameFile(m_TempPath, FinalFileName, Ec);
 
 	if (Ec)
 	{
@@ -792,10 +679,45 @@ BasicFileWriter::~BasicFileWriter()
 }
 
 void
+BasicFileWriter::AddPadding(uint64_t Padding)
+{
+	while (Padding)
+	{
+		const uint64_t BufferOffset			   = m_BufferEnd - m_BufferStart;
+		const uint64_t RemainingBufferCapacity = m_BufferSize - BufferOffset;
+		const uint64_t BlockPadBytes		   = Min(RemainingBufferCapacity, Padding);
+
+		memset(m_Buffer + BufferOffset, 0, BlockPadBytes);
+		m_BufferEnd += BlockPadBytes;
+		Padding -= BlockPadBytes;
+
+		if ((BufferOffset + BlockPadBytes) == m_BufferSize)
+		{
+			Flush();
+		}
+	}
+}
+
+uint64_t
+BasicFileWriter::AlignTo(uint64_t Alignment)
+{
+	uint64_t AlignedPos = RoundUp(m_BufferEnd, Alignment);
+	uint64_t Padding	= AlignedPos - m_BufferEnd;
+	AddPadding(Padding);
+	return AlignedPos;
+}
+
+void
 BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
 {
 	if (m_Buffer == nullptr || (Size >= m_BufferSize))
 	{
+		if (FileOffset == m_BufferEnd)
+		{
+			Flush();
+			m_BufferStart = m_BufferEnd = FileOffset + Size;
+		}
+
 		m_Base.Write(Data, Size, FileOffset);
 		return;
 	}
@@ -872,7 +794,7 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path)
 	{
 		uint64_t			  Offset		= 0;
 		static const uint64_t BufferingSize = 256u * 1024u;
-		//		BasicFileWriter		  BufferedOutput(BlockFile, BufferingSize / 2);
+		BasicFileWriter		  BufferedOutput(Temp, Min(BufferingSize, BufferSize));
 		for (const SharedBuffer& Segment : Buffer.GetSegments())
 		{
 			size_t SegmentSize = Segment.GetSize();
@@ -884,14 +806,14 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path)
 						 FileRef.FileChunkOffset,
 						 FileRef.FileChunkSize,
 						 BufferingSize,
-						 [&Temp, &Offset](const void* Data, size_t Size) {
-							 Temp.Write(Data, Size, Offset);
+						 [&BufferedOutput, &Offset](const void* Data, size_t Size) {
+							 BufferedOutput.Write(Data, Size, Offset);
 							 Offset += Size;
 						 });
 			}
 			else
 			{
-				Temp.Write(Segment.GetData(), SegmentSize, Offset);
+				BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset);
 				Offset += SegmentSize;
 			}
 		}
@@ -978,9 +900,9 @@ TEST_CASE("TemporaryFile")
 			TmpFile.CreateTemporary(std::filesystem::current_path(), Ec);
 			CHECK(!Ec);
 			Path = TmpFile.GetPath();
-			CHECK(std::filesystem::exists(Path));
+			CHECK(IsFile(Path));
 		}
-		CHECK(std::filesystem::exists(Path) == false);
+		CHECK(IsFile(Path) == false);
 	}
 
 	SUBCASE("MoveIntoPlace")
@@ -991,11 +913,11 @@ TEST_CASE("TemporaryFile")
 		CHECK(!Ec);
 		std::filesystem::path TempPath	= TmpFile.GetPath();
 		std::filesystem::path FinalPath = std::filesystem::current_path() / "final";
-		CHECK(std::filesystem::exists(TempPath));
+		CHECK(IsFile(TempPath));
 		TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec);
 		CHECK(!Ec);
-		CHECK(std::filesystem::exists(TempPath) == false);
-		CHECK(std::filesystem::exists(FinalPath));
+		CHECK(IsFile(TempPath) == false);
+		CHECK(IsFile(FinalPath));
 	}
 }
 
diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp
index 4a77aa49a..054f0d3a0 100644
--- a/src/zencore/blake3.cpp
+++ b/src/zencore/blake3.cpp
@@ -151,6 +151,28 @@ BLAKE3Stream::Append(const void* data, size_t byteCount)
 	return *this;
 }
 
+BLAKE3Stream&
+BLAKE3Stream::Append(const IoBuffer& Buffer)
+{
+	blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState);
+
+	size_t				  BufferSize	= Buffer.GetSize();
+	static const uint64_t BufferingSize = 256u * 1024u;
+	IoBufferFileReference FileRef;
+	if (BufferSize >= (BufferingSize + BufferingSize / 2) && Buffer.GetFileReference(FileRef))
+	{
+		ScanFile(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, [&b3h](const void* Data, size_t Size) {
+			blake3_hasher_update(b3h, Data, Size);
+		});
+	}
+	else
+	{
+		blake3_hasher_update(b3h, Buffer.GetData(), BufferSize);
+	}
+
+	return *this;
+}
+
 BLAKE3
 BLAKE3Stream::GetHash()
 {
diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp
index d8c8a8584..68ed09549 100644
--- a/src/zencore/compactbinaryjson.cpp
+++ b/src/zencore/compactbinaryjson.cpp
@@ -293,6 +293,7 @@ private:
 		const uint32_t EncodedSize	= Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
 		const size_t   EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
 		Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+		Builder << '"';
 	}
 
 private:
diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp
index 6f53bba69..833649b88 100644
--- a/src/zencore/compactbinaryvalidation.cpp
+++ b/src/zencore/compactbinaryvalidation.cpp
@@ -135,6 +135,37 @@ ValidateCbFloat64(MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
 }
 
 /**
+ * Validate and read a fixed-size value from the view.
+ *
+ * Modifies the view to start at the end of the value, and adds error flags if applicable.
+ */
+static MemoryView
+ValidateCbFixedValue(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, uint64_t Size)
+{
+	ZEN_UNUSED(Mode);
+
+	const MemoryView Value = View.Left(Size);
+	View += Size;
+	if (Value.GetSize() < Size)
+	{
+		AddError(Error, CbValidateError::OutOfBounds);
+	}
+	return Value;
+};
+
+/**
+ * Validate and read a value from the view where the view begins with the value size.
+ *
+ * Modifies the view to start at the end of the value, and adds error flags if applicable.
+ */
+static MemoryView
+ValidateCbDynamicValue(MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+{
+	const uint64_t ValueSize = ValidateCbUInt(View, Mode, Error);
+	return ValidateCbFixedValue(View, Mode, Error, ValueSize);
+}
+
+/**
  * Validate and read a string from the view.
  *
  * Modifies the view to start at the end of the string, and adds error flags if applicable.
@@ -378,8 +409,20 @@ ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, c
 			ValidateFixedPayload(12);
 			break;
 		case CbFieldType::CustomById:
+			{
+				MemoryView Value = ValidateCbDynamicValue(View, Mode, Error);
+				ValidateCbUInt(Value, Mode, Error);
+			}
+			break;
 		case CbFieldType::CustomByName:
-			ZEN_NOT_IMPLEMENTED();	// TODO: FIX!
+			{
+				MemoryView			   Value	= ValidateCbDynamicValue(View, Mode, Error);
+				const std::string_view TypeName = ValidateCbString(Value, Mode, Error);
+				if (TypeName.empty() && !EnumHasAnyFlags(Error, CbValidateError::OutOfBounds))
+				{
+					AddError(Error, CbValidateError::InvalidType);
+				}
+			}
 			break;
 	}
 
diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 88c3bb5b9..d9f381811 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -7,6 +7,7 @@
 #include <zencore/compositebuffer.h>
 #include <zencore/crc32.h>
 #include <zencore/endian.h>
+#include <zencore/filesystem.h>
 #include <zencore/intmath.h>
 #include <zencore/iohash.h>
 #include <zencore/stream.h>
@@ -158,9 +159,10 @@ class BaseEncoder
 {
 public:
 	[[nodiscard]] virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0;
-	[[nodiscard]] virtual bool			  CompressToStream(const CompositeBuffer&												RawData,
-														   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
-														   uint64_t BlockSize = DefaultBlockSize) const							= 0;
+	[[nodiscard]] virtual bool			  CompressToStream(
+				   const CompositeBuffer&																							RawData,
+				   std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+				   uint64_t BlockSize = DefaultBlockSize) const = 0;
 };
 
 class BaseDecoder
@@ -189,11 +191,13 @@ public:
 												  uint64_t			   RawOffset,
 												  uint64_t			   RawSize) const = 0;
 
-	virtual bool DecompressToStream(const BufferHeader&													 Header,
-									const CompositeBuffer&												 CompressedData,
-									uint64_t															 RawOffset,
-									uint64_t															 RawSize,
-									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const = 0;
+	virtual bool DecompressToStream(
+		const BufferHeader&																								 Header,
+		const CompositeBuffer&																							 CompressedData,
+		uint64_t																										 RawOffset,
+		uint64_t																										 RawSize,
+		std::function<bool(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback)
+		const = 0;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -207,13 +211,50 @@ public:
 		return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned());
 	}
 
-	[[nodiscard]] virtual bool CompressToStream(const CompositeBuffer&												 RawData,
-												std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
-												uint64_t /* BlockSize */) const final
+	[[nodiscard]] virtual bool CompressToStream(
+		const CompositeBuffer&																							 RawData,
+		std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+		uint64_t /* BlockSize */) const final
 	{
-		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData));
-		Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize())));
-		Callback(HeaderData.GetSize(), RawData);
+		const uint64_t HeaderSize = CompressedBuffer::GetHeaderSizeForNoneEncoder();
+
+		uint64_t	 RawOffset = 0;
+		BLAKE3Stream HashStream;
+
+		for (const SharedBuffer& Segment : RawData.GetSegments())
+		{
+			IoBufferFileReference FileRef		= {nullptr, 0, 0};
+			IoBuffer			  SegmentBuffer = Segment.AsIoBuffer();
+			if (SegmentBuffer.GetFileReference(FileRef))
+			{
+				ZEN_ASSERT(FileRef.FileHandle != nullptr);
+
+				ScanFile(FileRef.FileHandle,
+						 FileRef.FileChunkOffset,
+						 FileRef.FileChunkSize,
+						 512u * 1024u,
+						 [&](const void* Data, size_t Size) {
+							 HashStream.Append(Data, Size);
+							 CompositeBuffer Tmp(SharedBuffer::MakeView(Data, Size));
+							 Callback(RawOffset, Size, HeaderSize + RawOffset, Tmp);
+							 RawOffset += Size;
+						 });
+			}
+			else
+			{
+				const uint64_t Size = SegmentBuffer.GetSize();
+				HashStream.Append(SegmentBuffer);
+				Callback(RawOffset, Size, HeaderSize + RawOffset, CompositeBuffer(Segment));
+				RawOffset += Size;
+			}
+		}
+
+		ZEN_ASSERT(RawOffset == RawData.GetSize());
+
+		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), HashStream.GetHash());
+		ZEN_ASSERT(HeaderData.GetSize() == HeaderSize);
+		Callback(0, 0, 0, CompositeBuffer(HeaderData.MoveToShared()));
+
 		return true;
 	}
 };
@@ -283,21 +324,41 @@ public:
 
 	[[nodiscard]] uint64_t GetHeaderSize(const BufferHeader&) const final { return sizeof(BufferHeader); }
 
-	virtual bool DecompressToStream(const BufferHeader&													 Header,
-									const CompositeBuffer&												 CompressedData,
-									uint64_t															 RawOffset,
-									uint64_t															 RawSize,
-									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final
+	virtual bool DecompressToStream(
+		const BufferHeader&																								 Header,
+		const CompositeBuffer&																							 CompressedData,
+		uint64_t																										 RawOffset,
+		uint64_t																										 RawSize,
+		std::function<bool(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback)
+		const final
 	{
 		if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() &&
 			Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize &&
 			(RawOffset + RawSize) <= Header.TotalRawSize)
 		{
-			if (!Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)))
+			bool				  Result  = true;
+			IoBufferFileReference FileRef = {nullptr, 0, 0};
+			if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 			{
-				return false;
+				ZEN_ASSERT(FileRef.FileHandle != nullptr);
+				uint64_t CallbackOffset = 0;
+				ScanFile(FileRef.FileHandle, sizeof(BufferHeader) + RawOffset, RawSize, 512u * 1024u, [&](const void* Data, size_t Size) {
+					if (Result)
+					{
+						CompositeBuffer Tmp(SharedBuffer::MakeView(Data, Size));
+						Result = Callback(sizeof(BufferHeader) + RawOffset + CallbackOffset, Size, CallbackOffset, Tmp);
+					}
+					CallbackOffset += Size;
+				});
+				return Result;
+			}
+			else
+			{
+				return Callback(sizeof(BufferHeader) + RawOffset,
+								RawSize,
+								0,
+								CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize));
 			}
-			return true;
 		}
 		return false;
 	}
@@ -309,9 +370,10 @@ class BlockEncoder : public BaseEncoder
 {
 public:
 	virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize) const final;
-	virtual bool			CompressToStream(const CompositeBuffer&												  RawData,
-											 std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
-											 uint64_t															  BlockSize) const final;
+	virtual bool			CompressToStream(
+				   const CompositeBuffer&																							RawData,
+				   std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+				   uint64_t BlockSize) const final;
 
 protected:
 	virtual CompressionMethod GetMethod() const															 = 0;
@@ -460,9 +522,10 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize)
 }
 
 bool
-BlockEncoder::CompressToStream(const CompositeBuffer&												RawData,
-							   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
-							   uint64_t																BlockSize = DefaultBlockSize) const
+BlockEncoder::CompressToStream(
+	const CompositeBuffer&																							 RawData,
+	std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+	uint64_t BlockSize = DefaultBlockSize) const
 {
 	ZEN_ASSERT(IsPow2(BlockSize) && (BlockSize <= (1u << 31)));
 
@@ -504,13 +567,17 @@ BlockEncoder::CompressToStream(const CompositeBuffer&												RawData,
 				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
 				if (RawBlockSize <= CompressedBlockSize)
 				{
-					Callback(FullHeaderSize + CompressedSize,
+					Callback(FileRef.FileChunkOffset + RawOffset,
+							 RawBlockSize,
+							 FullHeaderSize + CompressedSize,
 							 CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlockCopy.GetView().GetData(), RawBlockSize)));
 					CompressedBlockSize = RawBlockSize;
 				}
 				else
 				{
-					Callback(FullHeaderSize + CompressedSize,
+					Callback(FileRef.FileChunkOffset + RawOffset,
+							 RawBlockSize,
+							 FullHeaderSize + CompressedSize,
 							 CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize)));
 				}
 
@@ -540,12 +607,17 @@ BlockEncoder::CompressToStream(const CompositeBuffer&												RawData,
 				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
 				if (RawBlockSize <= CompressedBlockSize)
 				{
-					Callback(FullHeaderSize + CompressedSize, CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlock.GetData(), RawBlockSize)));
+					Callback(RawOffset,
+							 RawBlockSize,
+							 FullHeaderSize + CompressedSize,
+							 CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlock.GetData(), RawBlockSize)));
 					CompressedBlockSize = RawBlockSize;
 				}
 				else
 				{
-					Callback(FullHeaderSize + CompressedSize,
+					Callback(RawOffset,
+							 RawBlockSize,
+							 FullHeaderSize + CompressedSize,
 							 CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize)));
 				}
 
@@ -582,7 +654,7 @@ BlockEncoder::CompressToStream(const CompositeBuffer&												RawData,
 	HeaderBuffer.GetMutableView().Mid(sizeof(BufferHeader), MetaSize).CopyFrom(MakeMemoryView(CompressedBlockSizes));
 	Header.Write(HeaderBuffer.GetMutableView());
 
-	Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderBuffer.GetData(), HeaderBuffer.GetSize())));
+	Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderBuffer.GetData(), HeaderBuffer.GetSize())));
 	return true;
 }
 
@@ -615,11 +687,13 @@ public:
 									   MutableMemoryView	  RawView,
 									   uint64_t				  RawOffset) const final;
 
-	virtual bool DecompressToStream(const BufferHeader&													 Header,
-									const CompositeBuffer&												 CompressedData,
-									uint64_t															 RawOffset,
-									uint64_t															 RawSize,
-									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final;
+	virtual bool DecompressToStream(
+		const BufferHeader&																								 Header,
+		const CompositeBuffer&																							 CompressedData,
+		uint64_t																										 RawOffset,
+		uint64_t																										 RawSize,
+		std::function<bool(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback)
+		const final;
 
 protected:
 	virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0;
@@ -743,11 +817,12 @@ BlockDecoder::DecompressToComposite(const BufferHeader& Header, const CompositeB
 }
 
 bool
-BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
-								 const CompositeBuffer&												  CompressedData,
-								 uint64_t															  RawOffset,
-								 uint64_t															  RawSize,
-								 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+BlockDecoder::DecompressToStream(
+	const BufferHeader&																								 Header,
+	const CompositeBuffer&																							 CompressedData,
+	uint64_t																										 RawOffset,
+	uint64_t																										 RawSize,
+	std::function<bool(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
 {
 	if (Header.TotalCompressedSize != CompressedData.GetSize())
 	{
@@ -817,7 +892,9 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 					Source.Detach();
 					return false;
 				}
-				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+				if (!Callback(FileRef.FileChunkOffset + CompressedOffset,
+							  CompressedBlockSize,
+							  BlockIndex * BlockSize + OffsetInFirstBlock,
 							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
 				{
 					Source.Detach();
@@ -827,6 +904,8 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 			else
 			{
 				if (!Callback(
+						FileRef.FileChunkOffset + CompressedOffset,
+						BytesToUncompress,
 						BlockIndex * BlockSize + OffsetInFirstBlock,
 						CompositeBuffer(
 							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
@@ -870,7 +949,9 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 				{
 					return false;
 				}
-				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+				if (!Callback(CompressedOffset,
+							  UncompressedBlockSize,
+							  BlockIndex * BlockSize + OffsetInFirstBlock,
 							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
 				{
 					return false;
@@ -879,6 +960,8 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 			else
 			{
 				if (!Callback(
+						CompressedOffset,
+						BytesToUncompress,
 						BlockIndex * BlockSize + OffsetInFirstBlock,
 						CompositeBuffer(
 							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
@@ -1778,11 +1861,12 @@ CompressedBuffer::Compress(const SharedBuffer&	 RawData,
 }
 
 bool
-CompressedBuffer::CompressToStream(const CompositeBuffer&												RawData,
-								   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
-								   OodleCompressor														Compressor,
-								   OodleCompressionLevel												CompressionLevel,
-								   uint64_t																BlockSize)
+CompressedBuffer::CompressToStream(
+	const CompositeBuffer&																							 RawData,
+	std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+	OodleCompressor																									 Compressor,
+	OodleCompressionLevel																							 CompressionLevel,
+	uint64_t																										 BlockSize)
 {
 	using namespace detail;
 
@@ -1995,9 +2079,10 @@ CompressedBuffer::DecompressToComposite() const
 }
 
 bool
-CompressedBuffer::DecompressToStream(uint64_t															  RawOffset,
-									 uint64_t															  RawSize,
-									 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+CompressedBuffer::DecompressToStream(
+	uint64_t																										 RawOffset,
+	uint64_t																										 RawSize,
+	std::function<bool(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
 {
 	using namespace detail;
 	if (CompressedData)
diff --git a/src/zencore/except.cpp b/src/zencore/except.cpp
index d5eabea9d..610b0ced5 100644
--- a/src/zencore/except.cpp
+++ b/src/zencore/except.cpp
@@ -47,7 +47,7 @@ ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string
 {
 	if (HRESULT_FACILITY(hRes) == FACILITY_WIN32)
 	{
-		throw std::system_error(std::error_code(hRes & 0xffff, std::system_category()), std::string(Message));
+		throw std::system_error(std::error_code(HRESULT_CODE(hRes), std::system_category()), std::string(Message));
 	}
 	else
 	{
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 05e2bf049..46337ffc8 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -33,6 +33,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #	include <dirent.h>
 #	include <fcntl.h>
 #	include <sys/resource.h>
+#	include <sys/mman.h>
 #	include <sys/stat.h>
 #	include <pwd.h>
 #	include <unistd.h>
@@ -43,6 +44,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #	include <fcntl.h>
 #	include <libproc.h>
 #	include <sys/resource.h>
+#	include <sys/mman.h>
 #	include <sys/stat.h>
 #	include <sys/syslimits.h>
 #	include <pwd.h>
@@ -86,16 +88,9 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
 }
 
 bool
-CreateDirectories(const wchar_t* Dir)
+CreateDirectories(const wchar_t* Path)
 {
-	// This may be suboptimal, in that it appears to try and create directories
-	// from the root on up instead of from some directory which is known to
-	// be present
-	//
-	// We should implement a smarter version at some point since this can be
-	// pretty expensive in aggregate
-
-	return std::filesystem::create_directories(Dir);
+	return CreateDirectories(std::filesystem::path(Path));
 }
 
 // Erase all files and directories in a given directory, leaving an empty directory
@@ -212,75 +207,377 @@ DeleteDirectoriesInternal(const wchar_t* DirPath)
 bool
 CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles)
 {
-	if (std::filesystem::exists(DirPath))
+	if (IsDir(DirPath))
 	{
 		return WipeDirectory(DirPath, KeepDotFiles);
 	}
 
 	return CreateDirectories(DirPath);
 }
+
+#endif	// ZEN_PLATFORM_WINDOWS
+
+#if ZEN_PLATFORM_WINDOWS
+const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY;
+#else
+const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001;
 #endif	// ZEN_PLATFORM_WINDOWS
 
+const uint32_t FileModeWriteEnableFlags = 0222;
+
 bool
-CreateDirectories(const std::filesystem::path& Dir)
+IsFileAttributeReadOnly(uint32_t FileAttributes)
 {
-	if (Dir.string().ends_with(":"))
+#if ZEN_PLATFORM_WINDOWS
+	return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0;
+#else
+	return (FileAttributes & 0x00000001) != 0;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+IsFileModeReadOnly(uint32_t FileMode)
+{
+	return (FileMode & FileModeWriteEnableFlags) == 0;
+}
+
+uint32_t
+MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly)
+{
+	return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag);
+}
+
+uint32_t
+MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly)
+{
+	return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags);
+}
+
+#if ZEN_PLATFORM_WINDOWS
+
+static DWORD
+WinGetFileAttributes(const std::filesystem::path& Path, std::error_code& Ec)
+{
+	DWORD Attributes = ::GetFileAttributes(Path.native().c_str());
+	if (Attributes == INVALID_FILE_ATTRIBUTES)
 	{
+		DWORD LastError = GetLastError();
+		switch (LastError)
+		{
+			case ERROR_FILE_NOT_FOUND:
+			case ERROR_PATH_NOT_FOUND:
+			case ERROR_BAD_NETPATH:
+			case ERROR_INVALID_DRIVE:
+				break;
+			case ERROR_ACCESS_DENIED:
+				{
+					WIN32_FIND_DATA FindData;
+					HANDLE			FindHandle = ::FindFirstFile(Path.native().c_str(), &FindData);
+					if (FindHandle == INVALID_HANDLE_VALUE)
+					{
+						DWORD LastFindError = GetLastError();
+						if (LastFindError != ERROR_FILE_NOT_FOUND)
+						{
+							Ec = MakeErrorCode(LastError);
+						}
+					}
+					else
+					{
+						FindClose(FindHandle);
+						Attributes = FindData.dwFileAttributes;
+					}
+				}
+				break;
+			default:
+				Ec = MakeErrorCode(LastError);
+				break;
+		}
+	}
+	return Attributes;
+}
+
+#endif	// ZEN_PLATFORM_WINDOWS
+
+bool
+RemoveDirNative(const std::filesystem::path& Path, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	BOOL Success = ::RemoveDirectory(Path.native().c_str());
+	if (!Success)
+	{
+		DWORD LastError = GetLastError();
+		switch (LastError)
+		{
+			case ERROR_FILE_NOT_FOUND:
+			case ERROR_PATH_NOT_FOUND:
+				break;
+			default:
+				Ec = MakeErrorCode(LastError);
+				break;
+		}
 		return false;
 	}
-	while (!std::filesystem::is_directory(Dir))
+	return true;
+#else
+	return std::filesystem::remove(Path, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	const std::filesystem::path::value_type* NativePath = Path.native().c_str();
+	BOOL									 Success	= ::DeleteFile(NativePath);
+	if (!Success)
 	{
-		if (Dir.has_parent_path())
+		if (ForceRemoveReadOnlyFiles)
 		{
-			CreateDirectories(Dir.parent_path());
+			DWORD FileAttributes = WinGetFileAttributes(NativePath, Ec);
+			if (Ec)
+			{
+				return false;
+			}
+
+			if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0)
+			{
+				::SetFileAttributes(NativePath, MakeFileAttributeReadOnly(FileAttributes, false));
+				Success = ::DeleteFile(NativePath);
+			}
 		}
-		std::error_code ErrorCode;
-		std::filesystem::create_directory(Dir, ErrorCode);
-		if (ErrorCode)
+		if (!Success)
 		{
-			throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string()));
+			DWORD LastError = GetLastError();
+			switch (LastError)
+			{
+				case ERROR_FILE_NOT_FOUND:
+				case ERROR_PATH_NOT_FOUND:
+					break;
+				default:
+					Ec = MakeErrorCode(LastError);
+					break;
+			}
+			return false;
+		}
+	}
+	return true;
+#else
+	if (!ForceRemoveReadOnlyFiles)
+	{
+		struct stat Stat;
+		int			err = stat(Path.native().c_str(), &Stat);
+		if (err != 0)
+		{
+			int32_t err = errno;
+			if (err == ENOENT)
+			{
+				Ec.clear();
+				return false;
+			}
+		}
+		const uint32_t Mode = (uint32_t)Stat.st_mode;
+		if (IsFileModeReadOnly(Mode))
+		{
+			Ec = MakeErrorCode(EACCES);
+			return false;
+		}
+	}
+	return std::filesystem::remove(Path, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+static void
+WipeDirectoryContentInternal(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec)
+{
+	DirectoryContent LocalDirectoryContent;
+	GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
+	for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files)
+	{
+		RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			Ec.clear();
+			if (IsFile(LocalFilePath))
+			{
+				RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec);
+			}
+		}
+		if (Ec)
+		{
+			return;
+		}
+	}
+
+	for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories)
+	{
+		WipeDirectoryContentInternal(LocalDirPath, ForceRemoveReadOnlyFiles, Ec);
+		if (Ec)
+		{
+			return;
+		}
+
+		RemoveDirNative(LocalDirPath, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			Ec.clear();
+			if (IsDir(LocalDirPath))
+			{
+				RemoveDirNative(LocalDirPath, Ec);
+			}
+		}
+		if (Ec)
+		{
+			return;
 		}
-		return true;
 	}
-	return false;
 }
 
 bool
-DeleteDirectories(const std::filesystem::path& Dir)
+CreateDirectory(const std::filesystem::path& Path, std::error_code& Ec)
 {
-	std::error_code ErrorCode;
-	return std::filesystem::remove_all(Dir, ErrorCode);
+#if ZEN_PLATFORM_WINDOWS
+	BOOL Success = ::CreateDirectory(Path.native().c_str(), nullptr);
+	if (!Success)
+	{
+		DWORD LastError = GetLastError();
+		switch (LastError)
+		{
+			case ERROR_FILE_EXISTS:
+			case ERROR_ALREADY_EXISTS:
+				break;
+			default:
+				Ec = MakeErrorCode(LastError);
+				break;
+		}
+		return false;
+	}
+	return Success;
+#else
+	return std::filesystem::create_directory(Path, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
 }
 
 bool
-CleanDirectory(const std::filesystem::path& Dir)
+CreateDirectories(const std::filesystem::path& Path)
 {
-	if (std::filesystem::exists(Dir))
+	std::error_code Ec;
+	bool			Success = CreateDirectories(Path, Ec);
+	if (Ec)
 	{
-		bool Success = true;
+		throw std::system_error(Ec, fmt::format("Failed to create directories for '{}'", Path.string()));
+	}
+	return Success;
+}
 
-		for (const auto& Item : std::filesystem::directory_iterator(Dir))
-		{
-			std::error_code ErrorCode;
-			const uintmax_t RemovedCount = std::filesystem::remove_all(Item, ErrorCode);
+bool
+CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec)
+{
+	if (Path.string().ends_with(":"))
+	{
+		return false;
+	}
+	bool Exists = IsDir(Path, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	if (Exists)
+	{
+		return false;
+	}
 
-			Success = Success && !ErrorCode && RemovedCount;
+	if (Path.has_parent_path())
+	{
+		bool Result = CreateDirectories(Path.parent_path(), Ec);
+		if (Ec)
+		{
+			return Result;
 		}
+	}
+	return CreateDirectory(Path, Ec);
+}
 
-		return Success;
+bool
+CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles)
+{
+	std::error_code Ec;
+	bool			Result = CleanDirectory(Path, ForceRemoveReadOnlyFiles, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to clean directory for '{}'", Path.string()));
 	}
+	return Result;
+}
 
-	return CreateDirectories(Dir);
+bool
+CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec)
+{
+	bool Exists = IsDir(Path, Ec);
+	if (Ec)
+	{
+		return Exists;
+	}
+	if (Exists)
+	{
+		WipeDirectoryContentInternal(Path, ForceRemoveReadOnlyFiles, Ec);
+		return false;
+	}
+	return CreateDirectory(Path, Ec);
+}
+
+bool
+DeleteDirectories(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	bool			Result = DeleteDirectories(Path, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to delete directories for '{}'", Path.string()));
+	}
+	return Result;
+}
+
+bool
+DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec)
+{
+	bool Exists = IsDir(Path, Ec);
+	if (Ec)
+	{
+		return Exists;
+	}
+
+	if (Exists)
+	{
+		WipeDirectoryContentInternal(Path, false, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		bool Result = RemoveDirNative(Path, Ec);
+		for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+		{
+			Sleep(100 + int(Retries * 50));
+			Ec.clear();
+			if (IsDir(Path))
+			{
+				Result = RemoveDirNative(Path, Ec);
+			}
+		}
+		return Result;
+	}
+	return false;
 }
 
 bool
-CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir)
+CleanDirectoryExceptDotFiles(const std::filesystem::path& Path)
 {
 #if ZEN_PLATFORM_WINDOWS
 	const bool KeepDotFiles = true;
-	return CleanDirectory(Dir.c_str(), KeepDotFiles);
+	return CleanDirectory(Path.c_str(), KeepDotFiles);
 #else
-	ZEN_UNUSED(Dir);
+	ZEN_UNUSED(Path);
 
 	ZEN_NOT_IMPLEMENTED();
 #endif
@@ -637,7 +934,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 {
 	// Validate arguments
 
-	if (FromPath.empty() || !std::filesystem::is_directory(FromPath))
+	if (FromPath.empty() || !IsDir(FromPath))
 		throw std::runtime_error("invalid CopyTree source directory specified");
 
 	if (ToPath.empty())
@@ -646,16 +943,13 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 	if (Options.MustClone && !SupportsBlockRefCounting(FromPath))
 		throw std::runtime_error(fmt::format("cloning not possible from '{}'", FromPath));
 
-	if (std::filesystem::exists(ToPath))
+	if (IsFile(ToPath))
 	{
-		if (!std::filesystem::is_directory(ToPath))
-		{
-			throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath));
-		}
+		throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath));
 	}
-	else
+	if (!IsDir(ToPath))
 	{
-		std::filesystem::create_directories(ToPath);
+		CreateDirectories(ToPath);
 	}
 
 	if (Options.MustClone && !SupportsBlockRefCounting(ToPath))
@@ -760,6 +1054,100 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 }
 
 void
+WriteFile(void* NativeHandle, const void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec)
+{
+	ZEN_ASSERT(NativeHandle != nullptr);
+
+	Ec.clear();
+
+	while (Size)
+	{
+		const uint64_t NumberOfBytesToWrite = Min(Size, ChunkSize);
+
+#if ZEN_PLATFORM_WINDOWS
+		OVERLAPPED Ovl{};
+
+		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
+		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+
+		DWORD dwNumberOfBytesWritten = 0;
+
+		BOOL Success = ::WriteFile(NativeHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl);
+#else
+		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+		int	 Fd			  = int(uintptr_t(NativeHandle));
+		int	 BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset);
+		bool Success	  = (BytesWritten > 0);
+#endif
+
+		if (!Success)
+		{
+			Ec = MakeErrorCodeFromLastError();
+			return;
+		}
+
+		Size -= NumberOfBytesToWrite;
+		FileOffset += NumberOfBytesToWrite;
+		Data = reinterpret_cast<const uint8_t*>(Data) + NumberOfBytesToWrite;
+	}
+}
+
+void
+ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec)
+{
+	while (Size)
+	{
+		const uint64_t NumberOfBytesToRead = Min(Size, ChunkSize);
+		size_t		   BytesRead		   = 0;
+
+#if ZEN_PLATFORM_WINDOWS
+		OVERLAPPED Ovl{};
+
+		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
+		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+
+		DWORD dwNumberOfBytesRead = 0;
+		BOOL  Success			  = ::ReadFile(NativeHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl);
+		if (Success)
+		{
+			BytesRead = size_t(dwNumberOfBytesRead);
+		}
+		else if ((BytesRead != NumberOfBytesToRead))
+		{
+			Ec = MakeErrorCode(ERROR_HANDLE_EOF);
+			return;
+		}
+		else
+		{
+			Ec = MakeErrorCodeFromLastError();
+			return;
+		}
+#else
+		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+		int		Fd		   = int(uintptr_t(NativeHandle));
+		ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
+		if (ReadResult != -1)
+		{
+			BytesRead = size_t(ReadResult);
+		}
+		else if ((BytesRead != NumberOfBytesToRead))
+		{
+			Ec = MakeErrorCode(EIO);
+			return;
+		}
+		else
+		{
+			Ec = MakeErrorCodeFromLastError();
+			return;
+		}
+#endif
+		Size -= NumberOfBytesToRead;
+		FileOffset += NumberOfBytesToRead;
+		Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+	}
+}
+
+void
 WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount)
 {
 #if ZEN_PLATFORM_WINDOWS
@@ -811,7 +1199,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
 			{
 				Outfile.Close();
 				std::error_code DummyEc;
-				std::filesystem::remove(Path, DummyEc);
+				RemoveFile(Path, DummyEc);
 				ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str());
 			}
 #else
@@ -819,7 +1207,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
 			{
 				close(Fd);
 				std::error_code DummyEc;
-				std::filesystem::remove(Path, DummyEc);
+				RemoveFile(Path, DummyEc);
 				ThrowLastError(fmt::format("File write failed for '{}'", Path));
 			}
 #endif	// ZEN_PLATFORM_WINDOWS
@@ -1172,7 +1560,7 @@ void
 FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor)
 {
 #if ZEN_PLATFORM_WINDOWS
-	uint64_t FileInfoBuffer[8 * 1024];
+	std::vector<uint64_t> FileInfoBuffer(8 * 1024);
 
 	FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo;
 	bool					  Continue = true;
@@ -1183,7 +1571,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 
 	if (FAILED(hRes))
 	{
-		if (hRes == ERROR_FILE_NOT_FOUND || hRes == ERROR_PATH_NOT_FOUND)
+		if (HRESULT_CODE(hRes) == ERROR_FILE_NOT_FOUND || HRESULT_CODE(hRes) == ERROR_PATH_NOT_FOUND)
 		{
 			// Directory no longer exist, treat it as empty
 			return;
@@ -1193,8 +1581,9 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 
 	while (Continue)
 	{
-		BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer);
-		FibClass	 = FileIdBothDirectoryInfo;	 // Set up for next iteration
+		BOOL Success =
+			GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer.data(), (DWORD)(FileInfoBuffer.size() * sizeof(uint64_t)));
+		FibClass = FileIdBothDirectoryInfo;	 // Set up for next iteration
 
 		uint64_t EntryOffset = 0;
 
@@ -1213,7 +1602,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 		for (;;)
 		{
 			const FILE_ID_BOTH_DIR_INFO* DirInfo =
-				reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset);
+				reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer.data()) + EntryOffset);
 
 			std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t));
 
@@ -1338,6 +1727,156 @@ CanonicalPath(std::filesystem::path InPath, std::error_code& Ec)
 #endif
 }
 
+bool
+IsFile(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	bool			Result = IsFile(Path, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a file", Path.string()));
+	}
+	return Result;
+}
+
+bool
+IsFile(const std::filesystem::path& Path, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	DWORD Attributes = WinGetFileAttributes(Path, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	if (Attributes == INVALID_FILE_ATTRIBUTES)
+	{
+		return false;
+	}
+	return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
+#else
+	struct stat Stat;
+	int			err = stat(Path.native().c_str(), &Stat);
+	if (err != 0)
+	{
+		int32_t err = errno;
+		if (err == ENOENT)
+		{
+			Ec.clear();
+			return false;
+		}
+	}
+	if (S_ISREG(Stat.st_mode))
+	{
+		return true;
+	}
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+IsDir(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	bool			Result = IsDir(Path, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a directory", Path.string()));
+	}
+	return Result;
+}
+
+bool
+IsDir(const std::filesystem::path& Path, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	DWORD Attributes = WinGetFileAttributes(Path, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	if (Attributes == INVALID_FILE_ATTRIBUTES)
+	{
+		return false;
+	}
+	return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY;
+#else
+	struct stat Stat;
+	int			err = stat(Path.native().c_str(), &Stat);
+	if (err != 0)
+	{
+		int32_t err = errno;
+		if (err == ENOENT)
+		{
+			Ec.clear();
+			return false;
+		}
+	}
+	if (S_ISDIR(Stat.st_mode))
+	{
+		return true;
+	}
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+RemoveFile(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	bool			Success = RemoveFile(Path, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to remove file '{}'", Path.string()));
+	}
+	return Success;
+}
+
+bool
+RemoveFile(const std::filesystem::path& Path, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return RemoveFileNative(Path, false, Ec);
+#else
+	bool IsDirectory = std::filesystem::is_directory(Path, Ec);
+	if (IsDirectory)
+	{
+		Ec = MakeErrorCode(EPERM);
+		return false;
+	}
+	Ec.clear();
+	return RemoveFileNative(Path, false, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+RemoveDir(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	bool			Success = RemoveDir(Path, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to remove directory '{}'", Path.string()));
+	}
+	return Success;
+}
+
+bool
+RemoveDir(const std::filesystem::path& Path, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return RemoveDirNative(Path, Ec);
+#else
+	bool IsFile = std::filesystem::is_regular_file(Path, Ec);
+	if (IsFile)
+	{
+		Ec = MakeErrorCode(EPERM);
+		return false;
+	}
+	Ec.clear();
+	return RemoveDirNative(Path, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
 std::filesystem::path
 PathFromHandle(void* NativeHandle, std::error_code& Ec)
 {
@@ -1435,23 +1974,84 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec)
 }
 
 uint64_t
-FileSizeFromHandle(void* NativeHandle)
+FileSizeFromPath(const std::filesystem::path& Path)
+{
+	std::error_code Ec;
+	uint64_t		Size = FileSizeFromPath(Path, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to get file size for path '{}'", Path.string()));
+	}
+	return Size;
+}
+
+uint64_t
+FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec)
 {
-	uint64_t FileSize = ~0ull;
+#if ZEN_PLATFORM_WINDOWS
+	void* Handle = ::CreateFile(Path.native().c_str(),
+								GENERIC_READ,
+								FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+								nullptr,
+								OPEN_EXISTING,
+								0,
+								nullptr);
+	if (Handle == INVALID_HANDLE_VALUE)
+	{
+		DWORD LastError = GetLastError();
+		Ec				= MakeErrorCode(LastError);
+		return 0;
+	}
+	auto		  _ = MakeGuard([Handle]() { CloseHandle(Handle); });
+	LARGE_INTEGER FileSize;
+	BOOL		  Success = GetFileSizeEx(Handle, &FileSize);
+	if (!Success)
+	{
+		Ec = MakeErrorCodeFromLastError();
+		return 0;
+	}
+	return FileSize.QuadPart;
+#else
+	return std::filesystem::file_size(Path, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
 
+uint64_t
+FileSizeFromHandle(void* NativeHandle, std::error_code& Ec)
+{
 #if ZEN_PLATFORM_WINDOWS
 	BY_HANDLE_FILE_INFORMATION Bhfh = {};
 	if (GetFileInformationByHandle(NativeHandle, &Bhfh))
 	{
-		FileSize = uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow;
+		return uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow;
+	}
+	else
+	{
+		Ec = MakeErrorCodeFromLastError();
+		return 0;
 	}
 #else
-	int			Fd = int(intptr_t(NativeHandle));
+	int Fd = int(intptr_t(NativeHandle));
+	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
 	struct stat Stat;
-	fstat(Fd, &Stat);
-	FileSize	   = size_t(Stat.st_size);
+	if (fstat(Fd, &Stat) == -1)
+	{
+		Ec = MakeErrorCodeFromLastError();
+		return 0;
+	}
+	return uint64_t(Stat.st_size);
 #endif
+}
 
+uint64_t
+FileSizeFromHandle(void* NativeHandle)
+{
+	std::error_code Ec;
+	uint64_t		FileSize = FileSizeFromHandle(NativeHandle, Ec);
+	if (Ec)
+	{
+		return ~0ull;
+	}
 	return FileSize;
 }
 
@@ -1483,7 +2083,13 @@ GetModificationTickFromPath(const std::filesystem::path& Filename)
 	// PathFromHandle
 	void* Handle;
 #if ZEN_PLATFORM_WINDOWS
-	Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
+	Handle = CreateFileW(Filename.c_str(),
+						 GENERIC_READ,
+						 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+						 nullptr,
+						 OPEN_EXISTING,
+						 0,
+						 nullptr);
 	if (Handle == INVALID_HANDLE_VALUE)
 	{
 		ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename));
@@ -1493,7 +2099,7 @@ GetModificationTickFromPath(const std::filesystem::path& Filename)
 	uint64_t		ModificatonTick = GetModificationTickFromHandle(Handle, Ec);
 	if (Ec)
 	{
-		ThrowSystemError(Ec.value(), Ec.message());
+		throw std::system_error(Ec, fmt::format("Failed to get modification tick for path '{}'", Filename.string()));
 	}
 	return ModificatonTick;
 #else
@@ -1507,6 +2113,102 @@ GetModificationTickFromPath(const std::filesystem::path& Filename)
 #endif
 }
 
+bool
+TryGetFileProperties(const std::filesystem::path& Path,
+					 uint64_t&					  OutSize,
+					 uint64_t&					  OutModificationTick,
+					 uint32_t&					  OutNativeModeOrAttributes)
+{
+#if ZEN_PLATFORM_WINDOWS
+	const std::filesystem::path::value_type* NativePath = Path.native().c_str();
+	{
+		void* Handle = CreateFileW(NativePath,
+								   GENERIC_READ,
+								   FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+								   nullptr,
+								   OPEN_EXISTING,
+								   0,
+								   nullptr);
+		if (Handle == INVALID_HANDLE_VALUE)
+		{
+			return false;
+		}
+		auto _ = MakeGuard([Handle]() { CloseHandle(Handle); });
+
+		BY_HANDLE_FILE_INFORMATION Bhfh = {};
+		if (!GetFileInformationByHandle(Handle, &Bhfh))
+		{
+			return false;
+		}
+		OutSize					  = uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow;
+		OutModificationTick		  = ((uint64_t(Bhfh.ftLastWriteTime.dwHighDateTime) << 32) | Bhfh.ftLastWriteTime.dwLowDateTime);
+		OutNativeModeOrAttributes = Bhfh.dwFileAttributes;
+		return true;
+	}
+#else
+	struct stat Stat;
+	int			err = stat(Path.native().c_str(), &Stat);
+	if (err)
+	{
+		return false;
+	}
+	OutModificationTick		  = gsl::narrow<uint64_t>(Stat.st_mtime);
+	OutSize					  = size_t(Stat.st_size);
+	OutNativeModeOrAttributes = (uint32_t)Stat.st_mode;
+	return true;
+#endif
+}
+
+void
+RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
+{
+	std::error_code Ec;
+	RenameFile(SourcePath, TargetPath, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to rename path from '{}' to '{}'", SourcePath.string(), TargetPath.string()));
+	}
+}
+
+void
+RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	BOOL Success = ::MoveFileEx(SourcePath.native().c_str(), TargetPath.native().c_str(), MOVEFILE_REPLACE_EXISTING);
+	if (!Success)
+	{
+		Ec = MakeErrorCodeFromLastError();
+	}
+#else
+	return std::filesystem::rename(SourcePath, TargetPath, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+void
+RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath)
+{
+	std::error_code Ec;
+	RenameDirectory(SourcePath, TargetPath, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to rename directory from '{}' to '{}'", SourcePath.string(), TargetPath.string()));
+	}
+}
+
+void
+RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	BOOL Success = ::MoveFile(SourcePath.native().c_str(), TargetPath.native().c_str());
+	if (!Success)
+	{
+		Ec = MakeErrorCodeFromLastError();
+	}
+#else
+	return std::filesystem::rename(SourcePath, TargetPath, Ec);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
 std::filesystem::path
 GetRunningExecutablePath()
 {
@@ -1570,6 +2272,43 @@ MaximizeOpenFileCount()
 #endif
 }
 
+bool
+PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize)
+{
+	bool Result = true;
+#if ZEN_PLATFORM_WINDOWS
+
+	BY_HANDLE_FILE_INFORMATION Information;
+	if (GetFileInformationByHandle(FileHandle, &Information))
+	{
+		if ((Information.dwFileAttributes & FILE_ATTRIBUTE_SPARSE_FILE) == 0)
+		{
+			DWORD _	 = 0;
+			BOOL  Ok = DeviceIoControl(FileHandle, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &_, nullptr);
+			if (!Ok)
+			{
+				std::error_code DummyEc;
+				ZEN_DEBUG("Unable to set sparse mode for file '{}'", PathFromHandle(FileHandle, DummyEc));
+				Result = false;
+			}
+		}
+	}
+
+	FILE_ALLOCATION_INFO AllocationInfo	   = {};
+	AllocationInfo.AllocationSize.QuadPart = LONGLONG(FinalSize);
+	if (!SetFileInformationByHandle(FileHandle, FileAllocationInfo, &AllocationInfo, DWORD(sizeof(AllocationInfo))))
+	{
+		std::error_code DummyEc;
+		ZEN_DEBUG("Unable to set file allocation size to {} for file '{}'", FinalSize, PathFromHandle(FileHandle, DummyEc));
+		Result = false;
+	}
+
+#else	// ZEN_PLATFORM_WINDOWS
+	ZEN_UNUSED(FileHandle, FinalSize);
+#endif	// ZEN_PLATFORM_WINDOWS
+	return Result;
+}
+
 void
 GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent)
 {
@@ -1706,12 +2445,17 @@ GetDirectoryContent(const std::filesystem::path& RootDir,
 											 RelativeRoot	  = RelativeRoot / DirectoryName]() {
 						ZEN_ASSERT(Visitor);
 						auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
+						try
 						{
 							MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
 							FileSystemTraversal	 Traversal;
 							Traversal.TraverseFileSystem(Path, SubVisitor);
 							Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
 						}
+						catch (const std::exception& Ex)
+						{
+							ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'", Path / RelativeRoot, Ex.what());
+						}
 					});
 				}
 				catch (const std::exception Ex)
@@ -1793,7 +2537,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles)
 	};
 
 	auto IsEmpty = [](const std::filesystem::path& Path, std::error_code& Ec) -> bool {
-		bool Exists = std::filesystem::exists(Path, Ec);
+		bool Exists = IsFile(Path, Ec);
 		if (Ec)
 		{
 			return false;
@@ -1802,7 +2546,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles)
 		{
 			return true;
 		}
-		uintmax_t Size = std::filesystem::file_size(Path, Ec);
+		uintmax_t Size = FileSizeFromPath(Path, Ec);
 		if (Ec)
 		{
 			return false;
@@ -1821,17 +2565,17 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles)
 		for (auto i = MaxFiles; i > 0; i--)
 		{
 			std::filesystem::path src = GetFileName(i - 1);
-			if (!std::filesystem::exists(src))
+			if (!IsFile(src))
 			{
 				continue;
 			}
 			std::error_code		  DummyEc;
 			std::filesystem::path target = GetFileName(i);
-			if (std::filesystem::exists(target, DummyEc))
+			if (IsFile(target, DummyEc))
 			{
-				std::filesystem::remove(target, DummyEc);
+				RemoveFile(target, DummyEc);
 			}
-			std::filesystem::rename(src, target, DummyEc);
+			RenameFile(src, target, DummyEc);
 		}
 	}
 
@@ -1868,16 +2612,16 @@ RotateDirectories(const std::filesystem::path& DirectoryName, std::size_t MaxDir
 	{
 		const std::filesystem::path SourcePath = GetPathForIndex(i - 1);
 
-		if (std::filesystem::exists(SourcePath))
+		if (IsDir(SourcePath))
 		{
 			std::filesystem::path TargetPath = GetPathForIndex(i);
 
 			std::error_code DummyEc;
-			if (std::filesystem::exists(TargetPath, DummyEc))
+			if (IsDir(TargetPath, DummyEc))
 			{
-				std::filesystem::remove_all(TargetPath, DummyEc);
+				DeleteDirectories(TargetPath, DummyEc);
 			}
-			std::filesystem::rename(SourcePath, TargetPath, DummyEc);
+			RenameDirectory(SourcePath, TargetPath, DummyEc);
 		}
 	}
 
@@ -1936,22 +2680,40 @@ PickDefaultSystemRootDirectory()
 #if ZEN_PLATFORM_WINDOWS
 
 uint32_t
+GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec)
+{
+	return WinGetFileAttributes(Filename, Ec);
+}
+
+uint32_t
 GetFileAttributes(const std::filesystem::path& Filename)
 {
-	DWORD Attributes = ::GetFileAttributes(Filename.native().c_str());
-	if (Attributes == INVALID_FILE_ATTRIBUTES)
+	std::error_code Ec;
+	uint32_t		Result = zen::GetFileAttributes(Filename, Ec);
+	if (Ec)
 	{
-		ThrowLastError(fmt::format("failed to get attributes of file {}", Filename));
+		throw std::system_error(Ec, fmt::format("failed to get attributes of file '{}'", Filename.string()));
 	}
-	return (uint32_t)Attributes;
+	return Result;
 }
 
 void
-SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes)
+SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec)
 {
 	if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0)
 	{
-		ThrowLastError(fmt::format("failed to set attributes of file {}", Filename));
+		Ec = MakeErrorCodeFromLastError();
+	}
+}
+
+void
+SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes)
+{
+	std::error_code Ec;
+	zen::SetFileAttributes(Filename, Attributes, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed to set attributes of file {}", Filename.string()));
 	}
 }
 
@@ -1962,98 +2724,318 @@ SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes)
 uint32_t
 GetFileMode(const std::filesystem::path& Filename)
 {
+	std::error_code Ec;
+	uint32_t		Result = GetFileMode(Filename, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to get mode of file {}", Filename));
+	}
+	return Result;
+}
+
+uint32_t
+GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec)
+{
 	struct stat Stat;
 	int			err = stat(Filename.native().c_str(), &Stat);
 	if (err)
 	{
-		ThrowLastError(fmt::format("Failed to get mode of file {}", Filename));
+		Ec = MakeErrorCodeFromLastError();
+		return 0;
 	}
 	return (uint32_t)Stat.st_mode;
 }
 
 void
-SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes)
+SetFileMode(const std::filesystem::path& Filename, uint32_t Mode)
+{
+	std::error_code Ec;
+	SetFileMode(Filename, Mode, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to set mode of file {}", Filename));
+	}
+}
+
+void
+SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec)
 {
-	int err = chmod(Filename.native().c_str(), (mode_t)Attributes);
+	int err = chmod(Filename.native().c_str(), (mode_t)Mode);
 	if (err)
 	{
-		ThrowLastError(fmt::format("Failed to set mode of file {}", Filename));
+		Ec = MakeErrorCodeFromLastError();
 	}
 }
 
 #endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 
-#if ZEN_PLATFORM_WINDOWS
-const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY;
-#else
-const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001;
-#endif	// ZEN_PLATFORM_WINDOWS
-
-const uint32_t FileModeWriteEnableFlags = 0222;
-
 bool
-IsFileAttributeReadOnly(uint32_t FileAttributes)
+SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec)
 {
 #if ZEN_PLATFORM_WINDOWS
-	return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0;
-#else
-	return (FileAttributes & 0x00000001) != 0;
+	uint32_t CurrentAttributes = GetFileAttributes(Filename, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	if (CurrentAttributes == INVALID_FILE_ATTRIBUTES)
+	{
+		Ec = MakeErrorCode(ERROR_FILE_NOT_FOUND);
+		return false;
+	}
+	uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly);
+	if (CurrentAttributes != NewAttributes)
+	{
+		SetFileAttributes(Filename, NewAttributes, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		return true;
+	}
 #endif	// ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	uint32_t CurrentMode = GetFileMode(Filename, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+	uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly);
+	if (CurrentMode != NewMode)
+	{
+		SetFileMode(Filename, NewMode, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		return true;
+	}
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	return false;
 }
 
 bool
-IsFileModeReadOnly(uint32_t FileMode)
-{
-	return (FileMode & FileModeWriteEnableFlags) == 0;
-}
-
-uint32_t
-MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly)
+SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly)
 {
-	return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag);
+	std::error_code Ec;
+	bool			Result = SetFileReadOnly(Filename, ReadOnly, Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed to set read only mode of file '{}'", Filename.string()));
+	}
+	return Result;
 }
 
-uint32_t
-MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly)
+class SharedMemoryImpl : public SharedMemory
 {
-	return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags);
-}
+public:
+	struct Data
+	{
+		void*		Handle	= nullptr;
+		void*		DataPtr = nullptr;
+		size_t		Size	= 0;
+		std::string Name;
+	};
 
-bool
-SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly)
-{
+	static Data Open(std::string_view Name, size_t Size, bool SystemGlobal)
+	{
 #if ZEN_PLATFORM_WINDOWS
-	uint32_t CurrentAttributes = GetFileAttributes(Filename);
-	uint32_t NewAttributes	   = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly);
-	if (CurrentAttributes != NewAttributes)
+		std::wstring InstanceMapName = Utf8ToWide(fmt::format("{}\\{}", SystemGlobal ? "Global" : "Local", Name));
+
+		HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, InstanceMapName.c_str());
+		if (hMap == NULL)
+		{
+			return {};
+		}
+		void* pBuf = MapViewOfFile(hMap,				 // handle to map object
+								   FILE_MAP_ALL_ACCESS,	 // read/write permission
+								   0,					 // offset high
+								   0,					 // offset low
+								   DWORD(Size));		 // size
+
+		if (pBuf == NULL)
+		{
+			CloseHandle(hMap);
+		}
+		return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)};
+#endif	// ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+		ZEN_UNUSED(SystemGlobal);
+		std::string InstanceMapName = fmt::format("/{}", Name);
+
+		int Fd = shm_open(InstanceMapName.c_str(), O_RDWR, 0666);
+		if (Fd < 0)
+		{
+			return {};
+		}
+		void* hMap = (void*)intptr_t(Fd);
+
+		struct stat Stat;
+		fstat(Fd, &Stat);
+
+		if (size_t(Stat.st_size) < Size)
+		{
+			close(Fd);
+			return {};
+		}
+
+		void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0);
+		if (pBuf == MAP_FAILED)
+		{
+			close(Fd);
+			return {};
+		}
+		return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)};
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	}
+
+	static Data Create(std::string_view Name, size_t Size, bool SystemGlobal)
 	{
-		SetFileAttributes(Filename, NewAttributes);
-		return true;
+#if ZEN_PLATFORM_WINDOWS
+		std::wstring InstanceMapName = Utf8ToWide(fmt::format("{}\\{}", SystemGlobal ? "Global" : "Local", Name));
+
+		SECURITY_ATTRIBUTES m_Attributes{};
+		SECURITY_DESCRIPTOR m_Sd{};
+
+		m_Attributes.nLength		= sizeof m_Attributes;
+		m_Attributes.bInheritHandle = false;  // Disable inheritance
+
+		const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION);
+
+		if (Success)
+		{
+			if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE))
+			{
+				ThrowLastError("SetSecurityDescriptorDacl failed");
+			}
+
+			m_Attributes.lpSecurityDescriptor = &m_Sd;
+		}
+
+		HANDLE hMap = CreateFileMapping(INVALID_HANDLE_VALUE,  // use paging file
+										&m_Attributes,		   // allow anyone to access
+										PAGE_READWRITE,		   // read/write access
+										0,					   // maximum object size (high-order DWORD)
+										DWORD(Size),		   // maximum object size (low-order DWORD)
+										InstanceMapName.c_str());
+		if (hMap == NULL)
+		{
+			return {};
+		}
+		void* pBuf = MapViewOfFile(hMap,				 // handle to map object
+								   FILE_MAP_ALL_ACCESS,	 // read/write permission
+								   0,					 // offset high
+								   0,					 // offset low
+								   DWORD(Size));		 // size
+
+		if (pBuf == NULL)
+		{
+			CloseHandle(hMap);
+			return {};
+		}
+		return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)};
+#endif	// ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+		ZEN_UNUSED(SystemGlobal);
+		std::string InstanceMapName = fmt::format("/{}", Name);
+
+		int Fd = shm_open(InstanceMapName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666);
+		if (Fd < 0)
+		{
+			return {};
+		}
+		fchmod(Fd, 0666);
+		void* hMap = (void*)intptr_t(Fd);
+
+		int Result = ftruncate(Fd, Size);
+		ZEN_UNUSED(Result);
+
+		void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0);
+		if (pBuf == MAP_FAILED)
+		{
+			close(Fd);
+			return {};
+		}
+		return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)};
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 	}
+
+	static void Close(Data&& MemMap, bool Delete)
+	{
+#if ZEN_PLATFORM_WINDOWS
+		ZEN_UNUSED(Delete);
+		if (MemMap.DataPtr != nullptr)
+		{
+			UnmapViewOfFile(MemMap.DataPtr);
+			MemMap.DataPtr = nullptr;
+		}
+		if (MemMap.Handle != nullptr)
+		{
+			CloseHandle(MemMap.Handle);
+			MemMap.Handle = nullptr;
+		}
 #endif	// ZEN_PLATFORM_WINDOWS
 #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	uint32_t CurrentMode = GetFileMode(Filename);
-	uint32_t NewMode	 = MakeFileModeReadOnly(CurrentMode, ReadOnly);
-	if (CurrentMode != NewMode)
+		if (MemMap.DataPtr != nullptr)
+		{
+			munmap(MemMap.DataPtr, MemMap.Size);
+			MemMap.DataPtr = nullptr;
+		}
+
+		if (MemMap.Handle != nullptr)
+		{
+			int Fd = int(intptr_t(MemMap.Handle));
+			close(Fd);
+			MemMap.Handle = nullptr;
+		}
+		if (Delete)
+		{
+			std::string InstanceMapName = fmt::format("/{}", MemMap.Name);
+			shm_unlink(InstanceMapName.c_str());
+		}
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	}
+
+	SharedMemoryImpl(Data&& MemMap, bool IsOwned) : m_MemMap(std::move(MemMap)), m_IsOwned(IsOwned) {}
+	virtual ~SharedMemoryImpl()
 	{
-		SetFileMode(Filename, NewMode);
-		return true;
+		try
+		{
+			Close(std::move(m_MemMap), /*Delete*/ m_IsOwned);
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_ERROR("SharedMemoryImpl::~SharedMemoryImpl threw exception: {}", Ex.what());
+		}
 	}
-#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	return false;
-}
 
-std::filesystem::path
-StringToPath(const std::string_view& Path)
+	virtual void* GetData() override { return m_MemMap.DataPtr; }
+
+private:
+	Data	   m_MemMap;
+	const bool m_IsOwned = false;
+};
+
+std::unique_ptr<SharedMemory>
+OpenSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal)
 {
-	if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"')
+	SharedMemoryImpl::Data MemMap = SharedMemoryImpl::Open(Name, Size, SystemGlobal);
+	if (MemMap.DataPtr)
 	{
-		return std::filesystem::path(Path.substr(1, Path.length() - 2)).make_preferred();
+		return std::make_unique<SharedMemoryImpl>(std::move(MemMap), /*IsOwned*/ false);
 	}
-	else
+	return {};
+}
+
+std::unique_ptr<SharedMemory>
+CreateSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal)
+{
+	SharedMemoryImpl::Data MemMap = SharedMemoryImpl::Create(Name, Size, SystemGlobal);
+	if (MemMap.DataPtr)
 	{
-		return std::filesystem::path(Path).make_preferred();
+		return std::make_unique<SharedMemoryImpl>(std::move(MemMap), /*IsOwned*/ true);
 	}
+	return {};
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -2076,7 +3058,7 @@ TEST_CASE("filesystem")
 	path	   BinPath	   = GetRunningExecutablePath();
 	const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver";
 	CHECK(ExpectedExe);
-	CHECK(is_regular_file(BinPath));
+	CHECK(IsFile(BinPath));
 
 	// PathFromHandle
 	void* Handle;
@@ -2129,6 +3111,80 @@ TEST_CASE("filesystem")
 	CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize());
 }
 
+TEST_CASE("Filesystem.Basics")
+{
+	std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test";
+	CleanDirectory(TestBaseDir, true);
+	DeleteDirectories(TestBaseDir);
+	CHECK(!IsDir(TestBaseDir));
+	CHECK(CleanDirectory(TestBaseDir, false));
+	CHECK(IsDir(TestBaseDir));
+	CHECK(!CleanDirectory(TestBaseDir, false));
+	CHECK(!IsDir(TestBaseDir / "no_such_thing"));
+	CHECK(!IsDir("hgjda/cev_/q12"));
+	CHECK(!IsFile(TestBaseDir));
+	CHECK(!IsFile(TestBaseDir / "no_such_thing"));
+	CHECK(!IsFile("hgjda/cev_/q12"));
+	CHECK_THROWS(FileSizeFromPath(TestBaseDir) == 0);
+	CHECK_THROWS(FileSizeFromPath(TestBaseDir / "no_such_file"));
+	CHECK(!CreateDirectories(TestBaseDir));
+	CHECK(CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep"));
+	CHECK(!CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep"));
+	CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep"));
+	CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit"));
+	CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep" / "no"));
+	CHECK_THROWS(WriteFile(TestBaseDir / "nested" / "a", IoBuffer(20)));
+	CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "a" / "yo", IoBuffer(20)));
+	CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo"));
+	CHECK(FileSizeFromPath(TestBaseDir / "nested" / "a" / "yo") == 20);
+	CHECK(!IsFile(TestBaseDir / "nested" / "a"));
+	CHECK(DeleteDirectories(TestBaseDir / "nested" / "a" / "bit"));
+	CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo"));
+	CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit"));
+	CHECK(!DeleteDirectories(TestBaseDir / "nested" / "a" / "bit"));
+	CHECK(IsDir(TestBaseDir / "nested" / "a"));
+	CHECK(DeleteDirectories(TestBaseDir / "nested"));
+	CHECK(!IsFile(TestBaseDir / "nested" / "a" / "yo"));
+	CHECK(CreateDirectories(TestBaseDir / "nested" / "deeper"));
+	CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "deeper" / "yo", IoBuffer(20)));
+	CHECK_NOTHROW(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place"));
+	CHECK(IsFile(TestBaseDir / "new_place" / "yo"));
+	CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20);
+	CHECK(IsDir(TestBaseDir / "new_place"));
+	CHECK(!IsFile(TestBaseDir / "new_place"));
+	CHECK_THROWS(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place"));
+	CHECK(!RemoveDir(TestBaseDir / "nested" / "deeper"));
+	CHECK(RemoveFile(TestBaseDir / "new_place" / "yo"));
+	CHECK(!IsFile(TestBaseDir / "new_place" / "yo"));
+	CHECK_THROWS(FileSizeFromPath(TestBaseDir / "new_place" / "yo"));
+	CHECK(!RemoveFile(TestBaseDir / "new_place" / "yo"));
+	CHECK_THROWS(RemoveFile(TestBaseDir / "nested"));
+	CHECK_THROWS(RemoveDir(TestBaseDir));
+	CHECK_NOTHROW(WriteFile(TestBaseDir / "yo", IoBuffer(20)));
+	CHECK_NOTHROW(RenameFile(TestBaseDir / "yo", TestBaseDir / "new_place" / "yo"));
+	CHECK(!IsFile(TestBaseDir / "yo"));
+	CHECK(IsFile(TestBaseDir / "new_place" / "yo"));
+	CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20);
+	CHECK_THROWS(RemoveDir(TestBaseDir / "new_place" / "yo"));
+	CHECK(DeleteDirectories(TestBaseDir));
+	CHECK(!IsFile(TestBaseDir / "new_place" / "yo"));
+	CHECK(!IsDir(TestBaseDir));
+	CHECK(!IsDir(TestBaseDir / "nested"));
+	CHECK(CreateDirectories(TestBaseDir / "nested"));
+	CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20)));
+	CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true));
+	CHECK_THROWS(RemoveFile(TestBaseDir / "nested" / "readonly"));
+	CHECK_THROWS(CleanDirectory(TestBaseDir, false));
+	CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", false));
+	CHECK(RemoveFile(TestBaseDir / "nested" / "readonly"));
+	CHECK(!CleanDirectory(TestBaseDir, false));
+	CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20)));
+	CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true));
+	CHECK(!CleanDirectory(TestBaseDir / "nested", true));
+	CHECK(!CleanDirectory(TestBaseDir, false));
+	CHECK(RemoveDir(TestBaseDir));
+}
+
 TEST_CASE("WriteFile")
 {
 	std::filesystem::path TempFile = GetRunningExecutablePath().parent_path();
@@ -2163,7 +3219,7 @@ TEST_CASE("WriteFile")
 		CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0);
 	}
 
-	std::filesystem::remove(TempFile);
+	RemoveFile(TempFile);
 }
 
 TEST_CASE("DiskSpaceInfo")
@@ -2220,7 +3276,7 @@ TEST_CASE("PathBuilder")
 TEST_CASE("RotateDirectories")
 {
 	std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test";
-	CleanDirectory(TestBaseDir);
+	CleanDirectory(TestBaseDir, false);
 	std::filesystem::path RotateDir		= TestBaseDir / "rotate_dir" / "dir_to_rotate";
 	IoBuffer			  DummyFileData = IoBufferBuilder::MakeCloneFromMemory("blubb", 5);
 
@@ -2234,16 +3290,16 @@ TEST_CASE("RotateDirectories")
 	const int RotateMax = 10;
 
 	NewDir();
-	CHECK(std::filesystem::exists(RotateDir));
+	CHECK(IsDir(RotateDir));
 	RotateDirectories(RotateDir, RotateMax);
-	CHECK(!std::filesystem::exists(RotateDir));
-	CHECK(std::filesystem::exists(DirWithSuffix(1)));
+	CHECK(!IsDir(RotateDir));
+	CHECK(IsDir(DirWithSuffix(1)));
 	NewDir();
-	CHECK(std::filesystem::exists(RotateDir));
+	CHECK(IsDir(RotateDir));
 	RotateDirectories(RotateDir, RotateMax);
-	CHECK(!std::filesystem::exists(RotateDir));
-	CHECK(std::filesystem::exists(DirWithSuffix(1)));
-	CHECK(std::filesystem::exists(DirWithSuffix(2)));
+	CHECK(!IsDir(RotateDir));
+	CHECK(IsDir(DirWithSuffix(1)));
+	CHECK(IsDir(DirWithSuffix(2)));
 
 	for (int i = 0; i < RotateMax; ++i)
 	{
@@ -2253,19 +3309,35 @@ TEST_CASE("RotateDirectories")
 		CHECK_EQ(IsError, false);
 	}
 
-	CHECK(!std::filesystem::exists(RotateDir));
+	CHECK(!IsDir(RotateDir));
 
 	for (int i = 0; i < RotateMax; ++i)
 	{
-		CHECK(std::filesystem::exists(DirWithSuffix(i + 1)));
+		CHECK(IsDir(DirWithSuffix(i + 1)));
 	}
 
 	for (int i = RotateMax; i < RotateMax + 5; ++i)
 	{
-		CHECK(!std::filesystem::exists(DirWithSuffix(RotateMax + i + 1)));
+		CHECK(!IsDir(DirWithSuffix(RotateMax + i + 1)));
 	}
 }
 
+TEST_CASE("SharedMemory")
+{
+	CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false));
+	CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, true));
+
+	{
+		auto Mem0 = CreateSharedMemory("SharedMemoryTest0", 482, false);
+		CHECK(Mem0);
+		strcpy((char*)Mem0->GetData(), "this is the string we are looking for");
+		auto Mem1 = OpenSharedMemory("SharedMemoryTest0", 482, false);
+		CHECK_EQ(std::string((char*)Mem0->GetData()), std::string((char*)Mem1->GetData()));
+	}
+
+	CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false));
+}
+
 #endif
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h
index 57798b6f4..465499d2b 100644
--- a/src/zencore/include/zencore/basicfile.h
+++ b/src/zencore/include/zencore/basicfile.h
@@ -174,9 +174,11 @@ public:
 	BasicFileWriter(BasicFile& Base, uint64_t BufferSize);
 	~BasicFileWriter();
 
-	void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
-	void Write(const CompositeBuffer& Data, uint64_t FileOffset);
-	void Flush();
+	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+	void	 Write(const CompositeBuffer& Data, uint64_t FileOffset);
+	void	 AddPadding(uint64_t Padding);
+	uint64_t AlignTo(uint64_t Alignment);
+	void	 Flush();
 
 private:
 	BasicFile&	   m_Base;
diff --git a/src/zencore/include/zencore/blake3.h b/src/zencore/include/zencore/blake3.h
index 28bb348c0..f01e45266 100644
--- a/src/zencore/include/zencore/blake3.h
+++ b/src/zencore/include/zencore/blake3.h
@@ -53,6 +53,7 @@ struct BLAKE3Stream
 	void		  Reset();									   // Begin streaming hash compute (not needed on freshly constructed instance)
 	BLAKE3Stream& Append(const void* data, size_t byteCount);  // Append another chunk
 	BLAKE3Stream& Append(MemoryView DataView) { return Append(DataView.GetData(), DataView.GetSize()); }  // Append another chunk
+	BLAKE3Stream& Append(const IoBuffer& Buffer);														  // Append another chunk
 	BLAKE3		  GetHash();  // Obtain final hash. If you wish to reuse the instance call reset()
 
 private:
diff --git a/src/zencore/include/zencore/compactbinaryfmt.h b/src/zencore/include/zencore/compactbinaryfmt.h
index ae0c3eb42..b03683db4 100644
--- a/src/zencore/include/zencore/compactbinaryfmt.h
+++ b/src/zencore/include/zencore/compactbinaryfmt.h
@@ -14,7 +14,8 @@ template<typename T>
 requires DerivedFrom<T, zen::CbObjectView>
 struct fmt::formatter<T> : fmt::formatter<std::string_view>
 {
-	auto format(const zen::CbObject& a, format_context& ctx) const
+	template<typename FormatContext>
+	auto format(const zen::CbObject& a, FormatContext& ctx) const
 	{
 		zen::ExtendableStringBuilder<1024> ObjStr;
 		zen::CompactBinaryToJson(a, ObjStr);
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
index 74fd5f767..09fa6249d 100644
--- a/src/zencore/include/zencore/compress.h
+++ b/src/zencore/include/zencore/compress.h
@@ -74,11 +74,12 @@ public:
 															   OodleCompressor		 Compressor		  = OodleCompressor::Mermaid,
 															   OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
 															   uint64_t				 BlockSize		  = 0);
-	[[nodiscard]] ZENCORE_API static bool			  CompressToStream(const CompositeBuffer&												RawData,
-																	   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
-																	   OodleCompressor		 Compressor		  = OodleCompressor::Mermaid,
-																	   OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
-																	   uint64_t				 BlockSize		  = 0);
+	[[nodiscard]] ZENCORE_API static bool			  CompressToStream(
+					const CompositeBuffer&																							 RawData,
+					std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+					OodleCompressor		  Compressor	   = OodleCompressor::Mermaid,
+					OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
+					uint64_t			  BlockSize		   = 0);
 
 	/**
 	 * Construct from a compressed buffer previously created by Compress().
@@ -207,9 +208,10 @@ public:
 	 *
 	 * @return True if the buffer is valid and can be decompressed.
 	 */
-	[[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t															   RawOffset,
-													  uint64_t															   RawSize,
-													  std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const;
+	[[nodiscard]] ZENCORE_API bool DecompressToStream(
+		uint64_t																										 RawOffset,
+		uint64_t																										 RawSize,
+		std::function<bool(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const;
 
 	/** A null compressed buffer. */
 	static const CompressedBuffer Null;
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 9a2b15d1d..36d4d1b68 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -20,21 +20,35 @@ class WorkerThreadPool;
 
 /** Delete directory (after deleting any contents)
  */
-ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir);
+ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path);
+
+/** Delete directory (after deleting any contents)
+ */
+ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec);
+
+/** Ensure directory exists.
+
+	Will also create any required parent direCleanDirectoryctories
+ */
+ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path);
 
 /** Ensure directory exists.
 
 	Will also create any required parent directories
  */
-ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
+ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec);
+
+/** Ensure directory exists and delete contents (if any) before returning
+ */
+ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles);
 
 /** Ensure directory exists and delete contents (if any) before returning
  */
-ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
+ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec);
 
 /** Ensure directory exists and delete contents (if any) before returning
  */
-ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir);
+ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path);
 
 /** Map native file handle to a path
  */
@@ -44,10 +58,54 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_
  */
 ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec);
 
+/** Query file size
+ */
+ZENCORE_API bool IsFile(const std::filesystem::path& Path);
+
+/** Query file size
+ */
+ZENCORE_API bool IsFile(const std::filesystem::path& Path, std::error_code& Ec);
+
+/** Query file size
+ */
+ZENCORE_API bool IsDir(const std::filesystem::path& Path);
+
+/** Query file size
+ */
+ZENCORE_API bool IsDir(const std::filesystem::path& Path, std::error_code& Ec);
+
+/** Query file size
+ */
+ZENCORE_API bool RemoveFile(const std::filesystem::path& Path);
+
+/** Query file size
+ */
+ZENCORE_API bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec);
+
+/** Query file size
+ */
+ZENCORE_API bool RemoveDir(const std::filesystem::path& Path);
+
+/** Query file size
+ */
+ZENCORE_API bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec);
+
+/** Query file size
+ */
+ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path);
+
+/** Query file size
+ */
+ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec);
+
 /** Query file size from native file handle
  */
 ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle);
 
+/** Query file size from native file handle
+ */
+ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec);
+
 /** Get a native time tick of last modification time
  */
 ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec);
@@ -56,12 +114,35 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro
  */
 ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename);
 
+ZENCORE_API bool TryGetFileProperties(const std::filesystem::path& Path,
+									  uint64_t&					   OutSize,
+									  uint64_t&					   OutModificationTick,
+									  uint32_t&					   OutNativeModeOrAttributes);
+
+/** Move a file, if the files are not on the same drive the function will fail
+ */
+ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath);
+
+/** Move a file, if the files are not on the same drive the function will fail
+ */
+ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec);
+
+/** Move a directory, if the files are not on the same drive the function will fail
+ */
+ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath);
+
+/** Move a directory, if the files are not on the same drive the function will fail
+ */
+ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec);
+
 ZENCORE_API std::filesystem::path GetRunningExecutablePath();
 
 /** Set the max open file handle count to max allowed for the current process on Linux and MacOS
  */
 ZENCORE_API void MaximizeOpenFileCount();
 
+ZENCORE_API bool PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize);
+
 struct FileContents
 {
 	std::vector<IoBuffer> Data;
@@ -90,6 +171,13 @@ ZENCORE_API void ScanFile(void*												   NativeHandle,
 						  uint64_t											   Size,
 						  uint64_t											   ChunkSize,
 						  std::function<void(const void* Data, size_t Size)>&& ProcessFunc);
+ZENCORE_API void WriteFile(void*			NativeHandle,
+						   const void*		Data,
+						   uint64_t			Size,
+						   uint64_t			FileOffset,
+						   uint64_t			ChunkSize,
+						   std::error_code& Ec);
+ZENCORE_API void ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec);
 
 struct CopyFileOptions
 {
@@ -277,12 +365,16 @@ std::filesystem::path PickDefaultSystemRootDirectory();
 
 #if ZEN_PLATFORM_WINDOWS
 uint32_t GetFileAttributes(const std::filesystem::path& Filename);
+uint32_t GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec);
 void	 SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes);
+void	 SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec);
 #endif	// ZEN_PLATFORM_WINDOWS
 
 #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 uint32_t GetFileMode(const std::filesystem::path& Filename);
-void	 SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes);
+uint32_t GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec);
+void	 SetFileMode(const std::filesystem::path& Filename, uint32_t Mode);
+void	 SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec);
 #endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 
 bool	 IsFileAttributeReadOnly(uint32_t FileAttributes);
@@ -290,9 +382,18 @@ bool	 IsFileModeReadOnly(uint32_t FileMode);
 uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly);
 uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly);
 
+bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec);
 bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly);
 
-std::filesystem::path StringToPath(const std::string_view& Path);
+class SharedMemory
+{
+public:
+	virtual ~SharedMemory() {}
+	virtual void* GetData() = 0;
+};
+
+std::unique_ptr<SharedMemory> OpenSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal);
+std::unique_ptr<SharedMemory> CreateSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal);
 
 //////////////////////////////////////////////////////////////////////////
 
diff --git a/src/zencore/include/zencore/fmtutils.h b/src/zencore/include/zencore/fmtutils.h
index 8482157fb..404e570fd 100644
--- a/src/zencore/include/zencore/fmtutils.h
+++ b/src/zencore/include/zencore/fmtutils.h
@@ -12,6 +12,7 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#include <chrono>
 #include <string_view>
 
 // Custom formatting for some zencore types
@@ -20,7 +21,8 @@ template<typename T>
 requires DerivedFrom<T, zen::StringBuilderBase>
 struct fmt::formatter<T> : fmt::formatter<std::string_view>
 {
-	auto format(const zen::StringBuilderBase& a, format_context& ctx) const
+	template<typename FormatContext>
+	auto format(const zen::StringBuilderBase& a, FormatContext& ctx) const
 	{
 		return fmt::formatter<std::string_view>::format(a.ToView(), ctx);
 	}
@@ -30,7 +32,8 @@ template<typename T>
 requires DerivedFrom<T, zen::NiceBase>
 struct fmt::formatter<T> : fmt::formatter<std::string_view>
 {
-	auto format(const zen::NiceBase& a, format_context& ctx) const
+	template<typename FormatContext>
+	auto format(const zen::NiceBase& a, FormatContext& ctx) const
 	{
 		return fmt::formatter<std::string_view>::format(std::string_view(a), ctx);
 	}
@@ -97,8 +100,22 @@ template<typename T>
 requires DerivedFrom<T, zen::PathBuilderBase>
 struct fmt::formatter<T> : fmt::formatter<std::string_view>
 {
-	auto format(const zen::PathBuilderBase& a, format_context& ctx) const
+	template<typename FormatContext>
+	auto format(const zen::PathBuilderBase& a, FormatContext& ctx) const
 	{
 		return fmt::formatter<std::string_view>::format(a.ToView(), ctx);
 	}
 };
+
+template<>
+struct fmt::formatter<std::chrono::system_clock::time_point> : formatter<string_view>
+{
+	template<typename FormatContext>
+	auto format(const std::chrono::system_clock::time_point& TimePoint, FormatContext& ctx) const
+	{
+		std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
+		char		TimeString[std::size("yyyy-mm-ddThh:mm:ss")];
+		std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time));
+		return fmt::format_to(ctx.out(), "{}", TimeString);
+	}
+};
diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h
index 7443e17b7..a619b0053 100644
--- a/src/zencore/include/zencore/iohash.h
+++ b/src/zencore/include/zencore/iohash.h
@@ -102,6 +102,12 @@ struct IoHashStream
 		return *this;
 	}
 
+	IoHashStream& Append(const IoBuffer& Buffer)
+	{
+		m_Blake3Stream.Append(Buffer);
+		return *this;
+	}
+
 	/// Append another chunk
 	IoHashStream& Append(MemoryView Data)
 	{
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index 335e3d909..98d352db6 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -99,10 +99,7 @@ ZENCORE_API int	 GetCurrentProcessId();
 int				 GetProcessId(CreateProcResult ProcId);
 
 std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc);
-std::error_code		  FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle);
-
-std::vector<std::string> ParseCommandLine(std::string_view CommandLine);
-std::vector<char*>		 StripCommandlineQuotes(std::vector<std::string>& InOutArgs);
+std::error_code		  FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf = true);
 
 #if ZEN_PLATFORM_LINUX
 void IgnoreChildSignals();
diff --git a/src/zenserver/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h
index 40e22af4e..faf1238b7 100644
--- a/src/zenserver/sentryintegration.h
+++ b/src/zencore/include/zencore/sentryintegration.h
@@ -31,8 +31,18 @@ public:
 	SentryIntegration();
 	~SentryIntegration();
 
-	void Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, bool AllowPII, const std::string& CommandLine);
-	void LogStartupInformation();
+	struct Config
+	{
+		std::string DatabasePath;
+		std::string AttachmentsPath;
+		std::string Dsn;
+		std::string Environment;
+		bool		AllowPII = false;
+		bool		Debug	 = false;
+	};
+
+	void		Initialize(const Config& Conf, const std::string& CommandLine);
+	void		LogStartupInformation();
 	static void ClearCaches();
 
 private:
diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h
index 8fb781571..d9fb5c023 100644
--- a/src/zencore/include/zencore/thread.h
+++ b/src/zencore/include/zencore/thread.h
@@ -183,6 +183,7 @@ public:
 	void CountDown()
 	{
 		std::ptrdiff_t Old = Counter.fetch_sub(1);
+		ZEN_ASSERT(Old > 0);
 		if (Old == 1)
 		{
 			Complete.Set();
@@ -197,8 +198,7 @@ public:
 	void AddCount(std::ptrdiff_t Count)
 	{
 		std::atomic_ptrdiff_t Old = Counter.fetch_add(Count);
-		ZEN_UNUSED(Old);
-		ZEN_ASSERT_SLOW(Old > 0);
+		ZEN_ASSERT(Old > 0);
 	}
 
 	bool Wait(int TimeoutMs = -1)
diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index 3b5c89c3e..8e9a37a27 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -297,49 +297,21 @@ IoBufferExtendedCore::Materialize() const
 
 		AllocateBuffer(m_DataBytes, sizeof(void*));
 		NewFlags |= kIsOwnedByThis;
-		int32_t Error	  = 0;
-		size_t	BytesRead = 0;
 
-#if ZEN_PLATFORM_WINDOWS
-		OVERLAPPED Ovl{};
-
-		Ovl.Offset	   = DWORD(m_FileOffset & 0xffff'ffffu);
-		Ovl.OffsetHigh = DWORD(m_FileOffset >> 32);
-
-		DWORD dwNumberOfBytesRead = 0;
-		BOOL  Success			  = ::ReadFile(m_FileHandle, (void*)m_DataPtr, DWORD(m_DataBytes), &dwNumberOfBytesRead, &Ovl) == TRUE;
-		if (Success)
-		{
-			BytesRead = size_t(dwNumberOfBytesRead);
-		}
-		else
-		{
-			Error = zen::GetLastError();
-		}
-#else
-		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
-		int		Fd		   = int(uintptr_t(m_FileHandle));
-		ssize_t ReadResult = pread(Fd, (void*)m_DataPtr, m_DataBytes, m_FileOffset);
-		if (ReadResult != -1)
-		{
-			BytesRead = size_t(ReadResult);
-		}
-		else
-		{
-			Error = zen::GetLastError();
-		}
-#endif	// ZEN_PLATFORM_WINDOWS
-		if (Error || (BytesRead != m_DataBytes))
+		std::error_code Ec;
+		ReadFile(m_FileHandle, (void*)m_DataPtr, m_DataBytes, m_FileOffset, DisableMMapSizeLimit, Ec);
+		if (Ec)
 		{
 			std::error_code DummyEc;
-			ZEN_WARN("IoBufferExtendedCore::Materialize: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}",
+			ZEN_WARN("IoBufferExtendedCore::Materialize: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {} ({})",
 					 m_FileOffset,
 					 m_DataBytes,
 					 zen::PathFromHandle(m_FileHandle, DummyEc),
 					 zen::FileSizeFromHandle(m_FileHandle),
-					 GetSystemErrorAsString(Error));
+					 Ec.message(),
+					 Ec.value());
 			throw std::system_error(
-				std::error_code(Error, std::system_category()),
+				Ec,
 				fmt::format("IoBufferExtendedCore::Materialize: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
 							m_FileOffset,
 							m_DataBytes,
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index 147b00966..f62731d94 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -60,7 +60,7 @@ GetPidStatus(int Pid, std::error_code& OutEc)
 {
 	std::filesystem::path EntryPath = std::filesystem::path("/proc") / fmt::format("{}", Pid);
 	std::filesystem::path StatPath	= EntryPath / "stat";
-	if (std::filesystem::is_regular_file(StatPath))
+	if (IsFile(StatPath))
 	{
 		FILE* StatFile = fopen(StatPath.c_str(), "r");
 		if (StatFile)
@@ -946,7 +946,7 @@ GetProcessExecutablePath(int Pid, std::error_code& OutEc)
 }
 
 std::error_code
-FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle)
+FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf)
 {
 #if ZEN_PLATFORM_WINDOWS
 	HANDLE ProcessSnapshotHandle = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
@@ -956,13 +956,15 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 	}
 	auto _ = MakeGuard([&]() { CloseHandle(ProcessSnapshotHandle); });
 
+	const DWORD ThisProcessId = ::GetCurrentProcessId();
+
 	PROCESSENTRY32 Entry;
 	Entry.dwSize = sizeof(PROCESSENTRY32);
 	if (Process32First(ProcessSnapshotHandle, (LPPROCESSENTRY32)&Entry))
 	{
 		do
 		{
-			if (ExecutableImage.filename() == Entry.szExeFile)
+			if ((IncludeSelf || (Entry.th32ProcessID != ThisProcessId)) && (ExecutableImage.filename() == Entry.szExeFile))
 			{
 				std::error_code		  Ec;
 				std::filesystem::path EntryPath = GetProcessExecutablePath(Entry.th32ProcessID, Ec);
@@ -987,6 +989,7 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 				}
 			}
 		} while (::Process32Next(ProcessSnapshotHandle, (LPPROCESSENTRY32)&Entry));
+		return {};
 	}
 	return MakeErrorCodeFromLastError();
 #endif	// ZEN_PLATFORM_WINDOWS
@@ -997,6 +1000,8 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 	struct kinfo_proc* Processes = nullptr;
 	uint32_t		   ProcCount = 0;
 
+	const pid_t ThisProcessId = getpid();
+
 	if (sysctl(Mib, 4, NULL, &BufferSize, NULL, 0) != -1 && BufferSize > 0)
 	{
 		struct kinfo_proc* Processes = (struct kinfo_proc*)malloc(BufferSize);
@@ -1007,36 +1012,46 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 			char Buffer[PROC_PIDPATHINFO_MAXSIZE];
 			for (uint32_t ProcIndex = 0; ProcIndex < ProcCount; ProcIndex++)
 			{
-				pid_t				  Pid = Processes[ProcIndex].kp_proc.p_pid;
-				std::error_code		  Ec;
-				std::filesystem::path EntryPath = GetProcessExecutablePath(Pid, Ec);
-				if (!Ec)
+				pid_t Pid = Processes[ProcIndex].kp_proc.p_pid;
+				if (IncludeSelf || (Pid != ThisProcessId))
 				{
-					if (EntryPath == ExecutableImage)
+					std::error_code		  Ec;
+					std::filesystem::path EntryPath = GetProcessExecutablePath(Pid, Ec);
+					if (!Ec)
 					{
-						if (Processes[ProcIndex].kp_proc.p_stat != SZOMB)
+						if (EntryPath == ExecutableImage)
 						{
-							OutHandle.Initialize(Pid, Ec);
-							return Ec;
+							if (Processes[ProcIndex].kp_proc.p_stat != SZOMB)
+							{
+								OutHandle.Initialize(Pid, Ec);
+								return Ec;
+							}
 						}
 					}
+					Ec.clear();
 				}
 			}
+			return {};
 		}
 	}
 	return MakeErrorCodeFromLastError();
 #endif	// ZEN_PLATFORM_MAC
 #if ZEN_PLATFORM_LINUX
+	const pid_t ThisProcessId = getpid();
+
 	std::vector<uint32_t> RunningPids;
 	DirectoryContent	  ProcList;
 	GetDirectoryContent("/proc", DirectoryContentFlags::IncludeDirs, ProcList);
 	for (const std::filesystem::path& EntryPath : ProcList.Directories)
 	{
 		std::string				EntryName = EntryPath.stem();
-		std::optional<uint32_t> Pid		  = ParseInt<uint32_t>(EntryName);
-		if (Pid.has_value())
+		std::optional<uint32_t> PidMaybe  = ParseInt<uint32_t>(EntryName);
+		if (PidMaybe.has_value())
 		{
-			RunningPids.push_back(Pid.value());
+			if (pid_t Pid = PidMaybe.value(); IncludeSelf || (Pid != ThisProcessId))
+			{
+				RunningPids.push_back(Pid);
+			}
 		}
 	}
 
@@ -1059,123 +1074,12 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 				}
 			}
 		}
+		Ec.clear();
 	}
 	return {};
 #endif	// ZEN_PLATFORM_LINUX
 }
 
-std::vector<std::string>
-ParseCommandLine(std::string_view CommandLine)
-{
-	auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) {
-		if (Pos == CommandLine.length())
-		{
-			return true;
-		}
-		if (CommandLine[Pos] == ' ')
-		{
-			return true;
-		}
-		return false;
-	};
-
-	bool IsParsingArg = false;
-	bool IsInQuote	  = false;
-
-	std::string::size_type	 Pos	  = 0;
-	std::string::size_type	 ArgStart = 0;
-	std::vector<std::string> Args;
-	while (Pos < CommandLine.length())
-	{
-		if (IsInQuote)
-		{
-			if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1))
-			{
-				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1)));
-				Pos++;
-				IsInQuote	 = false;
-				IsParsingArg = false;
-			}
-			else
-			{
-				Pos++;
-			}
-		}
-		else if (IsParsingArg)
-		{
-			ZEN_ASSERT(Pos > ArgStart);
-			if (CommandLine[Pos] == ' ')
-			{
-				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart)));
-				Pos++;
-				IsParsingArg = false;
-			}
-			else if (CommandLine[Pos] == '"')
-			{
-				IsInQuote = true;
-				Pos++;
-			}
-			else
-			{
-				Pos++;
-			}
-		}
-		else if (CommandLine[Pos] == '"')
-		{
-			IsInQuote	 = true;
-			IsParsingArg = true;
-			ArgStart	 = Pos;
-			Pos++;
-		}
-		else if (CommandLine[Pos] != ' ')
-		{
-			IsParsingArg = true;
-			ArgStart	 = Pos;
-			Pos++;
-		}
-		else
-		{
-			Pos++;
-		}
-	}
-	if (IsParsingArg)
-	{
-		ZEN_ASSERT(Pos > ArgStart);
-		Args.push_back(std::string(CommandLine.substr(ArgStart)));
-	}
-
-	return Args;
-}
-
-std::vector<char*>
-StripCommandlineQuotes(std::vector<std::string>& InOutArgs)
-{
-	std::vector<char*> RawArgs;
-	RawArgs.reserve(InOutArgs.size());
-	for (std::string& Arg : InOutArgs)
-	{
-		std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1);
-		while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos)
-		{
-			Arg.erase(EscapedQuotePos, 1);
-			EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos);
-		}
-
-		if (Arg.starts_with("\""))
-		{
-			if (Arg.find('"', 1) == Arg.length() - 1)
-			{
-				if (Arg.find(' ', 1) == std::string::npos)
-				{
-					Arg = Arg.substr(1, Arg.length() - 2);
-				}
-			}
-		}
-		RawArgs.push_back(const_cast<char*>(Arg.c_str()));
-	}
-	return RawArgs;
-}
-
 #if ZEN_WITH_TESTS
 
 void
@@ -1194,10 +1098,24 @@ TEST_CASE("Process")
 
 TEST_CASE("FindProcess")
 {
-	ProcessHandle	Process;
-	std::error_code Ec = FindProcess(GetRunningExecutablePath(), Process);
-	CHECK(!Ec);
-	CHECK(Process.IsValid());
+	{
+		ProcessHandle	Process;
+		std::error_code Ec = FindProcess(GetRunningExecutablePath(), Process, /*IncludeSelf*/ true);
+		CHECK(!Ec);
+		CHECK(Process.IsValid());
+	}
+	{
+		ProcessHandle	Process;
+		std::error_code Ec = FindProcess(GetRunningExecutablePath(), Process, /*IncludeSelf*/ false);
+		CHECK(!Ec);
+		CHECK(!Process.IsValid());
+	}
+	{
+		ProcessHandle	Process;
+		std::error_code Ec = FindProcess("this/does\\not/exist\\123914921929412312312312asdad\\12134.no", Process, /*IncludeSelf*/ false);
+		CHECK(!Ec);
+		CHECK(!Process.IsValid());
+	}
 }
 
 TEST_CASE("BuildArgV")
@@ -1252,36 +1170,6 @@ TEST_CASE("BuildArgV")
 	}
 }
 
-TEST_CASE("CommandLine")
-{
-	std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\"");
-	CHECK_EQ(v1[0], "c:\\my\\exe.exe");
-	CHECK_EQ(v1[1], "\"quoted arg\"");
-	CHECK_EQ(v1[2], "\"one\",two,\"three\\\"");
-
-	std::vector<std::string> v2 = ParseCommandLine(
-		"--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog "
-		"--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" "
-		"--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" "
-		"--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5");
-
-	std::vector<char*> v2Stripped = StripCommandlineQuotes(v2);
-	CHECK_EQ(v2Stripped[0], std::string("--tracehost"));
-	CHECK_EQ(v2Stripped[1], std::string("127.0.0.1"));
-	CHECK_EQ(v2Stripped[2], std::string("builds"));
-	CHECK_EQ(v2Stripped[3], std::string("download"));
-	CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com"));
-	CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog"));
-	CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows"));
-	CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path"));
-	CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\""));
-	CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\""));
-	CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\""));
-	CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab"));
-	CHECK_EQ(v2Stripped[12], std::string("--build-part-name"));
-	CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5"));
-}
-
 TEST_SUITE_END(/* core.process */);
 
 #endif
diff --git a/src/zenserver/sentryintegration.cpp b/src/zencore/sentryintegration.cpp
index 7996f25bb..118c4158a 100644
--- a/src/zenserver/sentryintegration.cpp
+++ b/src/zencore/sentryintegration.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include "sentryintegration.h"
+#include <zencore/sentryintegration.h>
 
 #include <zencore/config.h>
 #include <zencore/logging.h>
@@ -31,6 +31,10 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace sentry {
 
+namespace {
+	static const std::string DefaultDsn("https://[email protected]/5919284");
+}
+
 struct SentryAssertImpl : zen::AssertImpl
 {
 	virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION
@@ -192,29 +196,37 @@ SentryIntegration::~SentryIntegration()
 }
 
 void
-SentryIntegration::Initialize(std::string		 SentryDatabasePath,
-							  std::string		 SentryAttachmentsPath,
-							  bool				 AllowPII,
-							  const std::string& CommandLine)
+SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine)
 {
-	m_AllowPII = AllowPII;
+	m_AllowPII = Conf.AllowPII;
 
+	std::string SentryDatabasePath = Conf.DatabasePath;
 	if (SentryDatabasePath.starts_with("\\\\?\\"))
 	{
 		SentryDatabasePath = SentryDatabasePath.substr(4);
 	}
 	sentry_options_t* SentryOptions = sentry_options_new();
-	sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284");
+
+	sentry_options_set_dsn(SentryOptions, Conf.Dsn.empty() ? sentry::DefaultDsn.c_str() : Conf.Dsn.c_str());
 	sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str());
 	sentry_options_set_logger(SentryOptions, SentryLogFunction, this);
-	if (SentryAttachmentsPath.starts_with("\\\\?\\"))
+	sentry_options_set_environment(SentryOptions, Conf.Environment.empty() ? "production" : Conf.Environment.c_str());
+
+	std::string SentryAttachmentsPath = Conf.AttachmentsPath;
+	if (!SentryAttachmentsPath.empty())
 	{
-		SentryAttachmentsPath = SentryAttachmentsPath.substr(4);
+		if (SentryAttachmentsPath.starts_with("\\\\?\\"))
+		{
+			SentryAttachmentsPath = SentryAttachmentsPath.substr(4);
+		}
+		sentry_options_add_attachment(SentryOptions, SentryAttachmentsPath.c_str());
 	}
-	sentry_options_add_attachment(SentryOptions, SentryAttachmentsPath.c_str());
 	sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION);
 
-	// sentry_options_set_debug(SentryOptions, 1);
+	if (Conf.Debug)
+	{
+		sentry_options_set_debug(SentryOptions, 1);
+	}
 
 	m_SentryErrorCode = sentry_init(SentryOptions);
 
diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp
index 641d5508a..9f50de032 100644
--- a/src/zencore/testutils.cpp
+++ b/src/zencore/testutils.cpp
@@ -4,6 +4,7 @@
 
 #if ZEN_WITH_TESTS
 
+#	include <zencore/filesystem.h>
 #	include <zencore/session.h>
 #	include "zencore/string.h"
 
@@ -19,8 +20,8 @@ CreateTemporaryDirectory()
 	std::error_code Ec;
 
 	std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str();
-	std::filesystem::remove_all(DirPath, Ec);
-	std::filesystem::create_directories(DirPath);
+	DeleteDirectories(DirPath, Ec);
+	CreateDirectories(DirPath);
 
 	return DirPath;
 }
@@ -32,14 +33,14 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporar
 ScopedTemporaryDirectory::ScopedTemporaryDirectory(std::filesystem::path Directory) : m_RootPath(Directory)
 {
 	std::error_code Ec;
-	std::filesystem::remove_all(Directory, Ec);
-	std::filesystem::create_directories(Directory);
+	DeleteDirectories(Directory, Ec);
+	CreateDirectories(Directory);
 }
 
 ScopedTemporaryDirectory::~ScopedTemporaryDirectory()
 {
 	std::error_code Ec;
-	std::filesystem::remove_all(m_RootPath, Ec);
+	DeleteDirectories(m_RootPath, Ec);
 }
 
 IoBuffer
diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua
index 13611a2e9..b3a33e052 100644
--- a/src/zencore/xmake.lua
+++ b/src/zencore/xmake.lua
@@ -64,6 +64,27 @@ target('zencore')
         {public=true}
     )
 
+    if has_config("zensentry") then
+        add_packages("vcpkg::sentry-native")
+
+        if is_plat("windows") then
+            add_links("dbghelp", "winhttp", "version")    -- for Sentry
+        end
+
+        if is_plat("linux") then
+            -- As sentry_native uses symbols from breakpad_client, the latter must
+            -- be specified after the former with GCC-like toolchains. xmake however
+            -- is unaware of this and simply globs files from vcpkg's output. The
+            -- line below forces breakpad_client to be to the right of sentry_native
+            add_syslinks("breakpad_client")
+        end
+
+        if is_plat("macosx") then
+            add_syslinks("bsm")
+        end
+
+    end
+
     if is_plat("linux") then
         add_syslinks("rt")
     end
diff --git a/src/zenhttp-test/zenhttp-test.cpp b/src/zenhttp-test/zenhttp-test.cpp
index 381e0a85e..116971b02 100644
--- a/src/zenhttp-test/zenhttp-test.cpp
+++ b/src/zenhttp-test/zenhttp-test.cpp
@@ -3,6 +3,7 @@
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
 #include <zencore/memory/newdelete.h>
+#include <zencore/trace.h>
 #include <zenhttp/zenhttp.h>
 
 #if ZEN_WITH_TESTS
@@ -21,6 +22,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 	zen::IgnoreChildSignals();
 #	endif
 
+	zen::TraceInit("zenhttp-test");
 	zen::logging::InitializeLogging();
 	zen::MaximizeOpenFileCount();
 
diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp
index 1a9892d5c..8f7befc80 100644
--- a/src/zenhttp/auth/authmgr.cpp
+++ b/src/zenhttp/auth/authmgr.cpp
@@ -379,7 +379,7 @@ private:
 				AuthState.EndArray();
 			}
 
-			std::filesystem::create_directories(m_Config.RootDirectory);
+			CreateDirectories(m_Config.RootDirectory);
 
 			std::optional<std::string> Reason;
 
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 30711a432..a2d323b5e 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -12,13 +12,19 @@
 #include <zencore/filesystem.h>
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
+#include <zencore/memory/memory.h>
 #include <zencore/session.h>
 #include <zencore/sharedbuffer.h>
 #include <zencore/stream.h>
 #include <zencore/string.h>
-#include <zencore/testing.h>
 #include <zencore/trace.h>
 
+#if ZEN_WITH_TESTS
+#	include <zencore/basicfile.h>
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#endif	// ZEN_WITH_TESTS
+
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <cpr/cpr.h>
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -42,9 +48,13 @@ namespace detail {
 	class TempPayloadFile
 	{
 	public:
+		TempPayloadFile(const TempPayloadFile&) = delete;
+		TempPayloadFile& operator=(const TempPayloadFile&) = delete;
+
 		TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {}
 		~TempPayloadFile()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Close");
 			try
 			{
 				if (m_FileHandle)
@@ -85,8 +95,9 @@ namespace detail {
 			}
 		}
 
-		std::error_code Open(const std::filesystem::path& TempFolderPath)
+		std::error_code Open(const std::filesystem::path& TempFolderPath, uint64_t FinalSize)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Open");
 			ZEN_ASSERT(m_FileHandle == nullptr);
 
 			std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) |
@@ -126,11 +137,14 @@ namespace detail {
 #endif	// ZEN_PLATFORM_WINDOWS
 			m_FileHandle = FileHandle;
 
+			PrepareFileForScatteredWrite(m_FileHandle, FinalSize);
+
 			return {};
 		}
 
 		std::error_code Write(std::string_view DataString)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Write");
 			const uint8_t* DataPtr	= (const uint8_t*)DataString.data();
 			size_t		   DataSize = DataString.size();
 			if (DataSize >= CacheBufferSize)
@@ -165,6 +179,7 @@ namespace detail {
 
 		IoBuffer DetachToIoBuffer()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::DetachToIoBuffer");
 			if (std::error_code Ec = Flush(); Ec)
 			{
 				ThrowSystemError(Ec.value(), Ec.message());
@@ -180,6 +195,7 @@ namespace detail {
 
 		IoBuffer BorrowIoBuffer()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::BorrowIoBuffer");
 			if (std::error_code Ec = Flush(); Ec)
 			{
 				ThrowSystemError(Ec.value(), Ec.message());
@@ -193,6 +209,7 @@ namespace detail {
 		uint64_t GetSize() const { return m_WriteOffset; }
 		void	 ResetWritePos(uint64_t WriteOffset)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::ResetWritePos");
 			Flush();
 			m_WriteOffset = WriteOffset;
 		}
@@ -200,6 +217,7 @@ namespace detail {
 	private:
 		std::error_code Flush()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Flush");
 			if (m_CacheBufferOffset == 0)
 			{
 				return {};
@@ -211,6 +229,7 @@ namespace detail {
 
 		std::error_code AppendData(const void* Data, uint64_t Size)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::AppendData");
 			ZEN_ASSERT(m_FileHandle != nullptr);
 			const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
 
@@ -261,6 +280,167 @@ namespace detail {
 		std::uint64_t			  m_CacheBufferOffset = 0;
 	};
 
+	class BufferedReadFileStream
+	{
+	public:
+		BufferedReadFileStream(const BufferedReadFileStream&) = delete;
+		BufferedReadFileStream& operator=(const BufferedReadFileStream&) = delete;
+
+		BufferedReadFileStream(void* FileHandle, uint64_t FileOffset, uint64_t FileSize, uint64_t BufferSize)
+		: m_FileHandle(FileHandle)
+		, m_FileSize(FileSize)
+		, m_FileEnd(FileOffset + FileSize)
+		, m_BufferSize(Min(BufferSize, FileSize))
+		, m_FileOffset(FileOffset)
+		{
+		}
+
+		~BufferedReadFileStream() { Memory::Free(m_Buffer); }
+		void Read(void* Data, uint64_t Size)
+		{
+			ZEN_ASSERT(Data != nullptr);
+			if (Size > m_BufferSize)
+			{
+				Read(Data, Size, m_FileOffset);
+				m_FileOffset += Size;
+				return;
+			}
+			uint8_t* WritePtr = ((uint8_t*)Data);
+			uint64_t Begin	  = m_FileOffset;
+			uint64_t End	  = m_FileOffset + Size;
+			ZEN_ASSERT(m_FileOffset >= m_BufferStart);
+			if (m_FileOffset < m_BufferEnd)
+			{
+				ZEN_ASSERT(m_Buffer != nullptr);
+				uint64_t Count = Min(m_BufferEnd, End) - m_FileOffset;
+				memcpy(WritePtr + Begin - m_FileOffset, m_Buffer + Begin - m_BufferStart, Count);
+				Begin += Count;
+				if (Begin == End)
+				{
+					m_FileOffset = End;
+					return;
+				}
+			}
+			if (End == m_FileEnd)
+			{
+				Read(WritePtr + Begin - m_FileOffset, End - Begin, Begin);
+			}
+			else
+			{
+				if (!m_Buffer)
+				{
+					m_BufferSize = Min(m_FileEnd - m_FileOffset, m_BufferSize);
+					m_Buffer	 = (uint8_t*)Memory::Alloc(gsl::narrow<size_t>(m_BufferSize));
+				}
+				m_BufferStart = Begin;
+				m_BufferEnd	  = Min(Begin + m_BufferSize, m_FileEnd);
+				Read(m_Buffer, m_BufferEnd - m_BufferStart, m_BufferStart);
+				uint64_t Count = Min(m_BufferEnd, End) - m_BufferStart;
+				memcpy(WritePtr + Begin - m_FileOffset, m_Buffer, Count);
+				ZEN_ASSERT(Begin + Count == End);
+			}
+			m_FileOffset = End;
+		}
+
+	private:
+		void Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
+		{
+			const uint64_t	MaxChunkSize = 1u * 1024 * 1024;
+			std::error_code Ec;
+			ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec);
+
+			if (Ec)
+			{
+				std::error_code DummyEc;
+				throw std::system_error(
+					Ec,
+					fmt::format(
+						"HttpClient::BufferedReadFileStream ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
+						FileOffset,
+						BytesToRead,
+						PathFromHandle(m_FileHandle, DummyEc).generic_string(),
+						m_FileSize));
+			}
+		}
+
+		void*		   m_FileHandle	 = nullptr;
+		const uint64_t m_FileSize	 = 0;
+		const uint64_t m_FileEnd	 = 0;
+		uint64_t	   m_BufferSize	 = 0;
+		uint8_t*	   m_Buffer		 = nullptr;
+		uint64_t	   m_BufferStart = 0;
+		uint64_t	   m_BufferEnd	 = 0;
+		uint64_t	   m_FileOffset	 = 0;
+	};
+
+	class CompositeBufferReadStream
+	{
+	public:
+		CompositeBufferReadStream(const CompositeBuffer& Data, uint64_t BufferSize)
+		: m_Data(Data)
+		, m_BufferSize(BufferSize)
+		, m_SegmentIndex(0)
+		, m_BytesLeftInSegment(0)
+		{
+		}
+		uint64_t Read(void* Data, uint64_t Size)
+		{
+			uint64_t Result	  = 0;
+			uint8_t* WritePtr = (uint8_t*)Data;
+			while ((Size > 0) && (m_SegmentIndex < m_Data.GetSegments().size()))
+			{
+				if (m_BytesLeftInSegment == 0)
+				{
+					const SharedBuffer&	  Segment = m_Data.GetSegments()[m_SegmentIndex];
+					IoBufferFileReference FileRef = {nullptr, 0, 0};
+					if (Segment.AsIoBuffer().GetFileReference(FileRef))
+					{
+						m_SegmentDiskBuffer = std::make_unique<BufferedReadFileStream>(FileRef.FileHandle,
+																					   FileRef.FileChunkOffset,
+																					   FileRef.FileChunkSize,
+																					   m_BufferSize);
+					}
+					else
+					{
+						m_SegmentMemoryBuffer = Segment.GetView();
+					}
+					m_BytesLeftInSegment = Segment.GetSize();
+				}
+				uint64_t BytesToRead = Min(m_BytesLeftInSegment, Size);
+				if (m_SegmentDiskBuffer)
+				{
+					m_SegmentDiskBuffer->Read(WritePtr, BytesToRead);
+				}
+				else
+				{
+					ZEN_ASSERT_SLOW(m_SegmentMemoryBuffer.GetSize() >= BytesToRead);
+					memcpy(WritePtr, m_SegmentMemoryBuffer.GetData(), BytesToRead);
+					m_SegmentMemoryBuffer.MidInline(BytesToRead);
+				}
+				WritePtr += BytesToRead;
+				Size -= BytesToRead;
+				Result += BytesToRead;
+
+				m_BytesLeftInSegment -= BytesToRead;
+				if (m_BytesLeftInSegment == 0)
+				{
+					m_SegmentDiskBuffer.reset();
+					m_SegmentMemoryBuffer.Reset();
+					m_SegmentIndex++;
+				}
+			}
+			return Result;
+		}
+
+	private:
+		const CompositeBuffer&					m_Data;
+		const uint64_t							m_BufferSize;
+		size_t									m_SegmentIndex;
+		std::unique_ptr<BufferedReadFileStream> m_SegmentDiskBuffer;
+		MemoryView								m_SegmentMemoryBuffer;
+		uint64_t								m_BytesLeftInSegment;
+	};
+
 }  // namespace detail
 
 //////////////////////////////////////////////////////////////////////////
@@ -314,7 +494,11 @@ CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffe
 	const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code);
 	if (HttpResponse.error)
 	{
-		ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse);
+		if (HttpResponse.error.code != cpr::ErrorCode::OPERATION_TIMEDOUT &&
+			HttpResponse.error.code != cpr::ErrorCode::CONNECTION_FAILURE && HttpResponse.error.code != cpr::ErrorCode::REQUEST_CANCELLED)
+		{
+			ZEN_WARN("HttpClient client failure (session: {}): {}", SessionId, HttpResponse);
+		}
 
 		// Client side failure code
 		return HttpClient::Response{
@@ -353,9 +537,10 @@ ShouldRetry(const cpr::Response& Response)
 	{
 		case cpr::ErrorCode::OK:
 			break;
-		case cpr::ErrorCode::OPERATION_TIMEDOUT:
+		case cpr::ErrorCode::INTERNAL_ERROR:
 		case cpr::ErrorCode::NETWORK_RECEIVE_ERROR:
 		case cpr::ErrorCode::NETWORK_SEND_FAILURE:
+		case cpr::ErrorCode::OPERATION_TIMEDOUT:
 			return true;
 		default:
 			return false;
@@ -365,6 +550,7 @@ ShouldRetry(const cpr::Response& Response)
 		case HttpResponseCode::RequestTimeout:
 		case HttpResponseCode::TooManyRequests:
 		case HttpResponseCode::InternalServerError:
+		case HttpResponseCode::BadGateway:
 		case HttpResponseCode::ServiceUnavailable:
 		case HttpResponseCode::GatewayTimeout:
 			return true;
@@ -376,6 +562,7 @@ ShouldRetry(const cpr::Response& Response)
 static bool
 ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile>& PayloadFile)
 {
+	ZEN_TRACE_CPU("ValidatePayload");
 	IoBuffer ResponseBuffer = (Response.text.empty() && PayloadFile) ? PayloadFile->BorrowIoBuffer()
 																	 : IoBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size());
 
@@ -397,6 +584,11 @@ ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile
 		}
 	}
 
+	if (Response.status_code == (long)HttpResponseCode::PartialContent)
+	{
+		return true;
+	}
+
 	if (auto JupiterHash = Response.header.find("X-Jupiter-IoHash"); JupiterHash != Response.header.end())
 	{
 		IoHash ExpectedPayloadHash;
@@ -414,11 +606,6 @@ ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile
 		}
 	}
 
-	if (Response.status_code == (long)HttpResponseCode::PartialContent)
-	{
-		return true;
-	}
-
 	if (auto ContentType = Response.header.find("Content-Type"); ContentType != Response.header.end())
 	{
 		if (ContentType->second == "application/x-ue-comp")
@@ -535,12 +722,14 @@ struct HttpClient::Impl : public RefCounted
 		inline cpr::Session* operator->() const { return CprSession; }
 		inline cpr::Response Get()
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Get");
 			cpr::Response Result = CprSession->Get();
 			ZEN_TRACE("GET {}", Result);
 			return Result;
 		}
 		inline cpr::Response Download(cpr::WriteCallback&& Write, std::optional<cpr::HeaderCallback>&& Header = {})
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Download");
 			if (Header)
 			{
 				CprSession->SetHeaderCallback(std::move(Header.value()));
@@ -553,12 +742,14 @@ struct HttpClient::Impl : public RefCounted
 		}
 		inline cpr::Response Head()
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Head");
 			cpr::Response Result = CprSession->Head();
 			ZEN_TRACE("HEAD {}", Result);
 			return Result;
 		}
 		inline cpr::Response Put(std::optional<cpr::ReadCallback>&& Read = {})
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Put");
 			if (Read)
 			{
 				CprSession->SetReadCallback(std::move(Read.value()));
@@ -570,6 +761,7 @@ struct HttpClient::Impl : public RefCounted
 		}
 		inline cpr::Response Post(std::optional<cpr::ReadCallback>&& Read = {})
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Post");
 			if (Read)
 			{
 				CprSession->SetReadCallback(std::move(Read.value()));
@@ -581,6 +773,7 @@ struct HttpClient::Impl : public RefCounted
 		}
 		inline cpr::Response Delete()
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Delete");
 			cpr::Response Result = CprSession->Delete();
 			ZEN_TRACE("DELETE {}", Result);
 			return Result;
@@ -620,6 +813,7 @@ HttpClient::Impl::Impl(LoggerRef Log) : m_Log(Log)
 
 HttpClient::Impl::~Impl()
 {
+	ZEN_TRACE_CPU("HttpClient::Impl::~Impl");
 	m_SessionLock.WithExclusiveLock([&] {
 		for (auto CprSession : m_Sessions)
 		{
@@ -638,6 +832,7 @@ HttpClient::Impl::AllocSession(const std::string_view				BaseUrl,
 							   const std::string_view				SessionId,
 							   std::optional<HttpClientAccessToken> AccessToken)
 {
+	ZEN_TRACE_CPU("HttpClient::Impl::AllocSession");
 	cpr::Session* CprSession = nullptr;
 	m_SessionLock.WithExclusiveLock([&] {
 		if (!m_Sessions.empty())
@@ -694,6 +889,7 @@ HttpClient::Impl::AllocSession(const std::string_view				BaseUrl,
 void
 HttpClient::Impl::ReleaseSession(cpr::Session* CprSession)
 {
+	ZEN_TRACE_CPU("HttpClient::Impl::ReleaseSession");
 	CprSession->SetUrl({});
 	CprSession->SetHeader({});
 	CprSession->SetBody({});
@@ -718,6 +914,7 @@ HttpClient::~HttpClient()
 bool
 HttpClient::Authenticate()
 {
+	ZEN_TRACE_CPU("HttpClient::Authenticate");
 	std::optional<HttpClientAccessToken> Token = GetAccessToken();
 	if (!Token)
 	{
@@ -729,6 +926,7 @@ HttpClient::Authenticate()
 const std::optional<HttpClientAccessToken>
 HttpClient::GetAccessToken()
 {
+	ZEN_TRACE_CPU("HttpClient::GetAccessToken");
 	if (!m_ConnectionSettings.AccessTokenProvider.has_value())
 	{
 		return {};
@@ -977,9 +1175,22 @@ HttpClient::Post(std::string_view Url, const IoBuffer& Payload, ZenContentType C
 			[&]() {
 				Impl::Session Sess =
 					m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken());
+				Sess->UpdateHeader({HeaderContentType(ContentType)});
 
+				IoBufferFileReference FileRef = {nullptr, 0, 0};
+				if (Payload.GetFileReference(FileRef))
+				{
+					uint64_t					   Offset = 0;
+					detail::BufferedReadFileStream Buffer(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, 512u * 1024u);
+					auto						   ReadCallback = [&Payload, &Offset, &Buffer](char* buffer, size_t& size, intptr_t) {
+						  size = Min<size_t>(size, Payload.GetSize() - Offset);
+						  Buffer.Read(buffer, size);
+						  Offset += size;
+						  return true;
+					};
+					return Sess.Post(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(Payload.GetSize()), ReadCallback));
+				}
 				Sess->SetBody(AsCprBody(Payload));
-				Sess->UpdateHeader({HeaderContentType(ContentType)});
 				return Sess.Post();
 			},
 			m_ConnectionSettings.RetryCount));
@@ -1021,19 +1232,15 @@ HttpClient::Post(std::string_view Url, const CompositeBuffer& Payload, ZenConten
 		DoWithRetry(
 			m_SessionId,
 			[&]() {
-				uint64_t				  SizeLeft	   = Payload.GetSize();
-				CompositeBuffer::Iterator BufferIt	   = Payload.GetIterator(0);
-				auto					  ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) {
-					 size = Min<size_t>(size, SizeLeft);
-					 MutableMemoryView Data(buffer, size);
-					 Payload.CopyTo(Data, BufferIt);
-					 SizeLeft -= size;
-					 return true;
-				};
 				Impl::Session Sess =
 					m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken());
 				Sess->UpdateHeader({HeaderContentType(ContentType)});
 
+				detail::CompositeBufferReadStream Reader(Payload, 512u * 1024u);
+				auto							  ReadCallback = [&Reader](char* buffer, size_t& size, intptr_t) {
+					 size = Reader.Read(buffer, size);
+					 return true;
+				};
 				return Sess.Post(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(Payload.GetSize()), ReadCallback));
 			},
 			m_ConnectionSettings.RetryCount));
@@ -1053,16 +1260,16 @@ HttpClient::Upload(std::string_view Url, const IoBuffer& Payload, const KeyValue
 					m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken());
 				Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())});
 
-				uint64_t Offset = 0;
-				if (Payload.IsWholeFile())
+				IoBufferFileReference FileRef = {nullptr, 0, 0};
+				if (Payload.GetFileReference(FileRef))
 				{
-					auto ReadCallback = [&Payload, &Offset](char* buffer, size_t& size, intptr_t) {
-						size						   = Min<size_t>(size, Payload.GetSize() - Offset);
-						IoBuffer		  PayloadRange = IoBuffer(Payload, Offset, size);
-						MutableMemoryView Data(buffer, size);
-						Data.CopyFrom(PayloadRange.GetView());
-						Offset += size;
-						return true;
+					uint64_t					   Offset = 0;
+					detail::BufferedReadFileStream Buffer(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, 512u * 1024u);
+					auto						   ReadCallback = [&Payload, &Offset, &Buffer](char* buffer, size_t& size, intptr_t) {
+						  size = Min<size_t>(size, Payload.GetSize() - Offset);
+						  Buffer.Read(buffer, size);
+						  Offset += size;
+						  return true;
 					};
 					return Sess.Put(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(Payload.GetSize()), ReadCallback));
 				}
@@ -1086,13 +1293,9 @@ HttpClient::Upload(std::string_view Url, const CompositeBuffer& Payload, ZenCont
 					m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken());
 				Sess->UpdateHeader({HeaderContentType(ContentType)});
 
-				uint64_t				  SizeLeft	   = Payload.GetSize();
-				CompositeBuffer::Iterator BufferIt	   = Payload.GetIterator(0);
-				auto					  ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) {
-					 size = Min<size_t>(size, SizeLeft);
-					 MutableMemoryView Data(buffer, size);
-					 Payload.CopyTo(Data, BufferIt);
-					 SizeLeft -= size;
+				detail::CompositeBufferReadStream Reader(Payload, 512u * 1024u);
+				auto							  ReadCallback = [&Reader](char* buffer, size_t& size, intptr_t) {
+					 size = Reader.Read(buffer, size);
 					 return true;
 				};
 				return Sess.Put(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(Payload.GetSize()), ReadCallback));
@@ -1185,7 +1388,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
 							   if (ContentLength.value() > 1024 * 1024)
 							   {
 								   PayloadFile = std::make_unique<detail::TempPayloadFile>();
-								   std::error_code Ec = PayloadFile->Open(TempFolderPath);
+								   std::error_code Ec = PayloadFile->Open(TempFolderPath, ContentLength.value());
 								   if (Ec)
 								   {
 									   ZEN_WARN("Failed to create temp file in '{}' for HttpClient::Download. Reason: {}",
@@ -1451,6 +1654,40 @@ HttpClient::Response::ThrowError(std::string_view ErrorPrefix)
 
 #if ZEN_WITH_TESTS
 
+namespace testutil {
+	IoHash HashComposite(const CompositeBuffer& Payload)
+	{
+		IoHashStream					  Hasher;
+		const uint64_t					  PayloadSize = Payload.GetSize();
+		std::vector<uint8_t>			  Buffer(64u * 1024u);
+		detail::CompositeBufferReadStream Stream(Payload, 137u * 1024u);
+		for (uint64_t Offset = 0; Offset < PayloadSize;)
+		{
+			uint64_t Count = Min(64u * 1024u, PayloadSize - Offset);
+			Stream.Read(Buffer.data(), Count);
+			Hasher.Append(Buffer.data(), Count);
+			Offset += Count;
+		}
+		return Hasher.GetHash();
+	};
+
+	IoHash HashFileStream(void* FileHandle, uint64_t FileOffset, uint64_t FileSize)
+	{
+		IoHashStream				   Hasher;
+		std::vector<uint8_t>		   Buffer(64u * 1024u);
+		detail::BufferedReadFileStream Stream(FileHandle, FileOffset, FileSize, 137u * 1024u);
+		for (uint64_t Offset = 0; Offset < FileSize;)
+		{
+			uint64_t Count = Min(64u * 1024u, FileSize - Offset);
+			Stream.Read(Buffer.data(), Count);
+			Hasher.Append(Buffer.data(), Count);
+			Offset += Count;
+		}
+		return Hasher.GetHash();
+	}
+
+}  // namespace testutil
+
 TEST_CASE("responseformat")
 {
 	using namespace std::literals;
@@ -1497,6 +1734,53 @@ TEST_CASE("responseformat")
 	}
 }
 
+TEST_CASE("BufferedReadFileStream")
+{
+	ScopedTemporaryDirectory TmpDir;
+
+	IoBuffer DiskBuffer = WriteToTempFile(CompositeBuffer(CreateRandomBlob(496 * 5 * 1024)), TmpDir.Path() / "diskbuffer1");
+
+	IoBufferFileReference FileRef = {nullptr, 0, 0};
+	CHECK(DiskBuffer.GetFileReference(FileRef));
+	CHECK_EQ(IoHash::HashBuffer(DiskBuffer), testutil::HashFileStream(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize));
+
+	IoBuffer Partial(DiskBuffer, 37 * 1024, 512 * 1024);
+	CHECK(Partial.GetFileReference(FileRef));
+	CHECK_EQ(IoHash::HashBuffer(Partial), testutil::HashFileStream(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize));
+
+	IoBuffer SmallDiskBuffer = WriteToTempFile(CompositeBuffer(CreateRandomBlob(63 * 1024)), TmpDir.Path() / "diskbuffer2");
+	CHECK(SmallDiskBuffer.GetFileReference(FileRef));
+	CHECK_EQ(IoHash::HashBuffer(SmallDiskBuffer),
+			 testutil::HashFileStream(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize));
+}
+
+TEST_CASE("CompositeBufferReadStream")
+{
+	ScopedTemporaryDirectory TmpDir;
+
+	IoBuffer MemoryBuffer1 = CreateRandomBlob(64);
+	CHECK_EQ(IoHash::HashBuffer(MemoryBuffer1), testutil::HashComposite(CompositeBuffer(SharedBuffer(MemoryBuffer1))));
+
+	IoBuffer MemoryBuffer2 = CreateRandomBlob(561 * 1024);
+	CHECK_EQ(IoHash::HashBuffer(MemoryBuffer2), testutil::HashComposite(CompositeBuffer(SharedBuffer(MemoryBuffer2))));
+
+	IoBuffer DiskBuffer1 = WriteToTempFile(CompositeBuffer(CreateRandomBlob(267 * 3 * 1024)), TmpDir.Path() / "diskbuffer1");
+	CHECK_EQ(IoHash::HashBuffer(DiskBuffer1), testutil::HashComposite(CompositeBuffer(SharedBuffer(DiskBuffer1))));
+
+	IoBuffer DiskBuffer2 = WriteToTempFile(CompositeBuffer(CreateRandomBlob(3 * 1024)), TmpDir.Path() / "diskbuffer2");
+	CHECK_EQ(IoHash::HashBuffer(DiskBuffer2), testutil::HashComposite(CompositeBuffer(SharedBuffer(DiskBuffer2))));
+
+	IoBuffer DiskBuffer3 = WriteToTempFile(CompositeBuffer(CreateRandomBlob(496 * 5 * 1024)), TmpDir.Path() / "diskbuffer3");
+	CHECK_EQ(IoHash::HashBuffer(DiskBuffer3), testutil::HashComposite(CompositeBuffer(SharedBuffer(DiskBuffer3))));
+
+	CompositeBuffer Data(SharedBuffer(std::move(MemoryBuffer1)),
+						 SharedBuffer(std::move(DiskBuffer1)),
+						 SharedBuffer(std::move(DiskBuffer2)),
+						 SharedBuffer(std::move(MemoryBuffer2)),
+						 SharedBuffer(std::move(DiskBuffer3)));
+	CHECK_EQ(IoHash::HashBuffer(Data), testutil::HashComposite(Data));
+}
+
 TEST_CASE("httpclient")
 {
 	using namespace std::literals;
diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp
index 7fb3224f1..39efe1d0c 100644
--- a/src/zenhttp/httpclientauth.cpp
+++ b/src/zenhttp/httpclientauth.cpp
@@ -2,15 +2,26 @@
 
 #include <zenhttp/httpclientauth.h>
 
+#include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/uid.h>
 #include <zenhttp/auth/authmgr.h>
 
+#include <ctime>
+
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <cpr/cpr.h>
 #include <fmt/format.h>
 #include <json11.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#if ZEN_PLATFORM_WINDOWS
+#	define timegm _mkgmtime
+#endif	// ZEN_PLATFORM_WINDOWS
+
 namespace zen { namespace httpclientauth {
 
 	using namespace std::literals;
@@ -76,4 +87,101 @@ namespace zen { namespace httpclientauth {
 		return CreateFromOpenIdProvider(AuthManager, "Default"sv);
 	}
 
+	static HttpClientAccessToken GetOidcTokenFromExe(const std::filesystem::path& OidcExecutablePath,
+													 std::string_view			  CloudHost,
+													 bool						  Unattended)
+	{
+		Stopwatch Timer;
+
+		CreateProcOptions ProcOptions;
+
+		const std::filesystem::path AuthTokenPath(std::filesystem::temp_directory_path() / fmt::format(".zen-auth-{}", Oid::NewOid()));
+		auto						_ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); });
+
+		const std::string ProcArgs = fmt::format("{} --AuthConfigUrl {} --OutFile {} --Unattended={}",
+												 OidcExecutablePath,
+												 CloudHost,
+												 AuthTokenPath,
+												 Unattended ? "true"sv : "false"sv);
+		ZEN_DEBUG("Running: {}", ProcArgs);
+		ProcessHandle Proc;
+		Proc.Initialize(CreateProc(OidcExecutablePath, ProcArgs, ProcOptions));
+		if (!Proc.IsValid())
+		{
+			throw std::runtime_error(fmt::format("failed to launch '{}'", OidcExecutablePath));
+		}
+
+		int ExitCode = Proc.WaitExitCode();
+
+		auto EndTime = std::chrono::system_clock::now();
+
+		if (ExitCode == 0)
+		{
+			IoBuffer	Body = IoBufferBuilder::MakeFromFile(AuthTokenPath);
+			std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize());
+
+			std::string	 JsonError;
+			json11::Json Json = json11::Json::parse(JsonText, JsonError);
+
+			if (JsonError.empty() == false)
+			{
+				ZEN_WARN("Unable to parse Oidcs json response from {}. Reason: '{}'", AuthTokenPath, JsonError);
+				return HttpClientAccessToken{};
+			}
+			std::string Token			   = Json["Token"].string_value();
+			std::string ExpiresAtUTCString = Json["ExpiresAtUtc"].string_value();
+			ZEN_ASSERT(!ExpiresAtUTCString.empty());
+
+			int Year		= 0;
+			int Month		= 0;
+			int Day			= 0;
+			int Hour		= 0;
+			int Minute		= 0;
+			int Second		= 0;
+			int Millisecond = 0;
+			sscanf(ExpiresAtUTCString.c_str(), "%d-%d-%dT%d:%d:%d.%dZ", &Year, &Month, &Day, &Hour, &Minute, &Second, &Millisecond);
+
+			std::tm Time = {
+				Second,
+				Minute,
+				Hour,
+				Day,
+				Month - 1,
+				Year - 1900,
+			};
+
+			time_t							 UTCTime	= timegm(&Time);
+			HttpClientAccessToken::TimePoint ExpireTime = std::chrono::system_clock::from_time_t(UTCTime);
+			ExpireTime += std::chrono::microseconds(Millisecond);
+
+			return HttpClientAccessToken{.Value = fmt::format("Bearer {}"sv, Token), .ExpireTime = ExpireTime};
+		}
+		else
+		{
+			ZEN_WARN("Failed running {} to get auth token, error code {}", OidcExecutablePath, ExitCode);
+		}
+		return HttpClientAccessToken{};
+	}
+
+	std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath,
+																						std::string_view			 CloudHost)
+	{
+		HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, false);
+		if (InitialToken.IsValid())
+		{
+			return [OidcExecutablePath = std::filesystem::path(OidcExecutablePath),
+					CloudHost		   = std::string(CloudHost),
+					InitialToken]() mutable {
+				if (InitialToken.IsValid())
+				{
+					HttpClientAccessToken Result = InitialToken;
+					InitialToken				 = {};
+					return Result;
+				}
+				return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, true);
+			};
+		}
+		return {};
+	}
+
 }}	// namespace zen::httpclientauth
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 27a09f339..764f2a2a7 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -787,120 +787,131 @@ HttpRpcHandler::AddRpc(std::string_view RpcId, std::function<void(CbObject& RpcA
 
 //////////////////////////////////////////////////////////////////////////
 
-enum class HttpServerClass
-{
-	kHttpAsio,
-	kHttpSys,
-	kHttpPlugin,
-	kHttpMulti,
-	kHttpNull
-};
-
 Ref<HttpServer>
-CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config)
+CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig& Config)
 {
-	switch (Class)
+	if (ServerClass == "asio"sv)
 	{
-		default:
-		case HttpServerClass::kHttpAsio:
-			ZEN_INFO("using asio HTTP server implementation");
-			return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount);
-
-		case HttpServerClass::kHttpMulti:
-			{
-				ZEN_INFO("using multi HTTP server implementation");
-				Ref<HttpMultiServer> Server{new HttpMultiServer()};
-
-				// This is hardcoded for now, but should be configurable in the future
-				Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpSys, Config));
-				Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpPlugin, Config));
+		ZEN_INFO("using asio HTTP server implementation")
+		return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount);
+	}
+#if ZEN_WITH_HTTPSYS
+	else if (ServerClass == "httpsys"sv)
+	{
+		ZEN_INFO("using http.sys server implementation")
+		return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount			 = Config.ThreadCount,
+													.AsyncWorkThreadCount	 = Config.HttpSys.AsyncWorkThreadCount,
+													.IsAsyncResponseEnabled	 = Config.HttpSys.IsAsyncResponseEnabled,
+													.IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
+													.IsDedicatedServer		 = Config.IsDedicatedServer,
+													.ForceLoopback			 = Config.ForceLoopback}));
+	}
+#endif
+	else if (ServerClass == "null"sv)
+	{
+		ZEN_INFO("using null HTTP server implementation")
+		return Ref<HttpServer>(new HttpNullServer);
+	}
+	else
+	{
+		ZEN_WARN("unknown HTTP server implementation '{}', falling back to default", ServerClass)
 
-				return Server;
-			}
+#if ZEN_WITH_HTTPSYS
+		return CreateHttpServerClass("httpsys"sv, Config);
+#else
+		return CreateHttpServerClass("asio"sv, Config);
+#endif
+	}
+}
 
 #if ZEN_WITH_PLUGINS
-		case HttpServerClass::kHttpPlugin:
-			{
-				ZEN_INFO("using plugin HTTP server implementation");
-				Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+Ref<HttpServer>
+CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig)
+{
+	const std::string& PluginName = PluginConfig.PluginName;
 
-				// This is hardcoded for now, but should be configurable in the future
+	ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName)
 
+	if (PluginName.starts_with("builtin:"sv))
+	{
 #	if 0
-				Ref<TransportPlugin> WinsockPlugin{CreateSocketTransportPlugin()};
-				WinsockPlugin->Configure("port", "8558");
-				Server->AddPlugin(WinsockPlugin);
-#	endif
+		Ref<TransportPlugin> Plugin = {};
+		if (PluginName == "builtin:winsock"sv)
+		{
+			Plugin = CreateSocketTransportPlugin();
+		}
+		else if (PluginName == "builtin:asio"sv)
+		{
+			Plugin = CreateAsioTransportPlugin();
+		}
+		else
+		{
+			ZEN_WARN("Unknown builtin plugin '{}'", PluginName)
+			return {};
+		}
 
-#	if 0
-				Ref<TransportPlugin> AsioPlugin{CreateAsioTransportPlugin()};
-				AsioPlugin->Configure("port", "8558");
-				Server->AddPlugin(AsioPlugin);
-#	endif
+		ZEN_ASSERT(!Plugin.IsNull());
 
-#	if 1
-				Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()};
-				DllPlugin->LoadDll("winsock");
-				DllPlugin->ConfigureDll("winsock", "port", "8558");
-				Server->AddPlugin(DllPlugin);
-#	endif
+		for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions)
+		{
+			Plugin->Configure(Option.first.c_str(), Option.second.c_str());
+		}
 
-				return Server;
-			}
-#endif
+		Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+		Server->AddPlugin(Plugin);
+		return Server;
+#	else
+		ZEN_WARN("Builtin plugin '{}' is not supported", PluginName)
+		return {};
+#	endif
+	}
 
-#if ZEN_WITH_HTTPSYS
-		case HttpServerClass::kHttpSys:
-			ZEN_INFO("using http.sys server implementation");
-			return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount			 = Config.ThreadCount,
-														.AsyncWorkThreadCount	 = Config.HttpSys.AsyncWorkThreadCount,
-														.IsAsyncResponseEnabled	 = Config.HttpSys.IsAsyncResponseEnabled,
-														.IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
-														.IsDedicatedServer		 = Config.IsDedicatedServer,
-														.ForceLoopback			 = Config.ForceLoopback}));
-#endif
+	Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()};
+	if (!DllPlugin->LoadDll(PluginName))
+	{
+		return {};
+	}
 
-		case HttpServerClass::kHttpNull:
-			ZEN_INFO("using null HTTP server implementation");
-			return Ref<HttpServer>(new HttpNullServer);
+	for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions)
+	{
+		DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str());
 	}
+
+	Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+	Server->AddPlugin(DllPlugin);
+	return Server;
 }
+#endif
 
 Ref<HttpServer>
 CreateHttpServer(const HttpServerConfig& Config)
 {
 	using namespace std::literals;
 
-	HttpServerClass Class = HttpServerClass::kHttpNull;
-
-#if ZEN_WITH_HTTPSYS
-	Class = HttpServerClass::kHttpSys;
-#else
-	Class = HttpServerClass::kHttpAsio;
-#endif
-
-	if (Config.ServerClass == "asio"sv)
-	{
-		Class = HttpServerClass::kHttpAsio;
-	}
-	else if (Config.ServerClass == "httpsys"sv)
-	{
-		Class = HttpServerClass::kHttpSys;
-	}
-	else if (Config.ServerClass == "plugin"sv)
-	{
-		Class = HttpServerClass::kHttpPlugin;
-	}
-	else if (Config.ServerClass == "null"sv)
+#if ZEN_WITH_PLUGINS
+	if (Config.PluginConfigs.empty())
 	{
-		Class = HttpServerClass::kHttpNull;
+		return CreateHttpServerClass(Config.ServerClass, Config);
 	}
-	else if (Config.ServerClass == "multi"sv)
+	else
 	{
-		Class = HttpServerClass::kHttpMulti;
-	}
+		Ref<HttpMultiServer> Server{new HttpMultiServer()};
+		Server->AddServer(CreateHttpServerClass(Config.ServerClass, Config));
 
-	return CreateHttpServerClass(Class, Config);
+		for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs)
+		{
+			Ref<HttpServer> PluginServer = CreateHttpServerPlugin(PluginConfig);
+			if (!PluginServer.IsNull())
+			{
+				Server->AddServer(PluginServer);
+			}
+		}
+
+		return Server;
+	}
+#else
+	return CreateHttpServerClass(Config.ServerClass, Config);
+#endif
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h
index 74da9ab05..05a23d675 100644
--- a/src/zenhttp/include/zenhttp/formatters.h
+++ b/src/zenhttp/include/zenhttp/formatters.h
@@ -73,9 +73,11 @@ struct fmt::formatter<cpr::Response>
 		if (zen::IsHttpSuccessCode(Response.status_code))
 		{
 			return fmt::format_to(Ctx.out(),
-								  "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}",
+								  "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}",
 								  Response.url.str(),
 								  Response.status_code,
+								  Response.error.message,
+								  int(Response.error.code),
 								  Response.uploaded_bytes,
 								  Response.downloaded_bytes,
 								  NiceResponseTime.c_str());
@@ -92,29 +94,35 @@ struct fmt::formatter<cpr::Response>
 				zen::ExtendableStringBuilder<256> Sb;
 				std::string_view				  Json = Obj.ToJson(Sb).ToView();
 
-				return fmt::format_to(Ctx.out(),
-									  "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
-									  Response.url.str(),
-									  Response.status_code,
-									  Response.uploaded_bytes,
-									  Response.downloaded_bytes,
-									  NiceResponseTime.c_str(),
-									  Json,
-									  Response.reason);
+				return fmt::format_to(
+					Ctx.out(),
+					"Url: {}, Status: {}, Error: '{}' ({}). Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
+					Response.url.str(),
+					Response.status_code,
+					Response.error.message,
+					int(Response.error.code),
+					Response.uploaded_bytes,
+					Response.downloaded_bytes,
+					NiceResponseTime.c_str(),
+					Json,
+					Response.reason);
 			}
 			else
 			{
 				zen::BodyLogFormatter Body(Response.text);
 
-				return fmt::format_to(Ctx.out(),
-									  "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
-									  Response.url.str(),
-									  Response.status_code,
-									  Response.uploaded_bytes,
-									  Response.downloaded_bytes,
-									  NiceResponseTime.c_str(),
-									  Body.GetText(),
-									  Response.reason);
+				return fmt::format_to(
+					Ctx.out(),
+					"Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
+					Response.url.str(),
+					Response.status_code,
+					Response.error.message,
+					int(Response.error.code),
+					Response.uploaded_bytes,
+					Response.downloaded_bytes,
+					NiceResponseTime.c_str(),
+					Body.GetText(),
+					Response.reason);
 			}
 		}
 	}
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index a46b9fd83..c991a71ea 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -34,7 +34,7 @@ struct HttpClientAccessToken
 	using Clock		= std::chrono::system_clock;
 	using TimePoint = Clock::time_point;
 
-	static constexpr int64_t ExpireMarginInSeconds = 30;
+	static constexpr int64_t ExpireMarginInSeconds = 60 * 5;
 
 	std::string Value;
 	TimePoint	ExpireTime;
@@ -97,7 +97,7 @@ public:
 		HttpResponseCode StatusCode = HttpResponseCode::ImATeapot;
 		IoBuffer		 ResponsePayload;  // Note: this also includes the content type
 
-		// Contains the reponse headers
+		// Contains the response headers
 		KeyValueMap Header;
 
 		// The number of bytes sent as part of the request
diff --git a/src/zenhttp/include/zenhttp/httpclientauth.h b/src/zenhttp/include/zenhttp/httpclientauth.h
index aa07620ca..5b9b9d305 100644
--- a/src/zenhttp/include/zenhttp/httpclientauth.h
+++ b/src/zenhttp/include/zenhttp/httpclientauth.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <zenhttp/httpclient.h>
+#include <optional>
 
 namespace zen {
 
@@ -24,6 +25,9 @@ namespace httpclientauth {
 
 	std::function<HttpClientAccessToken()> CreateFromOpenIdProvider(AuthMgr& AuthManager, std::string_view OpenIdProvider);
 	std::function<HttpClientAccessToken()> CreateFromDefaultOpenIdProvider(AuthMgr& AuthManager);
+
+	std::optional<std::function<HttpClientAccessToken()>> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath,
+																						std::string_view			 CloudHost);
 }  // namespace httpclientauth
 
 }  // namespace zen
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index 217455dba..03e547bf3 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -184,12 +184,19 @@ public:
 	virtual void Close()												 = 0;
 };
 
+struct HttpServerPluginConfig
+{
+	std::string										 PluginName;
+	std::vector<std::pair<std::string, std::string>> PluginOptions;
+};
+
 struct HttpServerConfig
 {
-	bool		 IsDedicatedServer = false;	 // Should be set to true for shared servers
-	std::string	 ServerClass;				 // Choice of HTTP server implementation
-	bool		 ForceLoopback = false;
-	unsigned int ThreadCount   = 0;
+	bool								IsDedicatedServer = false;	// Should be set to true for shared servers
+	std::string							ServerClass;				// Choice of HTTP server implementation
+	std::vector<HttpServerPluginConfig> PluginConfigs;
+	bool								ForceLoopback = false;
+	unsigned int						ThreadCount	  = 0;
 
 	struct
 	{
diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp
index ae80851e4..9d423ecbc 100644
--- a/src/zenhttp/packageformat.cpp
+++ b/src/zenhttp/packageformat.cpp
@@ -279,11 +279,10 @@ FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* Tar
 		{
 			IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer();
 			ZEN_ASSERT(ObjIoBuffer.GetSize() > 0);
-			ResponseBuffers.emplace_back(std::move(ObjIoBuffer));
-
 			*AttachmentInfo++ = {.PayloadSize	 = ObjIoBuffer.Size(),
 								 .Flags			 = CbAttachmentEntry::kIsObject,
 								 .AttachmentHash = Attachment.GetHash()};
+			ResponseBuffers.emplace_back(std::move(ObjIoBuffer));
 		}
 		else if (const CompositeBuffer& AttachmentBinary = Attachment.AsCompositeBinary())
 		{
@@ -500,30 +499,25 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 		{
 			if (Entry.Flags & CbAttachmentEntry::kIsObject)
 			{
+				CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
+				if (!CompBuf)
+				{
+					// First payload is always a compact binary object
+					MalformedAttachments.push_back(
+						std::make_pair(i,
+									   fmt::format("Invalid format, expected compressed buffer for CbObject (size {}) for {}",
+												   AttachmentBuffer.GetSize(),
+												   Entry.AttachmentHash)));
+				}
+				CbObject AttachmentObject = LoadCompactBinaryObject(std::move(CompBuf));
 				if (i == 0)
 				{
-					CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
-					if (CompBuf)
-					{
-						Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf)));
-					}
-					else
-					{
-						// First payload is always a compact binary object
-						MalformedAttachments.push_back(
-							std::make_pair(i,
-										   fmt::format("Invalid format, expected compressed buffer for CbObject (size {}) for {}",
-													   AttachmentBuffer.GetSize(),
-													   Entry.AttachmentHash)));
-					}
+					// First payload is always a compact binary object
+					Package.SetObject(AttachmentObject);
 				}
 				else
 				{
-					MalformedAttachments.push_back(std::make_pair(
-						i,
-						fmt::format("Invalid format, compressed object attachments are not currently supported (size {}) for {}",
-									AttachmentBuffer.GetSize(),
-									Entry.AttachmentHash)));
+					Attachments.emplace_back(CbAttachment(AttachmentObject, Entry.AttachmentHash));
 				}
 			}
 			else
@@ -547,17 +541,14 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 		{
 			if (Entry.Flags & CbAttachmentEntry::kIsObject)
 			{
+				CbObject AttachmentObject = LoadCompactBinaryObject(AttachmentBuffer);
 				if (i == 0)
 				{
-					Package.SetObject(LoadCompactBinaryObject(AttachmentBuffer));
+					Package.SetObject(AttachmentObject);
 				}
 				else
 				{
-					MalformedAttachments.push_back(
-						std::make_pair(i,
-									   fmt::format("Invalid format, object attachments are not currently supported (size {}) for {}",
-												   AttachmentBuffer.GetSize(),
-												   Entry.AttachmentHash)));
+					Attachments.emplace_back(CbAttachment(AttachmentObject, Entry.AttachmentHash));
 				}
 			}
 			else if (AttachmentSize > 0)
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index fe59e3a6f..c1b7294c9 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -678,7 +678,7 @@ struct HttpAcceptor
 		if (BindErrorCode == asio::error::address_in_use)
 		{
 			// Do a retry after a short sleep on same port just to be sure
-			ZEN_INFO("Desired port %d is in use, retrying", BasePort);
+			ZEN_INFO("Desired port {} is in use, retrying", BasePort);
 			Sleep(100);
 			m_Acceptor.bind(asio::ip::tcp::endpoint(BindAddress, EffectivePort), BindErrorCode);
 		}
@@ -697,13 +697,20 @@ struct HttpAcceptor
 		{
 			ZEN_ERROR("Unable open asio service, error '{}'", BindErrorCode.message());
 		}
-		else if (BindAddress.is_loopback())
+		else
 		{
-			m_AlternateProtocolAcceptor.bind(asio::ip::tcp::endpoint(asio::ip::address_v4::loopback(), EffectivePort), BindErrorCode);
-			m_UseAlternateProtocolAcceptor = true;
-			ZEN_INFO("Registered local-only handler 'http://{}:{}/' - this is not accessible from remote hosts",
-					 "localhost",
-					 EffectivePort);
+			if (EffectivePort != BasePort)
+			{
+				ZEN_WARN("Desired port {} is in use, remapped to port {}", BasePort, EffectivePort);
+			}
+			if (BindAddress.is_loopback())
+			{
+				m_AlternateProtocolAcceptor.bind(asio::ip::tcp::endpoint(asio::ip::address_v4::loopback(), EffectivePort), BindErrorCode);
+				m_UseAlternateProtocolAcceptor = true;
+				ZEN_INFO("Registered local-only handler 'http://{}:{}/' - this is not accessible from remote hosts",
+						 "localhost",
+						 EffectivePort);
+			}
 		}
 
 #if ZEN_PLATFORM_WINDOWS
diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp
index 2a6a90d2e..f4dc1e15b 100644
--- a/src/zenhttp/servers/httpmulti.cpp
+++ b/src/zenhttp/servers/httpmulti.cpp
@@ -103,6 +103,10 @@ HttpMultiServer::RequestExit()
 void
 HttpMultiServer::Close()
 {
+	for (auto& Server : m_Servers)
+	{
+		Server->Close();
+	}
 }
 
 void
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 3bdcdf098..62dab02c4 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -535,7 +535,14 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB
 
 	if (IoResult != NO_ERROR)
 	{
-		ZEN_WARN("response aborted due to error: {}", GetSystemErrorAsString(IoResult));
+		ZEN_WARN("response '{}' ({}) aborted after transfering '{}', {} out of {} bytes, reason: {} ({})",
+				 ReasonStringForHttpResultCode(m_ResponseCode),
+				 m_ResponseCode,
+				 ToString(m_ContentType),
+				 NumberOfBytesTransferred,
+				 m_TotalDataSize,
+				 GetSystemErrorAsString(IoResult),
+				 IoResult);
 
 		// if one transmit failed there's really no need to go on
 		return nullptr;
@@ -684,7 +691,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
 		);
 	}
 
-	auto EmitReponseDetails = [&](StringBuilderBase& ResponseDetails) -> void {
+	auto EmitResponseDetails = [&](StringBuilderBase& ResponseDetails) -> void {
 		for (int i = 0; i < ThisRequestChunkCount; ++i)
 		{
 			const HTTP_DATA_CHUNK Chunk = m_HttpDataChunks[ThisRequestChunkOffset + i];
@@ -767,7 +774,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
 		// Emit diagnostics
 
 		ExtendableStringBuilder<256> ResponseDetails;
-		EmitReponseDetails(ResponseDetails);
+		EmitResponseDetails(ResponseDetails);
 
 		ZEN_WARN("failed to send HTTP response (error {}: '{}'), request URL: '{}', ({}.{}) response: {}",
 				 SendResult,
diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp
index e09e62ec5..fb3dd23b5 100644
--- a/src/zenhttp/transports/dlltransport.cpp
+++ b/src/zenhttp/transports/dlltransport.cpp
@@ -21,18 +21,31 @@ namespace zen {
 
 //////////////////////////////////////////////////////////////////////////
 
+class DllTransportLogger : public TransportLogger, public RefCounted
+{
+public:
+	DllTransportLogger(std::string_view PluginName);
+	virtual ~DllTransportLogger() = default;
+
+	void LogMessage(LogLevel Level, const char* Message) override;
+
+private:
+	std::string m_PluginName;
+};
+
 struct LoadedDll
 {
 	std::string			  Name;
 	std::filesystem::path LoadedFromPath;
+	DllTransportLogger*	  Logger = nullptr;
 	Ref<TransportPlugin>  Plugin;
 };
 
 class DllTransportPluginImpl : public DllTransportPlugin, RefCounted
 {
 public:
-	DllTransportPluginImpl();
-	~DllTransportPluginImpl();
+	DllTransportPluginImpl()  = default;
+	~DllTransportPluginImpl() = default;
 
 	virtual uint32_t	AddRef() const override;
 	virtual uint32_t	Release() const override;
@@ -42,7 +55,7 @@ public:
 	virtual const char* GetDebugName() override;
 	virtual bool		IsAvailable() override;
 
-	virtual void LoadDll(std::string_view Name) override;
+	virtual bool LoadDll(std::string_view Name) override;
 	virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) override;
 
 private:
@@ -51,12 +64,27 @@ private:
 	std::vector<LoadedDll> m_Transports;
 };
 
-DllTransportPluginImpl::DllTransportPluginImpl()
+DllTransportLogger::DllTransportLogger(std::string_view PluginName) : m_PluginName(PluginName)
 {
 }
 
-DllTransportPluginImpl::~DllTransportPluginImpl()
+void
+DllTransportLogger::LogMessage(LogLevel PluginLogLevel, const char* Message)
 {
+	logging::level::LogLevel Level;
+	// clang-format off
+	switch (PluginLogLevel)
+	{
+		case LogLevel::Trace:    Level = logging::level::Trace;    break;
+		case LogLevel::Debug:    Level = logging::level::Debug;    break;
+		case LogLevel::Info:     Level = logging::level::Info;     break;
+		case LogLevel::Warn:     Level = logging::level::Warn;     break;
+		case LogLevel::Err:      Level = logging::level::Err;      break;
+		case LogLevel::Critical: Level = logging::level::Critical; break;
+		default:                 Level = logging::level::Off;      break;
+	}
+	// clang-format on
+	ZEN_LOG(Log(), Level, "[{}] {}", m_PluginName, Message)
 }
 
 uint32_t
@@ -109,6 +137,7 @@ DllTransportPluginImpl::Shutdown()
 		try
 		{
 			Transport.Plugin->Shutdown();
+			Transport.Logger->Release();
 		}
 		catch (const std::exception&)
 		{
@@ -143,42 +172,73 @@ DllTransportPluginImpl::ConfigureDll(std::string_view Name, const char* OptionTa
 	}
 }
 
-void
+bool
 DllTransportPluginImpl::LoadDll(std::string_view Name)
 {
 	RwLock::ExclusiveLockScope _(m_Lock);
 
-	ExtendableStringBuilder<128> DllPath;
-	DllPath << Name << ".dll";
+	ExtendableStringBuilder<1024> DllPath;
+	DllPath << Name;
+	if (!Name.ends_with(".dll"))
+	{
+		DllPath << ".dll";
+	}
+
+	std::string FileName = std::filesystem::path(DllPath.c_str()).filename().replace_extension().string();
+
 	HMODULE DllHandle = LoadLibraryA(DllPath.c_str());
 
 	if (!DllHandle)
 	{
-		std::error_code Ec = MakeErrorCodeFromLastError();
-
-		throw std::system_error(Ec, fmt::format("failed to load transport DLL from '{}'", DllPath));
+		ZEN_WARN("Failed to load transport DLL from '{}' due to '{}'", DllPath, GetLastErrorAsString())
+		return false;
 	}
 
-	TransportPlugin* CreateTransportPlugin();
+	PfnGetTransportPluginVersion GetVersion	  = (PfnGetTransportPluginVersion)GetProcAddress(DllHandle, "GetTransportPluginVersion");
+	PfnCreateTransportPlugin	 CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin");
+
+	uint32_t APIVersion	   = 0;
+	uint32_t PluginVersion = 0;
+
+	if (GetVersion)
+	{
+		GetVersion(&APIVersion, &PluginVersion);
+	}
 
-	PfnCreateTransportPlugin CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin");
+	const bool bValidApiVersion = APIVersion == kTransportApiVersion;
 
-	if (!CreatePlugin)
+	if (!GetVersion || !CreatePlugin || !bValidApiVersion)
 	{
 		std::error_code Ec = MakeErrorCodeFromLastError();
 
 		FreeLibrary(DllHandle);
 
-		throw std::system_error(Ec, fmt::format("API mismatch detected in transport DLL loaded from '{}'", DllPath));
+		if (GetVersion && !bValidApiVersion)
+		{
+			ZEN_WARN("Failed to load transport DLL from '{}' due to invalid API version {}, supported API version is {}",
+					 DllPath,
+					 APIVersion,
+					 kTransportApiVersion)
+		}
+		else
+		{
+			ZEN_WARN("Failed to load transport DLL from '{}' due to not finding GetTransportPluginVersion or CreateTransportPlugin",
+					 DllPath)
+		}
+
+		return false;
 	}
 
 	LoadedDll NewDll;
 
 	NewDll.Name			  = Name;
 	NewDll.LoadedFromPath = DllPath.c_str();
-	NewDll.Plugin		  = CreatePlugin();
+	NewDll.Logger		  = new DllTransportLogger(FileName);
+	NewDll.Logger->AddRef();
+	NewDll.Plugin = CreatePlugin(NewDll.Logger);
 
 	m_Transports.emplace_back(std::move(NewDll));
+	return true;
 }
 
 DllTransportPlugin*
diff --git a/src/zenhttp/transports/dlltransport.h b/src/zenhttp/transports/dlltransport.h
index 9346a10ce..c49f888da 100644
--- a/src/zenhttp/transports/dlltransport.h
+++ b/src/zenhttp/transports/dlltransport.h
@@ -15,7 +15,7 @@ namespace zen {
 class DllTransportPlugin : public TransportPlugin
 {
 public:
-	virtual void LoadDll(std::string_view Name)														 = 0;
+	virtual bool LoadDll(std::string_view Name)														 = 0;
 	virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) = 0;
 };
 
diff --git a/src/zennet-test/zennet-test.cpp b/src/zennet-test/zennet-test.cpp
index 03b385085..a341bd344 100644
--- a/src/zennet-test/zennet-test.cpp
+++ b/src/zennet-test/zennet-test.cpp
@@ -2,6 +2,7 @@
 
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 #include <zennet/zennet.h>
 
 #include <zencore/memory/newdelete.h>
@@ -22,6 +23,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char** argv)
 	zen::IgnoreChildSignals();
 #	endif
 
+	zen::TraceInit("zennet-test");
 	zen::logging::InitializeLogging();
 	zen::MaximizeOpenFileCount();
 
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index b046bbada..78d8ab68a 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -104,6 +104,7 @@ main(int argc, char** argv)
 	IgnoreChildSignals();
 #	endif
 
+	zen::TraceInit("zenserver-test");
 	zen::logging::InitializeLogging();
 
 	zen::logging::SetLogLevel(zen::logging::level::Debug);
@@ -3021,9 +3022,9 @@ TEST_CASE("project.remote")
 		{
 			cpr::Response StatusResponse = Session.Get();
 			CHECK(IsHttpSuccessCode(StatusResponse.status_code));
-			CbObject ReponseObject =
+			CbObject ResponseObject =
 				LoadCompactBinaryObject(IoBuffer(IoBuffer::Wrap, StatusResponse.text.data(), StatusResponse.text.size()));
-			std::string_view Status = ReponseObject["Status"sv].AsString();
+			std::string_view Status = ResponseObject["Status"sv].AsString();
 			CHECK(Status != "Aborted"sv);
 			if (Status == "Complete"sv)
 			{
@@ -3310,18 +3311,18 @@ GenerateFolderContent(const std::filesystem::path& RootPath)
 	std::filesystem::path EmptyFolder(RootPath / "empty_folder");
 
 	std::filesystem::path FirstFolder(RootPath / "first_folder");
-	std::filesystem::create_directory(FirstFolder);
+	CreateDirectories(FirstFolder);
 	Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22)));
 	Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122)));
 
 	std::filesystem::path SecondFolder(RootPath / "second_folder");
-	std::filesystem::create_directory(SecondFolder);
+	CreateDirectories(SecondFolder);
 	Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522)));
 	Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122)));
 	Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225)));
 
 	std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second");
-	std::filesystem::create_directory(SecondFolderChild);
+	CreateDirectories(SecondFolderChild);
 	Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622)));
 
 	for (const auto& It : Result)
@@ -3477,7 +3478,7 @@ TEST_CASE("workspaces.create")
 	while (true)
 	{
 		std::error_code Ec;
-		std::filesystem::remove_all(Root2Path / Share2Path, Ec);
+		DeleteDirectories(Root2Path / Share2Path, Ec);
 		if (!Ec)
 			break;
 	}
@@ -3634,7 +3635,7 @@ TEST_CASE("workspaces.lifetimes")
 	}
 
 	// Wipe system config
-	std::filesystem::remove_all(SystemRootPath);
+	DeleteDirectories(SystemRootPath);
 
 	// Restart
 
@@ -3700,8 +3701,8 @@ TEST_CASE("workspaces.share")
 			uint64_t			  Size		  = FileObject["size"sv].AsUInt64();
 			std::u8string_view	  Path		  = FileObject["clientpath"sv].AsU8String();
 			std::filesystem::path AbsFilePath = SharePath / Path;
-			CHECK(std::filesystem::is_regular_file(AbsFilePath));
-			CHECK(std::filesystem::file_size(AbsFilePath) == Size);
+			CHECK(IsFile(AbsFilePath));
+			CHECK(FileSizeFromPath(AbsFilePath) == Size);
 			Files.insert_or_assign(ChunkId, std::make_pair(AbsFilePath, Size));
 		}
 	}
@@ -3724,7 +3725,7 @@ TEST_CASE("workspaces.share")
 				CHECK(ChunkId != Oid::Zero);
 				std::u8string_view	  Path		  = FileObject["clientpath"sv].AsU8String();
 				std::filesystem::path AbsFilePath = SharePath / Path;
-				CHECK(std::filesystem::is_regular_file(AbsFilePath));
+				CHECK(IsFile(AbsFilePath));
 			}
 		}
 	}
@@ -3744,7 +3745,7 @@ TEST_CASE("workspaces.share")
 			CHECK(ChunkId != Oid::Zero);
 			std::u8string_view	  Path		  = FileObject["clientpath"sv].AsU8String();
 			std::filesystem::path AbsFilePath = SharePath / Path;
-			CHECK(std::filesystem::is_regular_file(AbsFilePath));
+			CHECK(IsFile(AbsFilePath));
 		}
 	}
 
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 2888f5450..73166e608 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -20,6 +20,7 @@
 #include <zenstore/cidstore.h>
 #include <zenstore/gc.h>
 
+#include <zenstore/buildstore/buildstore.h>
 #include <zenstore/cache/structuredcachestore.h>
 #include <zenutil/workerpools.h>
 #include "config.h"
@@ -39,7 +40,7 @@ struct DirStats
 DirStats
 GetStatsForDirectory(std::filesystem::path Dir)
 {
-	if (!std::filesystem::exists(Dir))
+	if (!IsDir(Dir))
 		return {};
 
 	struct StatsTraversal : public GetDirectoryContentVisitor
@@ -105,6 +106,7 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 								   ZenCacheStore*		   CacheStore,
 								   CidStore*			   CidStore,
 								   ProjectStore*		   ProjectStore,
+								   BuildStore*			   BuildStore,
 								   const LogPaths&		   LogPaths,
 								   const ZenServerOptions& ServerOptions)
 : m_GcScheduler(Scheduler)
@@ -112,6 +114,7 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 , m_CacheStore(CacheStore)
 , m_CidStore(CidStore)
 , m_ProjectStore(ProjectStore)
+, m_BuildStore(BuildStore)
 , m_LogPaths(LogPaths)
 , m_ServerOptions(ServerOptions)
 {
@@ -306,6 +309,7 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 				Response << "Interval" << ToTimeSpan(State.Config.Interval);
 				Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration);
 				Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration);
+				Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration);
 				Response << "CollectSmallObjects" << State.Config.CollectSmallObjects;
 				Response << "Enabled" << State.Config.Enabled;
 				Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize);
@@ -401,6 +405,14 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 				}
 			}
 
+			if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false)
+			{
+				if (auto Value = ParseInt<uint64_t>(Param))
+				{
+					GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value());
+				}
+			}
+
 			if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false)
 			{
 				if (auto Value = ParseInt<uint64_t>(Param))
@@ -782,6 +794,10 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 			{
 				m_ProjectStore->Flush();
 			}
+			if (m_BuildStore)
+			{
+				m_BuildStore->Flush();
+			}
 			HttpReq.WriteResponse(HttpResponseCode::OK);
 		},
 		HttpVerb::kPost);
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 563c4f536..e7821dead 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -12,6 +12,7 @@ class JobQueue;
 class ZenCacheStore;
 class CidStore;
 class ProjectStore;
+class BuildStore;
 struct ZenServerOptions;
 
 class HttpAdminService : public zen::HttpService
@@ -28,6 +29,7 @@ public:
 					 ZenCacheStore*			 CacheStore,
 					 CidStore*				 CidStore,
 					 ProjectStore*			 ProjectStore,
+					 BuildStore*			 BuildStore,
 					 const LogPaths&		 LogPaths,
 					 const ZenServerOptions& ServerOptions);
 	~HttpAdminService();
@@ -42,6 +44,7 @@ private:
 	ZenCacheStore*			m_CacheStore;
 	CidStore*				m_CidStore;
 	ProjectStore*			m_ProjectStore;
+	BuildStore*				m_BuildStore;
 	LogPaths				m_LogPaths;
 	const ZenServerOptions& m_ServerOptions;
 };
diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp
new file mode 100644
index 000000000..bcec74ce6
--- /dev/null
+++ b/src/zenserver/buildstore/httpbuildstore.cpp
@@ -0,0 +1,573 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httpbuildstore.h"
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/compactbinaryvalue.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/trace.h>
+#include <zenhttp/packageformat.h>
+#include <zenstore/buildstore/buildstore.h>
+#include <zenutil/workerpools.h>
+
+#include <numeric>
+
+namespace zen {
+using namespace std::literals;
+
+ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv);
+
+HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store)
+: m_Log(logging::Get("builds"))
+, m_StatusService(StatusService)
+, m_StatsService(StatsService)
+, m_BuildStore(Store)
+{
+	Initialize();
+
+	m_StatusService.RegisterHandler("builds", *this);
+	m_StatsService.RegisterHandler("builds", *this);
+}
+
+HttpBuildStoreService::~HttpBuildStoreService()
+{
+	m_StatsService.UnregisterHandler("builds", *this);
+	m_StatusService.UnregisterHandler("builds", *this);
+}
+
+const char*
+HttpBuildStoreService::BaseUri() const
+{
+	return "/builds/";
+}
+
+void
+HttpBuildStoreService::Initialize()
+{
+	ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service");
+
+	m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)");
+	m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)");
+	m_Router.AddPattern("buildid", "([[:xdigit:]]{24})");
+	m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
+
+	m_Router.RegisterRoute(
+		"{namespace}/{bucket}/{buildid}/blobs/{hash}",
+		[this](HttpRouterRequest& Req) { PutBlobRequest(Req); },
+		HttpVerb::kPut);
+
+	m_Router.RegisterRoute(
+		"{namespace}/{bucket}/{buildid}/blobs/{hash}",
+		[this](HttpRouterRequest& Req) { GetBlobRequest(Req); },
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata",
+		[this](HttpRouterRequest& Req) { PutMetadataRequest(Req); },
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata",
+		[this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); },
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"{namespace}/{bucket}/{buildid}/blobs/exists",
+		[this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); },
+		HttpVerb::kPost);
+}
+
+void
+HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest");
+	metrics::OperationTiming::Scope $(m_HttpRequests);
+
+	m_BuildStoreStats.RequestCount++;
+	if (m_Router.HandleRequest(Request) == false)
+	{
+		ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri());
+		return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
+	}
+}
+
+void
+HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest");
+	HttpServerRequest&	   ServerRequest = Req.ServerRequest();
+	const std::string_view Namespace	 = Req.GetCapture(1);
+	const std::string_view Bucket		 = Req.GetCapture(2);
+	const std::string_view BuildId		 = Req.GetCapture(3);
+	const std::string_view Hash			 = Req.GetCapture(4);
+	ZEN_UNUSED(Namespace, Bucket, BuildId);
+	IoHash BlobHash;
+	if (!IoHash::TryParse(Hash, BlobHash))
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   fmt::format("Invalid blob hash '{}'", Hash));
+	}
+	m_BuildStoreStats.BlobWriteCount++;
+	IoBuffer Payload = ServerRequest.ReadPayload();
+	if (!Payload)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   fmt::format("Payload blob {} is empty", Hash));
+	}
+	if (Payload.GetContentType() != HttpContentType::kCompressedBinary)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(
+			HttpResponseCode::BadRequest,
+			HttpContentType::kText,
+			fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType())));
+	}
+	m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload());
+	//	ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize());
+	return ServerRequest.WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest");
+	HttpServerRequest& ServerRequest = Req.ServerRequest();
+	std::string_view   Namespace	 = Req.GetCapture(1);
+	std::string_view   Bucket		 = Req.GetCapture(2);
+	std::string_view   BuildId		 = Req.GetCapture(3);
+	std::string_view   Hash			 = Req.GetCapture(4);
+	ZEN_UNUSED(Namespace, Bucket, BuildId);
+	IoHash BlobHash;
+	if (!IoHash::TryParse(Hash, BlobHash))
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   fmt::format("Invalid blob hash '{}'", Hash));
+	}
+	zen::HttpRanges Ranges;
+	bool			HasRange = ServerRequest.TryGetRanges(Ranges);
+	if (Ranges.size() > 1)
+	{
+		// Only a single range is supported
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   "Multiple ranges in blob request is not supported");
+	}
+
+	m_BuildStoreStats.BlobReadCount++;
+	IoBuffer Blob = m_BuildStore.GetBlob(BlobHash);
+	if (!Blob)
+	{
+		return ServerRequest.WriteResponse(HttpResponseCode::NotFound,
+										   HttpContentType::kText,
+										   fmt::format("Blob with hash '{}' could not be found", Hash));
+	}
+	//	ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize());
+	m_BuildStoreStats.BlobHitCount++;
+	if (HasRange)
+	{
+		const HttpRange& Range		 = Ranges.front();
+		const uint64_t	 BlobSize	 = Blob.GetSize();
+		const uint64_t	 MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0;
+		const uint64_t	 RangeSize	 = Min(Range.End - Range.Start + 1, MaxBlobSize);
+		if (Range.Start + RangeSize > BlobSize)
+		{
+			return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+		}
+		Blob = IoBuffer(Blob, Range.Start, RangeSize);
+		return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
+	}
+	else
+	{
+		return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+	}
+}
+
+void
+HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest");
+	HttpServerRequest& ServerRequest = Req.ServerRequest();
+	std::string_view   Namespace	 = Req.GetCapture(1);
+	std::string_view   Bucket		 = Req.GetCapture(2);
+	std::string_view   BuildId		 = Req.GetCapture(3);
+
+	IoBuffer MetaPayload = ServerRequest.ReadPayload();
+	if (MetaPayload.GetContentType() != ZenContentType::kCbPackage)
+	{
+		throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'",
+											 ToString(MetaPayload.GetContentType()),
+											 ToString(ZenContentType::kCbPackage)));
+	}
+	CbPackage Message = ParsePackageMessage(MetaPayload);
+
+	CbObjectView MessageObject = Message.GetObject();
+	if (!MessageObject)
+	{
+		throw std::runtime_error("PutMetadataRequest payload object is missing");
+	}
+	CbArrayView BlobsArray	  = MessageObject["blobHashes"sv].AsArrayView();
+	CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView();
+
+	const uint64_t BlobCount = BlobsArray.Num();
+	if (BlobCount == 0)
+	{
+		throw std::runtime_error("PutMetadataRequest blobs array is empty");
+	}
+	if (BlobCount != MetadataArray.Num())
+	{
+		throw std::runtime_error(
+			fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount));
+	}
+
+	std::vector<IoHash>	  BlobHashes;
+	std::vector<IoBuffer> MetadataPayloads;
+
+	BlobHashes.reserve(BlobCount);
+	MetadataPayloads.reserve(BlobCount);
+
+	auto BlobsArrayIt	 = begin(BlobsArray);
+	auto MetadataArrayIt = begin(MetadataArray);
+	while (BlobsArrayIt != end(BlobsArray))
+	{
+		const IoHash BlobHash	  = (*BlobsArrayIt).AsHash();
+		const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment();
+
+		const CbAttachment* Attachment = Message.FindAttachment(MetadataHash);
+		if (Attachment == nullptr)
+		{
+			throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash));
+		}
+		BlobHashes.push_back(BlobHash);
+		if (Attachment->IsObject())
+		{
+			MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer());
+			MetadataPayloads.back().SetContentType(ZenContentType::kCbObject);
+		}
+		else if (Attachment->IsCompressedBinary())
+		{
+			MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer());
+			MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary);
+		}
+		else
+		{
+			ZEN_ASSERT(Attachment->IsBinary());
+			MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer());
+			MetadataPayloads.back().SetContentType(ZenContentType::kBinary);
+		}
+
+		BlobsArrayIt++;
+		MetadataArrayIt++;
+	}
+	m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads);
+	return ServerRequest.WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest");
+	HttpServerRequest& ServerRequest = Req.ServerRequest();
+	std::string_view   Namespace	 = Req.GetCapture(1);
+	std::string_view   Bucket		 = Req.GetCapture(2);
+	std::string_view   BuildId		 = Req.GetCapture(3);
+	ZEN_UNUSED(Namespace, Bucket, BuildId);
+	IoBuffer RequestPayload = ServerRequest.ReadPayload();
+	if (!RequestPayload)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   "Expected compact binary body for metadata request, body is missing");
+	}
+	if (RequestPayload.GetContentType() != HttpContentType::kCbObject)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(
+			HttpResponseCode::BadRequest,
+			HttpContentType::kText,
+			fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType())));
+	}
+	if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default);
+		ValidateError != CbValidateError::None)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(
+			HttpResponseCode::BadRequest,
+			HttpContentType::kText,
+			fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError)));
+	}
+	CbObject	RequestObject = LoadCompactBinaryObject(RequestPayload);
+	CbArrayView BlobsArray	  = RequestObject["blobHashes"sv].AsArrayView();
+	if (!BlobsArray)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   "Compact binary body for metadata request is missing 'blobHashes' array");
+	}
+	const uint64_t BlobCount = BlobsArray.Num();
+
+	std::vector<IoHash> BlobRawHashes;
+	BlobRawHashes.reserve(BlobCount);
+	for (CbFieldView BlockHashView : BlobsArray)
+	{
+		BlobRawHashes.push_back(BlockHashView.AsHash());
+		if (BlobRawHashes.back() == IoHash::Zero)
+		{
+			const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType();
+			return ServerRequest.WriteResponse(
+				HttpResponseCode::BadRequest,
+				HttpContentType::kText,
+				fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type));
+		}
+	}
+	m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size();
+	std::vector<IoBuffer> BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst));
+
+	CbPackage							   ResponsePackage;
+	std::vector<CbAttachment>			   Attachments;
+	tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes;
+	Attachments.reserve(BlobCount);
+	AttachmentHashes.reserve(BlobCount);
+	{
+		CbObjectWriter ResponseWriter;
+
+		ResponseWriter.BeginArray("blobHashes");
+		for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+		{
+			if (BlockMetadatas[BlockHashIndex])
+			{
+				const IoHash& BlockHash = BlobRawHashes[BlockHashIndex];
+				ResponseWriter.AddHash(BlockHash);
+			}
+		}
+		ResponseWriter.EndArray();	// blobHashes
+
+		ResponseWriter.BeginArray("metadatas");
+
+		for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+		{
+			if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata)
+			{
+				switch (Metadata.GetContentType())
+				{
+					case ZenContentType::kCbObject:
+						{
+							CbObject	 Object		= CbObject(SharedBuffer(std::move(Metadata)).MakeOwned());
+							const IoHash ObjectHash = Object.GetHash();
+							ResponseWriter.AddBinaryAttachment(ObjectHash);
+							if (!AttachmentHashes.contains(ObjectHash))
+							{
+								Attachments.push_back(CbAttachment(Object, ObjectHash));
+								AttachmentHashes.insert(ObjectHash);
+							}
+						}
+						break;
+					case ZenContentType::kCompressedBinary:
+						{
+							IoHash			 RawHash;
+							uint64_t		 _;
+							CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _);
+							ResponseWriter.AddBinaryAttachment(RawHash);
+							if (!AttachmentHashes.contains(RawHash))
+							{
+								Attachments.push_back(CbAttachment(Compressed, RawHash));
+								AttachmentHashes.insert(RawHash);
+							}
+						}
+						break;
+					default:
+						{
+							const IoHash RawHash = IoHash::HashBuffer(Metadata);
+							ResponseWriter.AddBinaryAttachment(RawHash);
+							if (!AttachmentHashes.contains(RawHash))
+							{
+								Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash));
+								AttachmentHashes.insert(RawHash);
+							}
+						}
+						break;
+				}
+			}
+		}
+
+		ResponseWriter.EndArray();	// metadatas
+
+		ResponsePackage.SetObject(ResponseWriter.Save());
+	}
+	ResponsePackage.AddAttachments(Attachments);
+
+	CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
+	ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
+}
+
+void
+HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest");
+	HttpServerRequest& ServerRequest = Req.ServerRequest();
+	std::string_view   Namespace	 = Req.GetCapture(1);
+	std::string_view   Bucket		 = Req.GetCapture(2);
+	std::string_view   BuildId		 = Req.GetCapture(3);
+	ZEN_UNUSED(Namespace, Bucket, BuildId);
+	IoBuffer RequestPayload = ServerRequest.ReadPayload();
+	if (!RequestPayload)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   "Expected compact binary body for blob exists request, body is missing");
+	}
+	if (RequestPayload.GetContentType() != HttpContentType::kCbObject)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(
+			HttpResponseCode::BadRequest,
+			HttpContentType::kText,
+			fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType())));
+	}
+	if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default);
+		ValidateError != CbValidateError::None)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(
+			HttpResponseCode::BadRequest,
+			HttpContentType::kText,
+			fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError)));
+	}
+	CbObject	RequestObject = LoadCompactBinaryObject(RequestPayload);
+	CbArrayView BlobsArray	  = RequestObject["blobHashes"sv].AsArrayView();
+	if (!BlobsArray)
+	{
+		m_BuildStoreStats.BadRequestCount++;
+		return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+										   HttpContentType::kText,
+										   "Compact binary body for blob exists request is missing 'blobHashes' array");
+	}
+
+	std::vector<IoHash> BlobRawHashes;
+	BlobRawHashes.reserve(BlobsArray.Num());
+	for (CbFieldView BlockHashView : BlobsArray)
+	{
+		BlobRawHashes.push_back(BlockHashView.AsHash());
+		if (BlobRawHashes.back() == IoHash::Zero)
+		{
+			const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType();
+			return ServerRequest.WriteResponse(
+				HttpResponseCode::BadRequest,
+				HttpContentType::kText,
+				fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type));
+		}
+	}
+
+	m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size();
+	std::vector<BuildStore::BlobExistsResult> BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes);
+	CbObjectWriter							  ResponseWriter(9 * BlobsExists.size());
+	ResponseWriter.BeginArray("blobExists"sv);
+	for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists)
+	{
+		ResponseWriter.AddBool(BlobExists.HasBody);
+		if (BlobExists.HasBody)
+		{
+			m_BuildStoreStats.BlobExistsBodyHitCount++;
+		}
+	}
+	ResponseWriter.EndArray();	// blobExist
+	ResponseWriter.BeginArray("metadataExists"sv);
+	for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists)
+	{
+		ResponseWriter.AddBool(BlobExists.HasBody);
+		if (BlobExists.HasMetadata)
+		{
+			m_BuildStoreStats.BlobExistsMetaHitCount++;
+		}
+	}
+	ResponseWriter.EndArray();	// metadataExists
+	CbObject ResponseObject = ResponseWriter.Save();
+	return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject);
+}
+
+void
+HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::Stats");
+	CbObjectWriter Cbo;
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+	Cbo.BeginObject("builds");
+	{
+		Cbo.BeginObject("blobs");
+		{
+			Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount"
+				<< m_BuildStoreStats.BlobHitCount;
+		}
+		Cbo.EndObject();
+
+		Cbo.BeginObject("metadata");
+		{
+			Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount"
+				<< m_BuildStoreStats.BlobMetaHitCount;
+		}
+		Cbo.EndObject();
+
+		Cbo << "requestcount" << m_BuildStoreStats.RequestCount;
+		Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount;
+	}
+	Cbo.EndObject();
+
+	Cbo.BeginObject("size");
+	{
+		BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats();
+
+		Cbo << "count" << StorageStats.EntryCount;
+		Cbo << "bytes" << StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes + StorageStats.MetadataByteCount;
+		Cbo.BeginObject("blobs");
+		{
+			Cbo << "count" << (StorageStats.LargeBlobCount + StorageStats.SmallBlobCount);
+			Cbo << "bytes" << (StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes);
+			Cbo.BeginObject("large");
+			{
+				Cbo << "count" << StorageStats.LargeBlobCount;
+				Cbo << "bytes" << StorageStats.LargeBlobBytes;
+			}
+			Cbo.EndObject();  // large
+			Cbo.BeginObject("small");
+			{
+				Cbo << "count" << StorageStats.SmallBlobCount;
+				Cbo << "bytes" << StorageStats.SmallBlobBytes;
+			}
+			Cbo.EndObject();  // small
+		}
+		Cbo.EndObject();  // blobs
+
+		Cbo.BeginObject("metadata");
+		{
+			Cbo << "count" << StorageStats.MetadataCount;
+			Cbo << "bytes" << StorageStats.MetadataByteCount;
+		}
+		Cbo.EndObject();  // metadata
+	}
+	Cbo.EndObject();  // size
+
+	return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
+HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+}  // namespace zen
diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h
new file mode 100644
index 000000000..50cb5db12
--- /dev/null
+++ b/src/zenserver/buildstore/httpbuildstore.h
@@ -0,0 +1,68 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/stats.h>
+#include <zenhttp/httpserver.h>
+#include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
+
+#include <filesystem>
+
+namespace zen {
+
+class BuildStore;
+
+class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider
+{
+public:
+	HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store);
+	virtual ~HttpBuildStoreService();
+
+	virtual const char* BaseUri() const override;
+	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+
+	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+
+private:
+	struct BuildStoreStats
+	{
+		std::atomic_uint64_t BlobReadCount{};
+		std::atomic_uint64_t BlobHitCount{};
+		std::atomic_uint64_t BlobWriteCount{};
+		std::atomic_uint64_t BlobMetaReadCount{};
+		std::atomic_uint64_t BlobMetaHitCount{};
+		std::atomic_uint64_t BlobMetaWriteCount{};
+		std::atomic_uint64_t BlobExistsCount{};
+		std::atomic_uint64_t BlobExistsBodyHitCount{};
+		std::atomic_uint64_t BlobExistsMetaHitCount{};
+		std::atomic_uint64_t RequestCount{};
+		std::atomic_uint64_t BadRequestCount{};
+	};
+
+	void Initialize();
+
+	inline LoggerRef Log() { return m_Log; }
+
+	LoggerRef m_Log;
+
+	void PutBlobRequest(HttpRouterRequest& Req);
+	void GetBlobRequest(HttpRouterRequest& Req);
+
+	void PutMetadataRequest(HttpRouterRequest& Req);
+	void GetMetadatasRequest(HttpRouterRequest& Req);
+
+	void BlobsExistsRequest(HttpRouterRequest& Req);
+
+	HttpRequestRouter m_Router;
+
+	HttpStatusService& m_StatusService;
+	HttpStatsService&  m_StatsService;
+
+	BuildStore&				 m_BuildStore;
+	BuildStoreStats			 m_BuildStoreStats;
+	metrics::OperationTiming m_HttpRequests;
+};
+
+}  // namespace zen
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index b9a9ca380..bb0c55618 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -25,6 +25,7 @@
 #include <zenutil/cache/cacherequests.h>
 #include <zenutil/cache/rpcrecording.h>
 #include <zenutil/jupiter/jupiterclient.h>
+#include <zenutil/parallelwork.h>
 #include <zenutil/workerpools.h>
 
 #include "upstream/upstreamcache.h"
@@ -1585,15 +1586,26 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext&  Co
 	WorkerThreadPool WorkerPool(ThreadCount);
 	uint64_t		 RequestCount = Replayer.GetRequestCount();
 	Stopwatch		 Timer;
-	auto  _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); });
-	Latch JobLatch(RequestCount);
+	auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); });
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
 	ZEN_INFO("Replaying {} requests", RequestCount);
 	for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex)
 	{
-		WorkerPool.ScheduleWork([this, &Context, &JobLatch, &Replayer, RequestIndex]() {
+		if (AbortFlag)
+		{
+			break;
+		}
+		Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic<bool>& AbortFlag) {
 			IoBuffer						Body;
 			zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body);
 
+			if (AbortFlag)
+			{
+				return;
+			}
+
 			if (Body)
 			{
 				uint32_t		 AcceptMagic = 0;
@@ -1634,16 +1646,15 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext&  Co
 					}
 				}
 			}
-			JobLatch.CountDown();
 		});
 	}
-	while (!JobLatch.Wait(10000))
-	{
+	Work.Wait(10000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+		ZEN_UNUSED(IsAborted, IsPaused);
 		ZEN_INFO("Replayed {} of {} requests, elapsed {}",
-				 RequestCount - JobLatch.Remaining(),
+				 RequestCount - PendingWork,
 				 RequestCount,
 				 NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
-	}
+	});
 }
 
 void
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 809092378..1f9ae5fb6 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -15,12 +15,15 @@
 #include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenhttp/zenhttp.h>
+#include <zenutil/commandlineoptions.h>
+#include <zenutil/environmentoptions.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
 #include <fmt/ranges.h>
 #include <zencore/logging.h>
 #include <cxxopts.hpp>
+#include <json11.hpp>
 #include <sol/sol.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
@@ -178,27 +181,6 @@ ParseBucketConfigs(std::span<std::string> Buckets)
 	return Cfg;
 }
 
-static std::string
-MakeSafePath(const std::string_view Path)
-{
-#if ZEN_PLATFORM_WINDOWS
-	if (Path.empty())
-	{
-		return std::string(Path);
-	}
-
-	std::string FixedPath(Path);
-	std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\');
-	if (!FixedPath.starts_with("\\\\?\\"))
-	{
-		FixedPath.insert(0, "\\\\?\\");
-	}
-	return FixedPath;
-#else
-	return std::string(Path);
-#endif
-};
-
 class CachePolicyOption : public LuaConfig::OptionValue
 {
 public:
@@ -324,7 +306,7 @@ public:
 					std::string Name	  = Bucket.value().get_or("name", std::string("Default"));
 					std::string Directory = Bucket.value().get_or("directory", std::string());
 
-					Value.Buckets.push_back({.Name = std::move(Name), .Directory = LuaConfig::MakeSafePath(Directory)});
+					Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)});
 				}
 			}
 		}
@@ -332,6 +314,97 @@ public:
 	ZenObjectStoreConfig& Value;
 };
 
+class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue
+{
+public:
+	ZenStructuredCacheBucketsConfigOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) : Value(Value) {}
+	virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override
+	{
+		if (Value.empty())
+		{
+			StringBuilder.Append("{}");
+			return;
+		}
+		LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent);
+		for (const std::pair<std::string, ZenStructuredCacheBucketConfig>& Bucket : Value)
+		{
+			Writer.BeginContainer("");
+			{
+				Writer.WriteValue("name", Bucket.first);
+				const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second;
+
+				Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize));
+				Writer.BeginContainer("memlayer");
+				{
+					Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold));
+				}
+				Writer.EndContainer();
+
+				Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment));
+				Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment));
+			}
+			Writer.EndContainer();
+		}
+	}
+	virtual void Parse(sol::object Object) override
+	{
+		if (sol::optional<sol::table> Buckets = Object.as<sol::table>())
+		{
+			for (const auto& Kv : Buckets.value())
+			{
+				if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>())
+				{
+					ZenStructuredCacheBucketConfig BucketConfig;
+					std::string					   Name = Kv.first.as<std::string>();
+					if (Name.empty())
+					{
+						throw zen::OptionParseException(fmt::format("cache bucket option must have a name."));
+					}
+
+					const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize);
+					if (MaxBlockSize == 0)
+					{
+						throw zen::OptionParseException(
+							fmt::format("maxblocksize option for cache bucket '{}' is invalid. It must be non-zero.", Name));
+					}
+					BucketConfig.MaxBlockSize = MaxBlockSize;
+
+					if (sol::optional<sol::table> Memlayer = Bucket.value().get_or("memlayer", sol::table()))
+					{
+						const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold);
+						if (MemCacheSizeThreshold == 0)
+						{
+							throw zen::OptionParseException(
+								fmt::format("memlayer.sizethreshold option for cache bucket '{}' is invalid. It must be non-zero.", Name));
+						}
+						BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold);
+					}
+
+					const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment);
+					if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment))
+					{
+						throw zen::OptionParseException(fmt::format(
+							"payloadalignment option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.",
+							Name));
+					}
+					BucketConfig.PayloadAlignment = PayloadAlignment;
+
+					const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold);
+					if (LargeObjectThreshold == 0)
+					{
+						throw zen::OptionParseException(
+							fmt::format("largeobjectthreshold option for cache bucket '{}' is invalid. It must be non-zero.", Name));
+					}
+					BucketConfig.LargeObjectThreshold = LargeObjectThreshold;
+
+					Value.push_back(std::make_pair(std::move(Name), BucketConfig));
+				}
+			}
+		}
+	}
+	std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value;
+};
+
 std::shared_ptr<LuaConfig::OptionValue>
 MakeOption(zen::UpstreamCachePolicy& Value)
 {
@@ -350,6 +423,35 @@ MakeOption(zen::ZenObjectStoreConfig& Value)
 	return std::make_shared<ZenObjectStoreConfigOption>(Value);
 };
 
+std::shared_ptr<LuaConfig::OptionValue>
+MakeOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value)
+{
+	return std::make_shared<ZenStructuredCacheBucketsConfigOption>(Value);
+};
+
+void
+ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult)
+{
+	using namespace std::literals;
+
+	EnvironmentOptions Options;
+	Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+	Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
+	Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
+
+	bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable;
+	Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv);
+
+	Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
+
+	Options.Parse(CmdLineResult);
+
+	if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable)
+	{
+		ServerOptions.SentryConfig.Disable = !EnvEnableSentry;
+	}
+}
+
 void
 ParseConfigFile(const std::filesystem::path& Path,
 				ZenServerOptions&			 ServerOptions,
@@ -363,12 +465,16 @@ ParseConfigFile(const std::filesystem::path& Path,
 	////// server
 	LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv);
 	LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv);
-	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv);
-	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv);
+	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv);
+	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+	LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
+	LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
+	LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
 	LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv);
 	LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
 	LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
 	LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
+	LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv);
 	LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
 	LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
 	LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv);
@@ -377,6 +483,10 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv);
 	LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig);
 
+	////// buildsstore
+	LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv);
+	LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit");
+
 	////// network
 	LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv);
 	LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv);
@@ -410,7 +520,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv);
 
 	LuaOptions.AddOption("cache.memlayer.sizethreshold"sv,
-						 ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold,
+						 ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold,
 						 "cache-memlayer-sizethreshold"sv);
 	LuaOptions.AddOption("cache.memlayer.targetfootprint"sv,
 						 ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes,
@@ -420,6 +530,19 @@ ParseConfigFile(const std::filesystem::path& Path,
 						 "cache-memlayer-triminterval"sv);
 	LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv);
 
+	LuaOptions.AddOption("cache.bucket.maxblocksize"sv,
+						 ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize,
+						 "cache-bucket-maxblocksize"sv);
+	LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv,
+						 ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold,
+						 "cache-bucket-memlayer-sizethreshold"sv);
+	LuaOptions.AddOption("cache.bucket.payloadalignment"sv,
+						 ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment,
+						 "cache-bucket-payloadalignment"sv);
+	LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv,
+						 ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold,
+						 "cache-bucket-largeobjectthreshold"sv);
+
 	////// cache.upstream
 	LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv);
 	LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv,
@@ -492,6 +615,9 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("gc.projectstore.duration.seconds"sv,
 						 ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds,
 						 "gc-projectstore-duration-seconds");
+	LuaOptions.AddOption("gc.buildstore.duration.seconds"sv,
+						 ServerOptions.GcConfig.BuildStore.MaxDurationSeconds,
+						 "gc-buildstore-duration-seconds");
 
 	////// security
 	LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv);
@@ -504,6 +630,8 @@ ParseConfigFile(const std::filesystem::path& Path,
 						 ServerOptions.WorksSpacesConfig.AllowConfigurationChanges,
 						 "workspaces-allow-changes"sv);
 
+	LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv);
+
 	LuaOptions.Parse(Path, CmdLineResult);
 
 	// These have special command line processing so we make sure we export them if they were configured on command line
@@ -515,10 +643,14 @@ ParseConfigFile(const std::filesystem::path& Path,
 	{
 		LuaOptions.Touch("server.objectstore.buckets"sv);
 	}
+	if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty())
+	{
+		LuaOptions.Touch("cache.buckets"sv);
+	}
 
 	if (!OutputConfigFile.empty())
 	{
-		std::filesystem::path			  WritePath(MakeSafePath(OutputConfigFile));
+		std::filesystem::path			  WritePath(MakeSafeAbsolutePath(OutputConfigFile));
 		zen::ExtendableStringBuilder<512> ConfigStringBuilder;
 		LuaOptions.Print(ConfigStringBuilder, CmdLineResult);
 		zen::BasicFile Output;
@@ -528,6 +660,68 @@ ParseConfigFile(const std::filesystem::path& Path,
 }
 
 void
+ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, int BasePort)
+{
+	using namespace std::literals;
+
+	IoBuffer	 Body = IoBufferBuilder::MakeFromFile(Path);
+	std::string	 JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize());
+	std::string	 JsonError;
+	json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError);
+	if (!JsonError.empty())
+	{
+		ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError);
+		return;
+	}
+	for (const json11::Json& PluginInfo : PluginsInfo.array_items())
+	{
+		if (!PluginInfo.is_object())
+		{
+			ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump());
+			continue;
+		}
+
+		HttpServerPluginConfig Config = {};
+
+		bool bNeedsPort = true;
+
+		for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items())
+		{
+			if (!Items.second.is_string())
+			{
+				ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'",
+						 Path,
+						 Items.second.dump());
+				continue;
+			}
+
+			const std::string& Name	 = Items.first;
+			const std::string& Value = Items.second.string_value();
+
+			if (Name == "name"sv)
+				Config.PluginName = Value;
+			else
+			{
+				Config.PluginOptions.push_back({Name, Value});
+
+				if (Name == "port"sv)
+				{
+					bNeedsPort = false;
+				}
+			}
+		}
+
+		// add a default base port in case if json config didn't provide one
+		if (bNeedsPort)
+		{
+			Config.PluginOptions.push_back({"port", std::to_string(BasePort)});
+		}
+
+		ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config);
+	}
+}
+
+void
 ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 {
 	const char* DefaultHttp = "asio";
@@ -559,6 +753,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	std::string ContentDir;
 	std::string AbsLogFile;
 	std::string ConfigFile;
+	std::string PluginsConfigFile;
 	std::string OutputConfigFile;
 	std::string BaseSnapshotDir;
 
@@ -586,13 +781,19 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 						  "Exit immediately after initialization is complete",
 						  cxxopts::value<bool>(ServerOptions.IsPowerCycle));
 	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
+	options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile));
 	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
 	options.add_options()("no-sentry",
 						  "Disable Sentry crash handler",
-						  cxxopts::value<bool>(ServerOptions.NoSentry)->default_value("false"));
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.Disable)->default_value("false"));
 	options.add_options()("sentry-allow-personal-info",
 						  "Allow personally identifiable information in sentry crash reports",
-						  cxxopts::value<bool>(ServerOptions.SentryAllowPII)->default_value("false"));
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.AllowPII)->default_value("false"));
+	options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryConfig.Dsn));
+	options.add_options()("sentry-environment", "Sentry environment", cxxopts::value<std::string>(ServerOptions.SentryConfig.Environment));
+	options.add_options()("sentry-debug",
+						  "Enable debug mode for Sentry",
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.Debug)->default_value("false"));
 	options.add_options()("detach",
 						  "Indicate whether zenserver should detach from parent process group",
 						  cxxopts::value<bool>(ServerOptions.Detach)->default_value("true"));
@@ -863,8 +1064,9 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 		"cache",
 		"",
 		"cache-memlayer-sizethreshold",
-		"The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.",
-		cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold)->default_value("1024"),
+		"The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching. "
+		"Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`",
+		cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"),
 		"");
 
 	options.add_option("cache",
@@ -888,6 +1090,36 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"),
 					   "");
 
+	options.add_option("cache",
+					   "",
+					   "cache-bucket-maxblocksize",
+					   "Max size of cache bucket blocks. Default set to 1073741824 (1GB).",
+					   cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"),
+					   "");
+
+	options.add_option("cache",
+					   "",
+					   "cache-bucket-payloadalignment",
+					   "Payload alignement for cache bucket blocks. Default set to 16.",
+					   cxxopts::value<uint32_t>(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"),
+					   "");
+
+	options.add_option(
+		"cache",
+		"",
+		"cache-bucket-largeobjectthreshold",
+		"Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).",
+		cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"),
+		"");
+
+	options.add_option(
+		"cache",
+		"",
+		"cache-bucket-memlayer-sizethreshold",
+		"The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.",
+		cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"),
+		"");
+
 	options.add_option("gc",
 					   "",
 					   "gc-cache-attachment-store",
@@ -960,6 +1192,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 	options.add_option("gc",
 					   "",
+					   "gc-buildstore-duration-seconds",
+					   "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)",
+					   cxxopts::value<int32_t>(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"),
+					   "");
+
+	options.add_option("gc",
+					   "",
 					   "disk-reserve-size",
 					   "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.",
 					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"),
@@ -1031,6 +1270,19 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   cxxopts::value<std::vector<std::string>>(BucketConfigs),
 					   "");
 
+	options.add_option("buildstore",
+					   "",
+					   "buildstore-enabled",
+					   "Whether the builds store is enabled or not.",
+					   cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"),
+					   "");
+	options.add_option("buildstore",
+					   "",
+					   "buildstore-disksizelimit",
+					   "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)",
+					   cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"),
+					   "");
+
 	options.add_option("stats",
 					   "",
 					   "statsd",
@@ -1083,12 +1335,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 		}
 		logging::RefreshLogLevels();
 
-		ServerOptions.SystemRootDir					  = MakeSafePath(SystemRootDir);
-		ServerOptions.DataDir						  = MakeSafePath(DataDir);
-		ServerOptions.BaseSnapshotDir				  = MakeSafePath(BaseSnapshotDir);
-		ServerOptions.ContentDir					  = MakeSafePath(ContentDir);
-		ServerOptions.AbsLogFile					  = MakeSafePath(AbsLogFile);
-		ServerOptions.ConfigFile					  = MakeSafePath(ConfigFile);
+		ServerOptions.SystemRootDir					  = MakeSafeAbsolutePath(SystemRootDir);
+		ServerOptions.DataDir						  = MakeSafeAbsolutePath(DataDir);
+		ServerOptions.BaseSnapshotDir				  = MakeSafeAbsolutePath(BaseSnapshotDir);
+		ServerOptions.ContentDir					  = MakeSafeAbsolutePath(ContentDir);
+		ServerOptions.AbsLogFile					  = MakeSafeAbsolutePath(AbsLogFile);
+		ServerOptions.ConfigFile					  = MakeSafeAbsolutePath(ConfigFile);
+		ServerOptions.PluginsConfigFile				  = MakeSafeAbsolutePath(PluginsConfigFile);
 		ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
 
 		if (!BaseSnapshotDir.empty())
@@ -1096,7 +1349,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 			if (DataDir.empty())
 				throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot");
 
-			if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir))
+			if (!IsDir(ServerOptions.BaseSnapshotDir))
 				throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir));
 		}
 
@@ -1113,6 +1366,8 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 		ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs);
 
+		ParseEnvVariables(ServerOptions, Result);
+
 		if (!ServerOptions.ConfigFile.empty())
 		{
 			ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile);
@@ -1122,6 +1377,11 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 			ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile);
 		}
 
+		if (!ServerOptions.PluginsConfigFile.empty())
+		{
+			ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort);
+		}
+
 		ValidateOptions(ServerOptions);
 	}
 	catch (const zen::OptionParseException& e)
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index c7781aada..9753e3ae2 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -59,11 +59,17 @@ struct ZenProjectStoreEvictionPolicy
 	int32_t MaxDurationSeconds = 7 * 24 * 60 * 60;
 };
 
+struct ZenBuildStoreEvictionPolicy
+{
+	int32_t MaxDurationSeconds = 3 * 24 * 60 * 60;
+};
+
 struct ZenGcConfig
 {
 	//	ZenCasEvictionPolicy   Cas;
 	ZenCacheEvictionPolicy		  Cache;
 	ZenProjectStoreEvictionPolicy ProjectStore;
+	ZenBuildStoreEvictionPolicy	  BuildStore;
 	int32_t						  MonitorIntervalSeconds			= 30;
 	int32_t						  IntervalSeconds					= 0;
 	bool						  CollectSmallObjects				= true;
@@ -113,15 +119,24 @@ struct ZenStatsConfig
 	int			StatsdPort = 8125;
 };
 
+struct ZenStructuredCacheBucketConfig
+{
+	uint64_t MaxBlockSize		   = 1ull << 30;
+	uint32_t PayloadAlignment	   = 1u << 4;
+	uint64_t MemCacheSizeThreshold = 1 * 1024;
+	uint64_t LargeObjectThreshold  = 128 * 1024;
+};
+
 struct ZenStructuredCacheConfig
 {
-	bool	 Enabled				 = true;
-	bool	 WriteLogEnabled		 = false;
-	bool	 AccessLogEnabled		 = false;
-	uint64_t MemCacheSizeThreshold	 = 1 * 1024;
-	uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024;
-	uint64_t MemTrimIntervalSeconds	 = 60;
-	uint64_t MemMaxAgeSeconds		 = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count());
+	bool																Enabled			 = true;
+	bool																WriteLogEnabled	 = false;
+	bool																AccessLogEnabled = false;
+	std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>> PerBucketConfigs;
+	ZenStructuredCacheBucketConfig										BucketConfig;
+	uint64_t															MemTargetFootprintBytes = 512 * 1024 * 1024;
+	uint64_t															MemTrimIntervalSeconds	= 60;
+	uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count());
 };
 
 struct ZenProjectStoreConfig
@@ -130,12 +145,27 @@ struct ZenProjectStoreConfig
 	bool StoreProjectAttachmentMetaData = false;
 };
 
+struct ZenBuildStoreConfig
+{
+	bool	 Enabled		   = false;
+	uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u;  // 1TB
+};
+
 struct ZenWorkspacesConfig
 {
 	bool Enabled				   = false;
 	bool AllowConfigurationChanges = false;
 };
 
+struct ZenSentryConfig
+{
+	bool		Disable	 = false;
+	bool		AllowPII = false;  // Allow personally identifiable information in sentry crash reports
+	std::string Dsn;
+	std::string Environment;
+	bool		Debug = false;	// Enable debug mode for Sentry
+};
+
 struct ZenServerOptions
 {
 	ZenUpstreamCacheConfig	 UpstreamCacheConfig;
@@ -145,13 +175,16 @@ struct ZenServerOptions
 	zen::HttpServerConfig	 HttpServerConfig;
 	ZenStructuredCacheConfig StructuredCacheConfig;
 	ZenProjectStoreConfig	 ProjectStoreConfig;
+	ZenBuildStoreConfig		 BuildStoreConfig;
 	ZenStatsConfig			 StatsConfig;
 	ZenWorkspacesConfig		 WorksSpacesConfig;
+	ZenSentryConfig			 SentryConfig;
 	std::filesystem::path	 SystemRootDir;				  // System root directory (used for machine level config)
 	std::filesystem::path	 DataDir;					  // Root directory for state (used for testing)
 	std::filesystem::path	 ContentDir;				  // Root directory for serving frontend content (experimental)
 	std::filesystem::path	 AbsLogFile;				  // Absolute path to main log file
 	std::filesystem::path	 ConfigFile;				  // Path to Lua config file
+	std::filesystem::path	 PluginsConfigFile;			  // Path to plugins config file
 	std::filesystem::path	 BaseSnapshotDir;			  // Path to server state snapshot (will be copied into data dir on start)
 	std::string				 ChildId;					  // Id assigned by parent process (used for lifetime management)
 	std::string				 LogId;						  // Id for tagging log output
@@ -168,9 +201,7 @@ struct ZenServerOptions
 	bool					 IsDedicated		= false;  // Indicates a dedicated/shared instance, with larger resource requirements
 	bool					 ShouldCrash		= false;  // Option for testing crash handling
 	bool					 IsFirstRun			= false;
-	bool					 NoSentry			= false;
-	bool					 SentryAllowPII		= false;  // Allow personally identifiable information in sentry crash reports
-	bool					 Detach				= true;	  // Whether zenserver should detach from existing process group (Mac/Linux)
+	bool					 Detach				= true;	 // Whether zenserver should detach from existing process group (Mac/Linux)
 	bool					 ObjectStoreEnabled = false;
 	bool					 NoConsoleOutput	= false;  // Control default use of stdout for diagnostics
 	std::string				 Loggers[zen::logging::level::LogLevelCount];
diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp
index f742fa34a..2c54de29e 100644
--- a/src/zenserver/config/luaconfig.cpp
+++ b/src/zenserver/config/luaconfig.cpp
@@ -4,27 +4,6 @@
 
 namespace zen::LuaConfig {
 
-std::string
-MakeSafePath(const std::string_view Path)
-{
-#if ZEN_PLATFORM_WINDOWS
-	if (Path.empty())
-	{
-		return std::string(Path);
-	}
-
-	std::string FixedPath(Path);
-	std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\');
-	if (!FixedPath.starts_with("\\\\?\\"))
-	{
-		FixedPath.insert(0, "\\\\?\\");
-	}
-	return FixedPath;
-#else
-	return std::string(Path);
-#endif
-};
-
 void
 EscapeBackslash(std::string& InOutString)
 {
@@ -101,7 +80,7 @@ FilePathOption::Parse(sol::object Object)
 	std::string Str = Object.as<std::string>();
 	if (!Str.empty())
 	{
-		Value = MakeSafePath(Str);
+		Value = MakeSafeAbsolutePath(Str);
 	}
 }
 
diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h
index 76b3088a3..ce7013a9a 100644
--- a/src/zenserver/config/luaconfig.h
+++ b/src/zenserver/config/luaconfig.h
@@ -4,10 +4,10 @@
 
 #include <zenbase/concepts.h>
 #include <zencore/fmtutils.h>
+#include <zenutil/commandlineoptions.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
-#include <cxxopts.hpp>
 #include <sol/sol.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
@@ -20,8 +20,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen::LuaConfig {
 
-std::string MakeSafePath(const std::string_view Path);
-void		EscapeBackslash(std::string& InOutString);
+void EscapeBackslash(std::string& InOutString);
 
 class OptionValue
 {
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 31d9e1c94..dfa710ae0 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -2,6 +2,7 @@
 
 #include "frontend.h"
 
+#include <zencore/compactbinarybuilder.h>
 #include <zencore/endian.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
@@ -26,7 +27,9 @@ static unsigned char gHtmlZipData[] = {
 namespace zen {
 
 ////////////////////////////////////////////////////////////////////////////////
-HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory)
+HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService)
+: m_Directory(Directory)
+, m_StatusService(StatusService)
 {
 	std::filesystem::path SelfPath = GetRunningExecutablePath();
 
@@ -50,7 +53,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
 			{
 				break;
 			}
-			if (std::filesystem::is_regular_file(ParentPath / "xmake.lua", ErrorCode))
+			if (IsFile(ParentPath / "xmake.lua", ErrorCode))
 			{
 				if (ErrorCode)
 				{
@@ -59,7 +62,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
 
 				std::filesystem::path HtmlDir = ParentPath / "src" / "zenserver" / "frontend" / "html";
 
-				if (std::filesystem::is_directory(HtmlDir, ErrorCode))
+				if (IsDir(HtmlDir, ErrorCode))
 				{
 					m_Directory = HtmlDir;
 				}
@@ -81,10 +84,12 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
 	{
 		ZEN_INFO("front-end is NOT AVAILABLE");
 	}
+	m_StatusService.RegisterHandler("dashboard", *this);
 }
 
 HttpFrontendService::~HttpFrontendService()
 {
+	m_StatusService.UnregisterHandler("dashboard", *this);
 }
 
 const char*
@@ -95,6 +100,14 @@ HttpFrontendService::BaseUri() const
 
 ////////////////////////////////////////////////////////////////////////////////
 void
+HttpFrontendService::HandleStatusRequest(zen::HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 {
 	using namespace std::literals;
diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h
index 6eac20620..84ffaac42 100644
--- a/src/zenserver/frontend/frontend.h
+++ b/src/zenserver/frontend/frontend.h
@@ -3,23 +3,26 @@
 #pragma once
 
 #include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
 #include "zipfs.h"
 
 #include <filesystem>
 
 namespace zen {
 
-class HttpFrontendService final : public zen::HttpService
+class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider
 {
 public:
-	HttpFrontendService(std::filesystem::path Directory);
+	HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService);
 	virtual ~HttpFrontendService();
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	ZipFs				  m_ZipFs;
 	std::filesystem::path m_Directory;
+	HttpStatusService&	  m_StatusService;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index e0a1888e6..5778fa3d2 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/frontend/html/indexer/cache.js b/src/zenserver/frontend/html/indexer/cache.js
index 390aa948d..b90194855 100644
--- a/src/zenserver/frontend/html/indexer/cache.js
+++ b/src/zenserver/frontend/html/indexer/cache.js
@@ -9,7 +9,7 @@ export class Cache
 	{
 		this._db_name = db_name;
 		this._store_names = store_names;
-		this._version = 1;
+		this._version = 2;
 		this._db = this._open();
 	}
 
diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js
index 4412e3a57..688bc71b0 100644
--- a/src/zenserver/frontend/html/indexer/indexer.js
+++ b/src/zenserver/frontend/html/indexer/indexer.js
@@ -43,9 +43,10 @@ class Indexer
 
 	*search(needle)
 	{
+		var needleLwr = needle.toLowerCase();
 		for (const page of this._pages)
 			for (const [_, name] of page)
-				if (name.indexOf(needle) >= 0)
+				if (name.toLowerCase().indexOf(needleLwr) >= 0)
 					yield name;
 	}
 
@@ -60,7 +61,7 @@ class Indexer
 	{
 		for (const page of this._pages)
 			for (const [_, name, size, raw_size] of page)
-				yield [name, size|0, raw_size|0];
+				yield [name, size|0n, raw_size|0n];
 	}
 }
 
diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js
index 25c8d7671..69ee234fa 100644
--- a/src/zenserver/frontend/html/indexer/worker.js
+++ b/src/zenserver/frontend/html/indexer/worker.js
@@ -77,11 +77,11 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride)
 				continue;
 
 			var id = 0n;
-			var size = 0;
-			var raw_size = 0;
+			var size = 0n;
+			var raw_size = 0n;
 			for (const item of pkg_data.as_array())
 			{
-				var found = 0, pkg_id;
+				var found = 0, pkg_id = undefined;
 				for (const field of item.as_object())
 				{
 					if (!id && field.is_named("id"))	pkg_id = field.as_value();
diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js
index 65a3ef39b..54fb11c18 100644
--- a/src/zenserver/frontend/html/pages/entry.js
+++ b/src/zenserver/frontend/html/pages/entry.js
@@ -59,6 +59,81 @@ export class Page extends ZenPage
 		}
 	}
 
+	_find_iohash_field(container, name)
+	{
+		const found_field = container.find(name);
+		if (found_field != undefined)
+		{
+			var found_value = found_field.as_value();
+			if (found_value instanceof Uint8Array)
+			{
+				var ret = "";
+				for (var x of found_value)
+					ret += x.toString(16).padStart(2, "0");
+				return ret;
+			}
+		}
+		return null;
+	}
+
+	async _build_meta(section, entry)
+	{
+		var tree = {}
+
+		for (const field of entry)
+		{
+			var visibleKey = undefined;
+			const name = field.get_name();
+			if (name == "CookPackageArtifacts")
+			{
+				visibleKey = name;
+			}
+			else if (name.startsWith("meta."))
+			{
+				visibleKey = name.slice(5);
+			}
+
+			if (visibleKey != undefined)
+			{
+				var found_value = field.as_value();
+				if (found_value instanceof Uint8Array)
+				{
+					var ret = "";
+					for (var x of found_value)
+						ret += x.toString(16).padStart(2, "0");
+					tree[visibleKey] = ret;
+				}
+			}
+
+		}
+
+		if (Object.keys(tree).length == 0)
+			return;
+
+		const sub_section = section.add_section("meta");
+
+		const table = sub_section.add_widget(
+			Table,
+			["name", "actions"], Table.Flag_PackRight
+		);
+		for (const key in tree)
+		{
+			const row = table.add_row(key);
+			const value = tree[key];
+
+			const project = this.get_param("project");
+			const oplog = this.get_param("oplog");
+			const link = row.get_cell(0).link(
+				"/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
+			);
+
+			const action_tb = new Toolbar(row.get_cell(-1), true);
+			action_tb.left().add("copy-hash").on_click(async (v) => {
+				await navigator.clipboard.writeText(v);
+			}, value);
+		}
+	}
+
 	async _build_page()
 	{
 		var entry = await this._entry;
@@ -78,8 +153,16 @@ export class Page extends ZenPage
 
 			delete tree["$id"];
 
-			const sub_section = section.add_section("deps");
-			this._build_deps(sub_section, tree);
+			if (Object.keys(tree).length != 0)
+			{
+				const sub_section = section.add_section("deps");
+				this._build_deps(sub_section, tree);
+			}
+		}
+
+		// meta
+		{
+			this._build_meta(section, entry);
 		}
 
 		// data
@@ -128,7 +211,6 @@ export class Page extends ZenPage
 					);
 					link.first_child().attr("download", `${io_hash}_${base_name}`);
 
-					const do_nothing = () => void(0);
 					const action_tb = new Toolbar(row.get_cell(-1), true);
 					action_tb.left().add("copy-hash").on_click(async (v) => {
 						await navigator.clipboard.writeText(v);
@@ -147,8 +229,11 @@ export class Page extends ZenPage
 
 	_display_unsupported(section, entry)
 	{
+		const replacer = (key, value) =>
+			typeof value === "bigint" ? { $bigint: value.toString() } : value;
+		
 		const object = entry.to_js_object();
-		const text = JSON.stringify(object, null, "  ");
+		const text = JSON.stringify(object, replacer, "  ");
 		section.tag("pre").text(text);
 	}
 
diff --git a/src/zenserver/frontend/html/pages/oplog.js b/src/zenserver/frontend/html/pages/oplog.js
index f22c2a58f..bef5bacce 100644
--- a/src/zenserver/frontend/html/pages/oplog.js
+++ b/src/zenserver/frontend/html/pages/oplog.js
@@ -142,9 +142,12 @@ export class Page extends ZenPage
 
 	async _search(needle)
 	{
-		needle = needle.trim();
-		if (needle.length < 3)
+		if (needle.length == 0)
+		{
+			this._build_table(this._index_start);
 			return;
+		}
+		needle = needle.trim();
 
 		this._entry_table.clear(this._index_start);
 
diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js
index 8c9df62f9..d1c13ccc7 100644
--- a/src/zenserver/frontend/html/pages/start.js
+++ b/src/zenserver/frontend/html/pages/start.js
@@ -5,6 +5,7 @@
 import { ZenPage } from "./page.js"
 import { Fetcher } from "../util/fetcher.js"
 import { Friendly } from "../util/friendly.js"
+import { Modal } from "../util/modal.js"
 import { Table, Toolbar } from "../util/widgets.js"
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -40,6 +41,41 @@ export class Page extends ZenPage
 			action_tb.left().add("drop").on_click((x) => this.drop_project(x), project.Id);
 		}
 
+		// cache
+		var section = this.add_section("z$");
+		columns = [
+			"namespace",
+			"dir",
+			"buckets",
+			"entries",
+			"size disk",
+			"size mem",
+			"actions",
+		]
+		var zcache_info = new Fetcher().resource("/z$/").json();
+		const cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight);
+		for (const namespace of (await zcache_info)["Namespaces"])
+		{
+			new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => {
+				const row = cache_table.add_row(
+					"",
+					data["Configuration"]["RootDir"],
+					data["Buckets"].length,
+					data["EntryCount"],
+					Friendly.kib(data["StorageSize"].DiskSize),
+					Friendly.kib(data["StorageSize"].MemorySize)
+				);
+				var cell = row.get_cell(0);
+				cell.tag().text(namespace).on_click(() => this.view_zcache(namespace));
+				row.get_cell(1).tag().text(namespace);
+
+				cell = row.get_cell(-1);
+				const action_tb = new Toolbar(cell, true);
+				action_tb.left().add("view").on_click(() => this.view_zcache(namespace));
+				action_tb.left().add("drop").on_click(() => this.drop_zcache(namespace));
+			});
+		}
+
 		// stats
 		section = this.add_section("stats");
 		columns = [
@@ -91,4 +127,23 @@ export class Page extends ZenPage
 			.option("Yes", () => drop())
 			.option("No");
 	}
+
+	view_zcache(namespace)
+	{
+		window.location = "?page=zcache&namespace=" + namespace;
+	}
+
+	drop_zcache(namespace)
+	{
+		const drop = async () => {
+			await new Fetcher().resource("z$", namespace).delete();
+			this.reload();
+		};
+
+		new Modal()
+			.title("Confirmation")
+			.message(`Drop zcache '${namespace}'?`)
+			.option("Yes", () => drop())
+			.option("No");
+	}
 }
diff --git a/src/zenserver/frontend/html/pages/zcache.js b/src/zenserver/frontend/html/pages/zcache.js
new file mode 100644
index 000000000..974893b21
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/zcache.js
@@ -0,0 +1,70 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { Modal } from "../util/modal.js"
+import { Table, PropTable, Toolbar } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+	async main()
+	{
+		const namespace = this.get_param("namespace");
+
+		var info = new Fetcher().resource(`/z$/${namespace}/`).json();
+
+		this.set_title("cache - " + namespace);
+
+		var section = this.add_section("info");
+		var cfg_table = section.add_section("config").add_widget(PropTable);
+		var storage_table = section.add_section("storage").add_widget(PropTable);
+
+		info = await info;
+
+		cfg_table.add_object(info["Configuration"], true);
+
+		storage_table.add_property("disk", Friendly.kib(info["StorageSize"]["DiskSize"]));
+		storage_table.add_property("mem", Friendly.kib(info["StorageSize"]["MemorySize"]));
+		storage_table.add_property("entries", Friendly.sep(info["EntryCount"]));
+
+		var column_names = ["name", "disk", "mem", "entries", "actions"];
+		var bucket_table = this.add_section("buckets").add_widget(
+			Table,
+			column_names,
+			Table.Flag_BiasLeft
+		);
+		for (const bucket of info["Buckets"])
+		{
+			const row = bucket_table.add_row(bucket);
+			new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => {
+				row.get_cell(1).text(Friendly.kib(data["StorageSize"]["DiskSize"]));
+				row.get_cell(2).text(Friendly.kib(data["StorageSize"]["MemorySize"]));
+				row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"]));
+
+				const cell = row.get_cell(-1);
+				const action_tb = new Toolbar(cell, true);
+				action_tb.left().add("view")
+				action_tb.left().add("drop").on_click(() => this.drop_bucket(bucket));
+			});
+		}
+	}
+
+	drop_bucket(bucket)
+	{
+		const drop = async () => {
+			const namespace = this.get_param("namespace");
+			await new Fetcher().resource("z$", namespace, bucket).delete();
+			this.reload();
+		};
+
+		new Modal()
+			.title("Confirmation")
+			.message(`Drop bucket '${bucket}'?`)
+			.option("Yes", () => drop())
+			.option("No");
+	}
+}
diff --git a/src/zenserver/frontend/html/util/compactbinary.js b/src/zenserver/frontend/html/util/compactbinary.js
index 366ec6aff..90e4249f6 100644
--- a/src/zenserver/frontend/html/util/compactbinary.js
+++ b/src/zenserver/frontend/html/util/compactbinary.js
@@ -284,7 +284,7 @@ CbFieldView.prototype.as_array = function()
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-CbFieldView.prototype.as_value = function(int_type=Number)
+CbFieldView.prototype.as_value = function(int_type=BigInt)
 {
 	switch (CbFieldTypeOps.get_type(this.get_type()))
 	{
@@ -388,8 +388,8 @@ CbObjectView.prototype.to_js_object = function()
 		}
 
 		if (node.is_string())	return node.as_value();
-		if (node.is_float())	return node.as_value();
 		if (node.is_integer())	return node.as_value();
+		if (node.is_float())	return node.as_value();
 
 		var ret = node.as_value();
 		if (ret instanceof Uint8Array)
diff --git a/src/zenserver/frontend/html/util/friendly.js b/src/zenserver/frontend/html/util/friendly.js
index b27721964..a15252faf 100644
--- a/src/zenserver/frontend/html/util/friendly.js
+++ b/src/zenserver/frontend/html/util/friendly.js
@@ -7,17 +7,17 @@ export class Friendly
 {
 	static sep(value, prec=0)
 	{
-		return (+value).toLocaleString("en", {
+		return (+Number(value)).toLocaleString("en", {
 			style: "decimal",
 			minimumFractionDigits : prec,
 			maximumFractionDigits : prec,
 		});
 	}
 
-	static k(x, p=0)	{ return Friendly.sep((x + 999)  / Math.pow(10, 3)|0, p) + "K"; }
-	static m(x, p=1)	{ return Friendly.sep( x         / Math.pow(10, 6),   p) + "M"; }
-	static g(x, p=2)	{ return Friendly.sep( x         / Math.pow(10, 9),   p) + "G"; }
-	static kib(x, p=0)	{ return Friendly.sep((x + 1023) / (1 << 10)|0, p) + " KiB"; }
-	static mib(x, p=1)	{ return Friendly.sep( x         / (1 << 20),   p) + " MiB"; }
-	static gib(x, p=2)	{ return Friendly.sep( x         / (1 << 30),   p) + " GiB"; }
+	static k(x, p=0)	{ return Friendly.sep((BigInt(x) + 999n)  / BigInt(Math.pow(10, 3))|0n, p) + "K"; }
+	static m(x, p=1)	{ return Friendly.sep( BigInt(x)         / BigInt(Math.pow(10, 6)),   p) + "M"; }
+	static g(x, p=2)	{ return Friendly.sep( BigInt(x)         / BigInt(Math.pow(10, 9)),   p) + "G"; }
+	static kib(x, p=0)	{ return Friendly.sep((BigInt(x) + 1023n) / (1n << 10n)|0n, p) + " KiB"; }
+	static mib(x, p=1)	{ return Friendly.sep( BigInt(x)         / (1n << 20n),   p) + " MiB"; }
+	static gib(x, p=2)	{ return Friendly.sep( BigInt(x)         / (1n << 30n),   p) + " GiB"; }
 }
diff --git a/src/zenserver/frontend/html/util/widgets.js b/src/zenserver/frontend/html/util/widgets.js
index d4f9875cd..32a3f4d28 100644
--- a/src/zenserver/frontend/html/util/widgets.js
+++ b/src/zenserver/frontend/html/util/widgets.js
@@ -173,7 +173,7 @@ export class PropTable extends Table
 					continue;
 				}
 
-				if (friendly && typeof value == "number")
+				if (friendly && ((typeof value == "number") || (typeof value == "bigint")))
 				{
 					if (key.indexOf("memory") >= 0)		value = Friendly.kib(value);
 					else if (key.indexOf("disk") >= 0)	value = Friendly.kib(value);
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 9ae54bdf1..6bd098da9 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -10,6 +10,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
+#include <zencore/sentryintegration.h>
 #include <zencore/session.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
@@ -27,7 +28,6 @@
 
 #include "config.h"
 #include "diag/logging.h"
-#include "sentryintegration.h"
 
 #if ZEN_PLATFORM_WINDOWS
 #	include <zencore/windows.h>
@@ -140,12 +140,18 @@ ZenEntryPoint::Run()
 #if ZEN_USE_SENTRY
 	SentryIntegration Sentry;
 
-	if (m_ServerOptions.NoSentry == false)
+	if (m_ServerOptions.SentryConfig.Disable == false)
 	{
 		std::string SentryDatabasePath	 = (m_ServerOptions.DataDir / ".sentry-native").string();
 		std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string();
 
-		Sentry.Initialize(SentryDatabasePath, SentryAttachmentPath, m_ServerOptions.SentryAllowPII, m_ServerOptions.CommandLine);
+		Sentry.Initialize({.DatabasePath	= SentryDatabasePath,
+						   .AttachmentsPath = SentryAttachmentPath,
+						   .Dsn				= m_ServerOptions.SentryConfig.Dsn,
+						   .Environment		= m_ServerOptions.SentryConfig.Environment,
+						   .AllowPII		= m_ServerOptions.SentryConfig.AllowPII,
+						   .Debug			= m_ServerOptions.SentryConfig.Debug},
+						  m_ServerOptions.CommandLine);
 	}
 #endif
 	try
@@ -475,17 +481,17 @@ main(int argc, char* argv[])
 
 		if (!DeleteReason.empty())
 		{
-			if (std::filesystem::exists(ServerOptions.DataDir))
+			if (IsDir(ServerOptions.DataDir))
 			{
 				ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason);
 				DeleteDirectories(ServerOptions.DataDir);
 			}
 		}
 
-		if (!std::filesystem::exists(ServerOptions.DataDir))
+		if (!IsDir(ServerOptions.DataDir))
 		{
 			ServerOptions.IsFirstRun = true;
-			std::filesystem::create_directories(ServerOptions.DataDir);
+			CreateDirectories(ServerOptions.DataDir);
 		}
 
 		if (!ServerOptions.BaseSnapshotDir.empty())
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index e757ef84e..8faf12165 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -219,13 +219,17 @@ private:
 	StringBuilderBase& Builder;
 };
 
-HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg))
+HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg)
+: m_StatusService(StatusService)
+, m_Cfg(std::move(Cfg))
 {
 	Inititalize();
+	m_StatusService.RegisterHandler("obj", *this);
 }
 
 HttpObjectStoreService::~HttpObjectStoreService()
 {
+	m_StatusService.UnregisterHandler("obj", *this);
 }
 
 const char*
@@ -245,13 +249,21 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
 }
 
 void
+HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpObjectStoreService::Inititalize()
 {
 	namespace fs = std::filesystem;
 	ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory);
 
 	const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets";
-	if (!fs::exists(BucketsPath))
+	if (!IsDir(BucketsPath))
 	{
 		CreateDirectories(BucketsPath);
 	}
@@ -324,7 +336,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
 	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
 		std::lock_guard _(BucketsMutex);
-		if (!fs::exists(BucketPath))
+		if (!IsDir(BucketPath))
 		{
 			CreateDirectories(BucketPath);
 			ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName);
@@ -406,7 +418,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 	Visitor				FileVisitor(BucketName, BucketRoot, RelativeBucketPath);
 	FileSystemTraversal Traversal;
 
-	if (std::filesystem::exists(FullPath))
+	if (IsDir(FullPath))
 	{
 		std::lock_guard _(BucketsMutex);
 		Traversal.TraverseFileSystem(FullPath, FileVisitor);
@@ -475,7 +487,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 	}
 
 	const fs::path FilePath = BucketDir / RelativeBucketPath;
-	if (!fs::exists(FilePath))
+	if (!IsFile(FilePath))
 	{
 		ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath);
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
@@ -576,7 +588,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
 	{
 		std::lock_guard _(BucketsMutex);
 
-		if (!fs::exists(FileDirectory))
+		if (!IsDir(FileDirectory))
 		{
 			CreateDirectories(FileDirectory);
 		}
diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h
index dae979c4c..44e50e208 100644
--- a/src/zenserver/objectstore/objectstore.h
+++ b/src/zenserver/objectstore/objectstore.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
 #include <atomic>
 #include <filesystem>
 #include <mutex>
@@ -23,14 +24,15 @@ struct ObjectStoreConfig
 	std::vector<BucketConfig> Buckets;
 };
 
-class HttpObjectStoreService final : public zen::HttpService
+class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider
 {
 public:
-	HttpObjectStoreService(ObjectStoreConfig Cfg);
+	HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg);
 	virtual ~HttpObjectStoreService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	void				  Inititalize();
@@ -41,6 +43,7 @@ private:
 	void				  GetObject(zen::HttpRouterRequest& Request, const std::string_view Path);
 	void				  PutObject(zen::HttpRouterRequest& Request);
 
+	HttpStatusService&	 m_StatusService;
 	ObjectStoreConfig	 m_Cfg;
 	std::mutex			 BucketsMutex;
 	HttpRequestRouter	 m_Router;
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index fbb9bc344..ab96ae92d 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -35,16 +35,10 @@ public:
 	, m_BuildId(BuildId)
 	, m_MetaData(MetaData)
 	, m_TempFilePath(TempFilePath)
+	, m_EnableBlocks(!ForceDisableBlocks)
+	, m_UseTempBlocks(!ForceDisableTempBlocks)
 	{
 		m_MetaData.MakeOwned();
-		if (ForceDisableBlocks)
-		{
-			m_EnableBlocks = false;
-		}
-		if (ForceDisableTempBlocks)
-		{
-			m_UseTempBlocks = false;
-		}
 	}
 
 	virtual RemoteStoreInfo GetInfo() const override
@@ -71,7 +65,7 @@ public:
 	{
 		ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
 
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 
 		IoBuffer Payload = m_MetaData;
 		Payload.SetContentType(ZenContentType::kCbObject);
@@ -95,7 +89,7 @@ public:
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
 	{
 		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
-		JupiterSession	   Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession	   Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		PutBuildPartResult PutResult =
 			Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload);
 		AddStats(PutResult);
@@ -120,7 +114,7 @@ public:
 												ChunkBlockDescription&& Block) override
 	{
 		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 
 		JupiterResult PutResult =
 			Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
@@ -183,7 +177,7 @@ public:
 		ZEN_UNUSED(RawHash);
 		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
 
-		JupiterSession			Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession			Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		FinalizeBuildPartResult FinalizeRefResult =
 			Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash);
 		AddStats(FinalizeRefResult);
@@ -222,7 +216,7 @@ public:
 	{
 		ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
 
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		JupiterResult  GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
 		AddStats(GetBuildResult);
 		LoadContainerResult Result{ConvertResult(GetBuildResult)};
@@ -307,8 +301,8 @@ public:
 	virtual GetKnownBlocksResult GetKnownBlocks() override
 	{
 		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
-		JupiterResult  FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId);
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
+		JupiterResult  FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1);
 		AddStats(FindResult);
 		GetKnownBlocksResult Result{ConvertResult(FindResult)};
 		if (Result.ErrorCode)
@@ -356,7 +350,7 @@ public:
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
 		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		JupiterResult  GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath);
 		AddStats(GetResult);
 
@@ -452,8 +446,9 @@ private:
 	IoBuffer			  m_MetaData;
 	Oid					  m_OplogBuildPartId = Oid::Zero;
 	std::filesystem::path m_TempFilePath;
-	bool				  m_EnableBlocks  = true;
-	bool				  m_UseTempBlocks = true;
+	const bool			  m_EnableBlocks  = true;
+	const bool			  m_UseTempBlocks = true;
+	const bool			  m_AllowRedirect = false;
 
 	std::atomic_uint64_t m_SentBytes		 = {};
 	std::atomic_uint64_t m_ReceivedBytes	 = {};
@@ -494,7 +489,15 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
 	{
 		TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
 	}
-	else
+	else if (!Options.OidcExePath.empty())
+	{
+		if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+		{
+			TokenProvider = TokenProviderMaybe.value();
+		}
+	}
+
+	if (!TokenProvider)
 	{
 		TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
 	}
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h
index 8b2c6c8c8..c52b13886 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.h
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.h
@@ -10,17 +10,18 @@ class AuthMgr;
 
 struct BuildsRemoteStoreOptions : RemoteStoreOptions
 {
-	std::string Url;
-	std::string Namespace;
-	std::string Bucket;
-	Oid			BuildId;
-	std::string OpenIdProvider;
-	std::string AccessToken;
-	AuthMgr&	AuthManager;
-	bool		ForceDisableBlocks	   = false;
-	bool		ForceDisableTempBlocks = false;
-	bool		AssumeHttp2			   = false;
-	IoBuffer	MetaData;
+	std::string			  Url;
+	std::string			  Namespace;
+	std::string			  Bucket;
+	Oid					  BuildId;
+	std::string			  OpenIdProvider;
+	std::string			  AccessToken;
+	AuthMgr&			  AuthManager;
+	std::filesystem::path OidcExePath;
+	bool				  ForceDisableBlocks	 = false;
+	bool				  ForceDisableTempBlocks = false;
+	bool				  AssumeHttp2			 = false;
+	IoBuffer			  MetaData;
 };
 
 std::shared_ptr<RemoteProjectStore> CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index 98e292d91..375e44e59 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -73,7 +73,7 @@ public:
 			ContainerObject.IterateAttachments([&](CbFieldView FieldView) {
 				IoHash				  AttachmentHash = FieldView.AsBinaryAttachment();
 				std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash);
-				if (!std::filesystem::exists(AttachmentPath))
+				if (!IsFile(AttachmentPath))
 				{
 					Result.Needs.insert(AttachmentHash);
 				}
@@ -111,7 +111,7 @@ public:
 		Stopwatch			  Timer;
 		SaveAttachmentResult  Result;
 		std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
-		if (!std::filesystem::exists(ChunkPath))
+		if (!IsFile(ChunkPath))
 		{
 			try
 			{
@@ -182,7 +182,7 @@ public:
 		for (const IoHash& RawHash : BlockHashes)
 		{
 			std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
-			if (std::filesystem::is_regular_file(ChunkPath))
+			if (IsFile(ChunkPath))
 			{
 				ExistingBlockHashes.push_back(RawHash);
 			}
@@ -203,7 +203,7 @@ public:
 		Stopwatch			  Timer;
 		LoadAttachmentResult  Result;
 		std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
-		if (!std::filesystem::is_regular_file(ChunkPath))
+		if (!IsFile(ChunkPath))
 		{
 			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
 			Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string());
@@ -246,7 +246,7 @@ private:
 		LoadContainerResult	  Result;
 		std::filesystem::path SourcePath = m_OutputPath;
 		SourcePath.append(Name);
-		if (!std::filesystem::is_regular_file(SourcePath))
+		if (!IsFile(SourcePath))
 		{
 			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
 			Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string());
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 47748dd90..317a419eb 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -8,6 +8,7 @@
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalidation.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -234,10 +235,15 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
+HttpProjectService::HttpProjectService(CidStore&		  Store,
+									   ProjectStore*	  Projects,
+									   HttpStatusService& StatusService,
+									   HttpStatsService&  StatsService,
+									   AuthMgr&			  AuthMgr)
 : m_Log(logging::Get("project"))
 , m_CidStore(Store)
 , m_ProjectStore(Projects)
+, m_StatusService(StatusService)
 , m_StatsService(StatsService)
 , m_AuthMgr(AuthMgr)
 {
@@ -245,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 	using namespace std::literals;
 
-	m_StatsService.RegisterHandler("prj", *this);
-
 	m_Router.AddPattern("project", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("log", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("op", "([[:digit:]]+?)");
@@ -365,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		"details\\$/{project}/{log}/{chunk}",
 		[this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); },
 		HttpVerb::kGet);
+
+	m_StatusService.RegisterHandler("prj", *this);
+	m_StatsService.RegisterHandler("prj", *this);
 }
 
 HttpProjectService::~HttpProjectService()
 {
 	m_StatsService.UnregisterHandler("prj", *this);
+	m_StatusService.UnregisterHandler("prj", *this);
 }
 
 const char*
@@ -465,6 +473,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 }
 
 void
+HttpProjectService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpProjectService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req)
 {
 	ZEN_TRACE_CPU("ProjectService::ProjectList");
@@ -885,10 +902,63 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req)
 		case HttpVerb::kGet:
 			{
 				IoBuffer								 Value;
-				std::pair<HttpResponseCode, std::string> Result =
-					m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr);
+				std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr);
 				if (Result.first == HttpResponseCode::OK)
 				{
+					if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary ||
+						AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+						AcceptType == ZenContentType::kCbObject)
+					{
+						CompressedBuffer Compressed			= CompressedBuffer::FromCompressedNoValidate(std::move(Value));
+						IoBuffer		 DecompressedBuffer = Compressed.Decompress().AsIoBuffer();
+
+						if (DecompressedBuffer)
+						{
+							if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+								AcceptType == ZenContentType::kCbObject)
+							{
+								CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default);
+								if (!!CbErr)
+								{
+									m_ProjectStats.BadRequestCount++;
+									ZEN_DEBUG(
+										"chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`",
+										ProjectId,
+										OplogId,
+										Cid,
+										ToString(AcceptType));
+									return HttpReq.WriteResponse(
+										HttpResponseCode::NotAcceptable,
+										HttpContentType::kText,
+										fmt::format("Content format not supported, requested {} format, but could not convert to object",
+													ToString(AcceptType)));
+								}
+
+								m_ProjectStats.ChunkHitCount++;
+								CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer);
+								return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject);
+							}
+							else
+							{
+								Value = DecompressedBuffer;
+								Value.SetContentType(ZenContentType::kBinary);
+							}
+						}
+						else
+						{
+							m_ProjectStats.BadRequestCount++;
+							ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`",
+									  ProjectId,
+									  OplogId,
+									  Cid,
+									  ToString(AcceptType));
+							return HttpReq.WriteResponse(
+								HttpResponseCode::NotAcceptable,
+								HttpContentType::kText,
+								fmt::format("Content format not supported, requested {} format, but could not decompress stored data",
+											ToString(AcceptType)));
+						}
+					}
 					m_ProjectStats.ChunkHitCount++;
 					return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
 				}
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 8e74c57a5..295defa5c 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -5,6 +5,7 @@
 #include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
 #include <zenstore/cidstore.h>
 
 namespace zen {
@@ -31,16 +32,21 @@ class ProjectStore;
 // refs:
 //
 
-class HttpProjectService : public HttpService, public IHttpStatsProvider
+class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
-	HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr);
+	HttpProjectService(CidStore&		  Store,
+					   ProjectStore*	  InProjectStore,
+					   HttpStatusService& StatusService,
+					   HttpStatsService&  StatsService,
+					   AuthMgr&			  AuthMgr);
 	~HttpProjectService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
 	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual void HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	struct ProjectStats
@@ -89,6 +95,7 @@ private:
 	CidStore&				 m_CidStore;
 	HttpRequestRouter		 m_Router;
 	Ref<ProjectStore>		 m_ProjectStore;
+	HttpStatusService&		 m_StatusService;
 	HttpStatsService&		 m_StatsService;
 	AuthMgr&				 m_AuthMgr;
 	ProjectStats			 m_ProjectStats;
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index e5839ad3b..3728babb5 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -31,15 +31,9 @@ public:
 	, m_Key(Key)
 	, m_OptionalBaseKey(OptionalBaseKey)
 	, m_TempFilePath(TempFilePath)
+	, m_EnableBlocks(!ForceDisableBlocks)
+	, m_UseTempBlocks(!ForceDisableTempBlocks)
 	{
-		if (ForceDisableBlocks)
-		{
-			m_EnableBlocks = false;
-		}
-		if (ForceDisableTempBlocks)
-		{
-			m_UseTempBlocks = false;
-		}
 	}
 
 	virtual RemoteStoreInfo GetInfo() const override
@@ -75,7 +69,7 @@ public:
 
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
 	{
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		PutRefResult   PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
 		AddStats(PutResult);
 
@@ -94,7 +88,7 @@ public:
 
 	virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override
 	{
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		JupiterResult  PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
 		AddStats(PutResult);
 
@@ -127,7 +121,7 @@ public:
 
 	virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override
 	{
-		JupiterSession	  Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession	  Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		FinalizeRefResult FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
 		AddStats(FinalizeRefResult);
 
@@ -164,7 +158,7 @@ public:
 				{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds}};
 		}
 
-		JupiterSession		Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession		Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		JupiterExistsResult ExistsResult =
 			Session.CompressedBlobExists(m_Namespace, std::set<IoHash>(BlockHashes.begin(), BlockHashes.end()));
 		AddStats(ExistsResult);
@@ -203,7 +197,7 @@ public:
 
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		JupiterResult  GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
 		AddStats(GetResult);
 
@@ -239,7 +233,7 @@ public:
 private:
 	LoadContainerResult LoadContainer(const IoHash& Key)
 	{
-		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
 		JupiterResult  GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
 		AddStats(GetResult);
 		if (GetResult.ErrorCode || !GetResult.Success)
@@ -329,8 +323,9 @@ private:
 	const IoHash		  m_Key;
 	const IoHash		  m_OptionalBaseKey;
 	std::filesystem::path m_TempFilePath;
-	bool				  m_EnableBlocks  = true;
-	bool				  m_UseTempBlocks = true;
+	const bool			  m_EnableBlocks  = true;
+	const bool			  m_UseTempBlocks = true;
+	const bool			  m_AllowRedirect = false;
 
 	std::atomic_uint64_t m_SentBytes		 = {};
 	std::atomic_uint64_t m_ReceivedBytes	 = {};
@@ -371,7 +366,15 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
 	{
 		TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
 	}
-	else
+	else if (!Options.OidcExePath.empty())
+	{
+		if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+		{
+			TokenProvider = TokenProviderMaybe.value();
+		}
+	}
+
+	if (!TokenProvider)
 	{
 		TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
 	}
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h
index 27f3d9b73..8bf79d563 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.h
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h
@@ -10,17 +10,18 @@ class AuthMgr;
 
 struct JupiterRemoteStoreOptions : RemoteStoreOptions
 {
-	std::string Url;
-	std::string Namespace;
-	std::string Bucket;
-	IoHash		Key;
-	IoHash		OptionalBaseKey;
-	std::string OpenIdProvider;
-	std::string AccessToken;
-	AuthMgr&	AuthManager;
-	bool		ForceDisableBlocks	   = false;
-	bool		ForceDisableTempBlocks = false;
-	bool		AssumeHttp2			   = false;
+	std::string			  Url;
+	std::string			  Namespace;
+	std::string			  Bucket;
+	IoHash				  Key;
+	IoHash				  OptionalBaseKey;
+	std::string			  OpenIdProvider;
+	std::string			  AccessToken;
+	AuthMgr&			  AuthManager;
+	std::filesystem::path OidcExePath;
+	bool				  ForceDisableBlocks	 = false;
+	bool				  ForceDisableTempBlocks = false;
+	bool				  AssumeHttp2			 = false;
 };
 
 std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 86791e29a..53e687983 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -22,6 +22,7 @@
 #include <zenstore/scrubcontext.h>
 #include <zenutil/cache/rpcrecording.h>
 #include <zenutil/openprocesscache.h>
+#include <zenutil/parallelwork.h>
 #include <zenutil/referencemetadata.h>
 #include <zenutil/workerpools.h>
 
@@ -58,7 +59,7 @@ namespace {
 		std::filesystem::path DroppedBucketPath;
 		do
 		{
-			if (!std::filesystem::exists(Dir))
+			if (!IsDir(Dir))
 			{
 				return true;
 			}
@@ -68,7 +69,7 @@ namespace {
 
 			std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), MovedId);
 			DroppedBucketPath		= Dir.parent_path() / DroppedName;
-			if (std::filesystem::exists(DroppedBucketPath))
+			if (IsDir(DroppedBucketPath))
 			{
 				if (!DeleteDirectories(DroppedBucketPath))
 				{
@@ -77,7 +78,7 @@ namespace {
 							 Dir);
 					continue;
 				}
-				if (std::filesystem::exists(DroppedBucketPath))
+				if (IsDir(DroppedBucketPath))
 				{
 					ZEN_INFO("Drop directory '{}' for '{}' still exists after remove, attempting different name.", DroppedBucketPath, Dir);
 					continue;
@@ -88,13 +89,13 @@ namespace {
 			do
 			{
 				std::error_code Ec;
-				std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+				RenameDirectory(Dir, DroppedBucketPath, Ec);
 				if (!Ec)
 				{
 					OutDeleteDir = DroppedBucketPath;
 					return true;
 				}
-				if (std::filesystem::exists(DroppedBucketPath))
+				if (IsDir(DroppedBucketPath))
 				{
 					ZEN_INFO("Can't rename '{}' to still existing drop directory '{}'. Reason: '{}'. Attempting different name.",
 							 Dir,
@@ -210,6 +211,16 @@ namespace {
 					AccessToken = GetEnvVariable(AccessTokenEnvVariable);
 				}
 			}
+			std::filesystem::path OidcExePath;
+			if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+			{
+				std::filesystem::path OidcExePathMaybe(OidcExePathString);
+				if (!IsFile(OidcExePathMaybe))
+				{
+					ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+					OidcExePath = std::move(OidcExePathMaybe);
+				}
+			}
 			std::string_view KeyParam = Cloud["key"sv].AsString();
 			if (KeyParam.empty())
 			{
@@ -252,6 +263,7 @@ namespace {
 												 std::string(OpenIdProvider),
 												 AccessToken,
 												 AuthManager,
+												 OidcExePath,
 												 ForceDisableBlocks,
 												 ForceDisableTempBlocks,
 												 AssumeHttp2};
@@ -307,6 +319,16 @@ namespace {
 					AccessToken = GetEnvVariable(AccessTokenEnvVariable);
 				}
 			}
+			std::filesystem::path OidcExePath;
+			if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+			{
+				std::filesystem::path OidcExePathMaybe(OidcExePathString);
+				if (!IsFile(OidcExePathMaybe))
+				{
+					ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+					OidcExePath = std::move(OidcExePathMaybe);
+				}
+			}
 			std::string_view BuildIdParam = Builds["buildsid"sv].AsString();
 			if (BuildIdParam.empty())
 			{
@@ -337,6 +359,7 @@ namespace {
 												std::string(OpenIdProvider),
 												AccessToken,
 												AuthManager,
+												OidcExePath,
 												ForceDisableBlocks,
 												ForceDisableTempBlocks,
 												AssumeHttp2,
@@ -486,7 +509,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 	[[nodiscard]] bool		  Exists() const { return Exists(m_OplogStoragePath); }
 	[[nodiscard]] static bool Exists(const std::filesystem::path& BasePath)
 	{
-		return std::filesystem::exists(GetLogPath(BasePath)) && std::filesystem::exists(GetBlobsPath(BasePath));
+		return IsFile(GetLogPath(BasePath)) && IsFile(GetBlobsPath(BasePath));
 	}
 	[[nodiscard]] bool		  IsValid() const { return IsValid(m_OplogStoragePath); }
 	[[nodiscard]] static bool IsValid(const std::filesystem::path& BasePath)
@@ -496,13 +519,13 @@ struct ProjectStore::OplogStorage : public RefCounted
 	void WipeState() const
 	{
 		std::error_code Ec;
-		std::filesystem::remove(GetLogPath(), Ec);
-		std::filesystem::remove(GetBlobsPath(), Ec);
+		RemoveFile(GetLogPath(), Ec);
+		RemoveFile(GetBlobsPath(), Ec);
 	}
 
 	static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); }
 
-	uint64_t OpBlobsSize() const { return std::filesystem::file_size(GetBlobsPath()); }
+	uint64_t OpBlobsSize() const { return FileSizeFromPath(GetBlobsPath()); }
 
 	uint64_t		OpsSize() const { return OpsSize(m_OplogStoragePath); }
 	static uint64_t OpsSize(const std::filesystem::path& BasePath)
@@ -510,7 +533,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 		if (Exists(BasePath))
 		{
 			std::error_code DummyEc;
-			return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath));
+			return FileSizeFromPath(GetLogPath(BasePath)) + FileSizeFromPath(GetBlobsPath(BasePath));
 		}
 		return 0;
 	}
@@ -689,7 +712,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 				m_OpBlobs.Close();
 
 				Oplog.Close();
-				std::filesystem::rename(OplogPath, GetLogPath(), Ec);
+				RenameFile(OplogPath, GetLogPath(), Ec);
 				if (Ec)
 				{
 					throw std::system_error(
@@ -702,9 +725,9 @@ struct ProjectStore::OplogStorage : public RefCounted
 				if (Ec)
 				{
 					// We failed late - clean everything up as best we can
-					std::filesystem::remove(OpBlobs.GetPath(), Ec);
-					std::filesystem::remove(GetLogPath(), Ec);
-					std::filesystem::remove(GetBlobsPath(), Ec);
+					RemoveFile(OpBlobs.GetPath(), Ec);
+					RemoveFile(GetLogPath(), Ec);
+					RemoveFile(GetBlobsPath(), Ec);
 					throw std::system_error(Ec,
 											fmt::format("Oplog::Compact failed to rename temporary oplog file from '{}' to '{}'",
 														OpBlobs.GetPath(),
@@ -739,7 +762,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 		}
 		catch (const std::exception& /*Ex*/)
 		{
-			std::filesystem::remove(OpBlobs.GetPath(), Ec);
+			RemoveFile(OpBlobs.GetPath(), Ec);
 			throw;
 		}
 	}
@@ -987,8 +1010,8 @@ struct ProjectStore::OplogStorage : public RefCounted
 							.OpCoreHash	  = OpData.OpCoreHash,
 							.OpKeyHash	  = OpData.KeyHash};
 
-		m_Oplog.Append(Entry);
 		m_OpBlobs.Write(OpData.Buffer.GetData(), WriteSize, WriteOffset);
+		m_Oplog.Append(Entry);
 
 		return Entry;
 	}
@@ -1108,7 +1131,7 @@ ProjectStore::Oplog::Oplog(std::string_view				Id,
 			ZEN_WARN("Invalid oplog found at '{}'. Wiping state for oplog.", m_BasePath);
 			m_Storage->WipeState();
 			std::error_code DummyEc;
-			std::filesystem::remove(m_MetaPath, DummyEc);
+			RemoveFile(m_MetaPath, DummyEc);
 		}
 	}
 	m_Storage->Open(/* IsCreate */ !StoreExists);
@@ -1116,7 +1139,7 @@ ProjectStore::Oplog::Oplog(std::string_view				Id,
 	m_MetaPath	= m_BasePath / "ops.meta"sv;
 	m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath());
 
-	CleanDirectory(m_TempPath);
+	CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false);
 }
 
 ProjectStore::Oplog::~Oplog()
@@ -1142,7 +1165,7 @@ ProjectStore::Oplog::Flush()
 	if (!m_MetaValid)
 	{
 		std::error_code DummyEc;
-		std::filesystem::remove(m_MetaPath, DummyEc);
+		RemoveFile(m_MetaPath, DummyEc);
 	}
 
 	uint64_t LogCount = m_Storage->LogCount();
@@ -1238,19 +1261,19 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath)
 
 	uint64_t			  Size			= OplogStorage::OpsSize(BasePath);
 	std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
-	if (std::filesystem::exists(StateFilePath))
+	if (IsFile(StateFilePath))
 	{
-		Size += std::filesystem::file_size(StateFilePath);
+		Size += FileSizeFromPath(StateFilePath);
 	}
 	std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv;
-	if (std::filesystem::exists(MetaFilePath))
+	if (IsFile(MetaFilePath))
 	{
-		Size += std::filesystem::file_size(MetaFilePath);
+		Size += FileSizeFromPath(MetaFilePath);
 	}
 	std::filesystem::path IndexFilePath = BasePath / "ops.zidx"sv;
-	if (std::filesystem::exists(IndexFilePath))
+	if (IsFile(IndexFilePath))
 	{
-		Size += std::filesystem::file_size(IndexFilePath);
+		Size += FileSizeFromPath(IndexFilePath);
 	}
 
 	return Size;
@@ -1303,7 +1326,7 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath)
 	using namespace std::literals;
 
 	std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
-	return std::filesystem::is_regular_file(StateFilePath);
+	return IsFile(StateFilePath);
 }
 
 bool
@@ -1337,7 +1360,7 @@ ProjectStore::Oplog::Read()
 	if (!m_MetaValid)
 	{
 		std::error_code DummyEc;
-		std::filesystem::remove(m_MetaPath, DummyEc);
+		RemoveFile(m_MetaPath, DummyEc);
 	}
 
 	ReadIndexSnapshot();
@@ -1438,7 +1461,7 @@ ProjectStore::Oplog::Reset()
 		m_Storage = new OplogStorage(this, m_BasePath);
 		m_Storage->Open(true);
 		m_MetaValid = false;
-		CleanDirectory(m_TempPath);
+		CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false);
 		Write();
 	}
 	// Erase content on disk
@@ -1457,7 +1480,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f
 	using namespace std::literals;
 
 	std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
-	if (std::filesystem::is_regular_file(StateFilePath))
+	if (IsFile(StateFilePath))
 	{
 		//		ZEN_INFO("oplog '{}/{}': config read from '{}'", m_OuterProject->Identifier, m_OplogId, StateFilePath);
 
@@ -1488,11 +1511,17 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo
 
 	ValidationResult Result;
 
+	const size_t OpCount = OplogCount();
+
 	std::vector<Oid>				 KeyHashes;
 	std::vector<std::string>		 Keys;
 	std::vector<std::vector<IoHash>> Attachments;
 	std::vector<OplogEntryMapping>	 Mappings;
 
+	KeyHashes.reserve(OpCount);
+	Keys.reserve(OpCount);
+	Mappings.reserve(OpCount);
+
 	IterateOplogWithKey([&](uint32_t LSN, const Oid& Key, CbObjectView OpView) {
 		Result.LSNLow  = Min(Result.LSNLow, LSN);
 		Result.LSNHigh = Max(Result.LSNHigh, LSN);
@@ -1517,77 +1546,90 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo
 		bool					 HasMissingEntries = false;
 		for (const ChunkMapping& Chunk : Mapping.Chunks)
 		{
-			if (IoBuffer Payload = m_CidStore.FindChunkByCid(Chunk.Hash); !Payload)
+			if (!m_CidStore.ContainsChunk(Chunk.Hash))
 			{
 				ResultLock.WithExclusiveLock([&]() { Result.MissingChunks.push_back({KeyHash, Chunk}); });
 				HasMissingEntries = true;
 			}
 		}
+
 		for (const ChunkMapping& Meta : Mapping.Meta)
 		{
-			if (IoBuffer Payload = m_CidStore.FindChunkByCid(Meta.Hash); !Payload)
+			if (!m_CidStore.ContainsChunk(Meta.Hash))
 			{
 				ResultLock.WithExclusiveLock([&]() { Result.MissingMetas.push_back({KeyHash, Meta}); });
 				HasMissingEntries = true;
 			}
 		}
+
 		for (const FileMapping& File : Mapping.Files)
 		{
 			if (File.Hash == IoHash::Zero)
 			{
 				std::filesystem::path FilePath = m_OuterProject->RootDir / File.ServerPath;
-				if (!std::filesystem::is_regular_file(FilePath))
+				if (!IsFile(FilePath))
 				{
 					ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); });
 					HasMissingEntries = true;
 				}
 			}
-			else
+			else if (!m_CidStore.ContainsChunk(File.Hash))
 			{
-				if (IoBuffer Payload = m_CidStore.FindChunkByCid(File.Hash); !Payload)
-				{
-					ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); });
-					HasMissingEntries = true;
-				}
+				ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); });
+				HasMissingEntries = true;
 			}
 		}
+
 		const std::vector<IoHash>& OpAttachments = Attachments[OpIndex];
 		for (const IoHash& Attachment : OpAttachments)
 		{
-			if (IoBuffer Payload = m_CidStore.FindChunkByCid(Attachment); !Payload)
+			if (!m_CidStore.ContainsChunk(Attachment))
 			{
 				ResultLock.WithExclusiveLock([&]() { Result.MissingAttachments.push_back({KeyHash, Attachment}); });
 				HasMissingEntries = true;
 			}
 		}
+
 		if (HasMissingEntries)
 		{
 			ResultLock.WithExclusiveLock([&]() { Result.OpKeys.push_back({KeyHash, Key}); });
 		}
 	};
 
-	Latch WorkLatch(1);
-
-	for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++)
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	try
 	{
-		if (OptionalWorkerPool)
-		{
-			WorkLatch.AddCount(1);
-			OptionalWorkerPool->ScheduleWork([&, Index = OpIndex]() {
-				ZEN_MEMSCOPE(GetProjectstoreTag());
-
-				auto _ = MakeGuard([&WorkLatch] { WorkLatch.CountDown(); });
-				ValidateOne(Index);
-			});
-		}
-		else
+		for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++)
 		{
-			ValidateOne(OpIndex);
+			if (AbortFlag)
+			{
+				break;
+			}
+			if (OptionalWorkerPool)
+			{
+				Work.ScheduleWork(*OptionalWorkerPool, [&ValidateOne, Index = OpIndex](std::atomic<bool>& AbortFlag) {
+					ZEN_MEMSCOPE(GetProjectstoreTag());
+					if (AbortFlag)
+					{
+						return;
+					}
+					ValidateOne(Index);
+				});
+			}
+			else
+			{
+				ValidateOne(OpIndex);
+			}
 		}
 	}
-
-	WorkLatch.CountDown();
-	WorkLatch.Wait();
+	catch (const std::exception& Ex)
+	{
+		AbortFlag.store(true);
+		ZEN_WARN("Failed validating oplogs in {}. Reason: '{}'", m_BasePath, Ex.what());
+	}
+	Work.Wait();
 
 	{
 		// Check if we were deleted while we were checking the references without a lock...
@@ -1621,31 +1663,9 @@ ProjectStore::Oplog::WriteIndexSnapshot()
 
 	namespace fs = std::filesystem;
 
-	fs::path IndexPath	   = m_BasePath / "ops.zidx";
-	fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp";
-
-	// Move index away, we keep it if something goes wrong
-	if (fs::is_regular_file(TempIndexPath))
-	{
-		std::error_code Ec;
-		if (!fs::remove(TempIndexPath, Ec) || Ec)
-		{
-			ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'",
-					 GetOuterProject()->Identifier,
-					 m_OplogId,
-					 TempIndexPath,
-					 Ec.message());
-			return;
-		}
-	}
-
+	const fs::path IndexPath = m_BasePath / "ops.zidx";
 	try
 	{
-		if (fs::is_regular_file(IndexPath))
-		{
-			fs::rename(IndexPath, TempIndexPath);
-		}
-
 		// Write the current state of the location map to a new index state
 		std::vector<uint32_t>		   LSNEntries;
 		std::vector<Oid>			   Keys;
@@ -1726,36 +1746,28 @@ ProjectStore::Oplog::WriteIndexSnapshot()
 
 			uint64_t Offset = 0;
 			IndexFile.Write(&Header, sizeof(OplogIndexHeader), Offset);
-			Offset += sizeof(OplogIndexHeader);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			IndexFile.Write(LSNEntries.data(), LSNEntries.size() * sizeof(uint32_t), Offset);
-			Offset += LSNEntries.size() * sizeof(uint32_t);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			IndexFile.Write(Keys.data(), Keys.size() * sizeof(Oid), Offset);
-			Offset += Keys.size() * sizeof(Oid);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			IndexFile.Write(AddressMapEntries.data(), AddressMapEntries.size() * sizeof(OplogEntryAddress), Offset);
-			Offset += AddressMapEntries.size() * sizeof(OplogEntryAddress);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			IndexFile.Write(LatestOpMapEntries.data(), LatestOpMapEntries.size() * sizeof(uint32_t), Offset);
-			Offset += LatestOpMapEntries.size() * sizeof(uint32_t);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			IndexFile.Write(ChunkMapEntries.data(), ChunkMapEntries.size() * sizeof(IoHash), Offset);
-			Offset += ChunkMapEntries.size() * sizeof(IoHash);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			IndexFile.Write(MetaMapEntries.data(), MetaMapEntries.size() * sizeof(IoHash), Offset);
-			Offset += MetaMapEntries.size() * sizeof(IoHash);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			IndexFile.Write(FilePathLengths.data(), FilePathLengths.size() * sizeof(uint32_t), Offset);
-			Offset += FilePathLengths.size() * sizeof(uint32_t);
-			Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+			Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
 
 			for (const auto& FilePath : FilePaths)
 			{
@@ -1767,7 +1779,11 @@ ProjectStore::Oplog::WriteIndexSnapshot()
 		ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec);
 		if (Ec)
 		{
-			throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", ObjectIndexFile.GetPath(), IndexPath));
+			throw std::system_error(Ec,
+									fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'",
+												ObjectIndexFile.GetPath(),
+												IndexPath,
+												Ec.message()));
 		}
 		EntryCount		   = LSNEntries.size();
 		m_LogFlushPosition = IndexLogPosition;
@@ -1775,35 +1791,6 @@ ProjectStore::Oplog::WriteIndexSnapshot()
 	catch (const std::exception& Err)
 	{
 		ZEN_WARN("oplog '{}/{}': snapshot FAILED, reason: '{}'", m_OuterProject->Identifier, m_OplogId, Err.what());
-
-		// Restore any previous snapshot
-
-		if (fs::is_regular_file(TempIndexPath))
-		{
-			std::error_code Ec;
-			fs::remove(IndexPath, Ec);	// We don't care if this fails, we try to move the old temp file regardless
-			fs::rename(TempIndexPath, IndexPath, Ec);
-			if (Ec)
-			{
-				ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'",
-						 m_OuterProject->Identifier,
-						 m_OplogId,
-						 TempIndexPath,
-						 Ec.message());
-			}
-		}
-	}
-	if (fs::is_regular_file(TempIndexPath))
-	{
-		std::error_code Ec;
-		if (!fs::remove(TempIndexPath, Ec) || Ec)
-		{
-			ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'",
-					 m_OuterProject->Identifier,
-					 m_OplogId,
-					 TempIndexPath,
-					 Ec.message());
-		}
 	}
 }
 
@@ -1813,8 +1800,8 @@ ProjectStore::Oplog::ReadIndexSnapshot()
 	ZEN_MEMSCOPE(GetProjectstoreTag());
 	ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot");
 
-	std::filesystem::path IndexPath = m_BasePath / "ops.zidx";
-	if (std::filesystem::is_regular_file(IndexPath))
+	const std::filesystem::path IndexPath = m_BasePath / "ops.zidx";
+	if (IsFile(IndexPath))
 	{
 		uint64_t   EntryCount = 0;
 		Stopwatch  Timer;
@@ -2139,66 +2126,81 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid>																	  ChunkIds,
 	}
 	if (OptionalWorkerPool)
 	{
-		std::atomic_bool Result = true;
-		Latch			 WorkLatch(1);
-
-		for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
+		std::atomic<bool> AbortFlag;
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
+		try
 		{
-			if (Result.load() == false)
+			for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
 			{
-				break;
-			}
-			WorkLatch.AddCount(1);
-			OptionalWorkerPool->ScheduleWork(
-				[this, &WorkLatch, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() {
-					auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
-					if (Result.load() == false)
-					{
-						return;
-					}
-					size_t						 FileChunkIndex = FileChunkIndexes[ChunkIndex];
-					const std::filesystem::path& FilePath		= FileChunkPaths[ChunkIndex];
-					try
-					{
-						IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
-						if (!Payload)
+				if (AbortFlag)
+				{
+					break;
+				}
+				Work.ScheduleWork(
+					*OptionalWorkerPool,
+					[this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback](
+						std::atomic<bool>& AbortFlag) {
+						if (AbortFlag)
 						{
-							ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[ChunkIndex], FilePath);
+							return;
 						}
+						size_t						 FileChunkIndex = FileChunkIndexes[ChunkIndex];
+						const std::filesystem::path& FilePath		= FileChunkPaths[ChunkIndex];
+						try
+						{
+							IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
+							if (!Payload)
+							{
+								ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath);
+							}
 
-						if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0))
+							if (!AsyncCallback(FileChunkIndex,
+											   Payload,
+											   IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0))
+							{
+								AbortFlag.store(true);
+							}
+						}
+						catch (const std::exception& Ex)
 						{
-							Result.store(false);
+							ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'",
+									 m_OuterProject->Identifier,
+									 m_OplogId,
+									 FileChunkIndex,
+									 FilePath,
+									 Ex.what());
 						}
-					}
-					catch (const std::exception& Ex)
-					{
-						ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'",
-								 m_OuterProject->Identifier,
-								 m_OplogId,
-								 FileChunkIndex,
-								 FilePath,
-								 Ex.what());
-					}
-				});
-		}
+					});
+			}
 
-		if (!CidChunkHashes.empty())
+			if (!CidChunkHashes.empty() && !AbortFlag)
+			{
+				m_CidStore.IterateChunks(
+					CidChunkHashes,
+					[&](size_t Index, const IoBuffer& Payload) {
+						size_t CidChunkIndex = CidChunkIndexes[Index];
+						if (AbortFlag)
+						{
+							return false;
+						}
+						return AsyncCallback(CidChunkIndex,
+											 Payload,
+											 IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
+					},
+					OptionalWorkerPool,
+					LargeSizeLimit);
+			}
+		}
+		catch (const std::exception& Ex)
 		{
-			m_CidStore.IterateChunks(
-				CidChunkHashes,
-				[&](size_t Index, const IoBuffer& Payload) {
-					size_t CidChunkIndex = CidChunkIndexes[Index];
-					return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
-				},
-				OptionalWorkerPool,
-				LargeSizeLimit);
+			AbortFlag.store(true);
+			ZEN_WARN("Failed iterating oplog chunks in {}. Reason: '{}'", m_BasePath, Ex.what());
 		}
 
-		WorkLatch.CountDown();
-		WorkLatch.Wait();
+		Work.Wait();
 
-		return Result.load();
+		return !AbortFlag;
 	}
 	else
 	{
@@ -3133,7 +3135,7 @@ ProjectStore::Project::~Project()
 bool
 ProjectStore::Project::Exists(const std::filesystem::path& BasePath)
 {
-	return std::filesystem::exists(BasePath / "Project.zcb");
+	return IsFile(BasePath / "Project.zcb");
 }
 
 void
@@ -3207,7 +3209,7 @@ ProjectStore::Project::ReadAccessTimes()
 	using namespace std::literals;
 
 	std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv;
-	if (!std::filesystem::exists(ProjectAccessTimesFilePath))
+	if (!IsFile(ProjectAccessTimesFilePath))
 	{
 		return;
 	}
@@ -3598,14 +3600,14 @@ ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath)
 
 	uint64_t			  Size				  = 0;
 	std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv;
-	if (std::filesystem::exists(AccessTimesFilePath))
+	if (IsFile(AccessTimesFilePath))
 	{
-		Size += std::filesystem::file_size(AccessTimesFilePath);
+		Size += FileSizeFromPath(AccessTimesFilePath);
 	}
 	std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv;
-	if (std::filesystem::exists(ProjectFilePath))
+	if (IsFile(ProjectFilePath))
 	{
-		Size += std::filesystem::file_size(ProjectFilePath);
+		Size += FileSizeFromPath(ProjectFilePath);
 	}
 
 	return Size;
@@ -3717,7 +3719,7 @@ ProjectStore::Project::IsExpired(const std::string&			  EntryName,
 	if (!MarkerPath.empty())
 	{
 		std::error_code Ec;
-		if (std::filesystem::exists(MarkerPath, Ec))
+		if (IsFile(MarkerPath, Ec))
 		{
 			if (Ec)
 			{
@@ -3870,7 +3872,7 @@ void
 ProjectStore::DiscoverProjects()
 {
 	ZEN_MEMSCOPE(GetProjectstoreTag());
-	if (!std::filesystem::exists(m_ProjectBasePath))
+	if (!IsDir(m_ProjectBasePath))
 	{
 		return;
 	}
@@ -3919,26 +3921,32 @@ ProjectStore::Flush()
 	}
 
 	WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst);
-	Latch			  WorkLatch(1);
-
-	for (const Ref<Project>& Project : Projects)
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	try
 	{
-		WorkLatch.AddCount(1);
-		WorkerPool.ScheduleWork([this, &WorkLatch, Project]() {
-			auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
-			try
-			{
-				Project->Flush();
-			}
-			catch (const std::exception& Ex)
-			{
-				ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what());
-			}
-		});
+		for (const Ref<Project>& Project : Projects)
+		{
+			Work.ScheduleWork(WorkerPool, [this, Project](std::atomic<bool>&) {
+				try
+				{
+					Project->Flush();
+				}
+				catch (const std::exception& Ex)
+				{
+					ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what());
+				}
+			});
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		AbortFlag.store(true);
+		ZEN_WARN("Failed projects in {}. Reason: '{}'", m_ProjectBasePath, Ex.what());
 	}
 
-	WorkLatch.CountDown();
-	WorkLatch.Wait();
+	Work.Wait();
 }
 
 void
@@ -3979,7 +3987,7 @@ ProjectStore::StorageSize() const
 
 	GcStorageSize Result;
 	{
-		if (std::filesystem::exists(m_ProjectBasePath))
+		if (IsDir(m_ProjectBasePath))
 		{
 			DirectoryContent ProjectsFolderContent;
 			GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent);
@@ -3987,7 +3995,7 @@ ProjectStore::StorageSize() const
 			for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories)
 			{
 				std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv;
-				if (std::filesystem::exists(ProjectStateFilePath))
+				if (IsFile(ProjectStateFilePath))
 				{
 					Result.DiskSize += Project::TotalSize(ProjectBasePath);
 					DirectoryContent DirContent;
@@ -4770,7 +4778,6 @@ std::pair<HttpResponseCode, std::string>
 ProjectStore::GetChunk(const std::string_view ProjectId,
 					   const std::string_view OplogId,
 					   const std::string_view Cid,
-					   ZenContentType		  AcceptType,
 					   IoBuffer&			  OutChunk,
 					   uint64_t*			  OptionalInOutModificationTag)
 {
@@ -4812,16 +4819,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
 		}
 	}
 
-	if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary)
-	{
-		CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk));
-		OutChunk					= Compressed.Decompress().AsIoBuffer();
-		OutChunk.SetContentType(ZenContentType::kBinary);
-	}
-	else
-	{
-		OutChunk.SetContentType(ZenContentType::kCompressedBinary);
-	}
+	OutChunk.SetContentType(ZenContentType::kCompressedBinary);
 	return {HttpResponseCode::OK, {}};
 }
 
@@ -7253,7 +7251,7 @@ TEST_CASE("project.store.gc")
 		CHECK(ProjectStore.OpenProject("proj2"sv));
 	}
 
-	std::filesystem::remove(Project1FilePath);
+	RemoveFile(Project1FilePath);
 
 	{
 		GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(24),
@@ -7282,7 +7280,7 @@ TEST_CASE("project.store.gc")
 		CHECK(ProjectStore.OpenProject("proj2"sv));
 	}
 
-	std::filesystem::remove(Project2Oplog1Path);
+	RemoveFile(Project2Oplog1Path);
 	{
 		GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(24),
 							   .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
@@ -7311,7 +7309,7 @@ TEST_CASE("project.store.gc")
 		CHECK(ProjectStore.OpenProject("proj2"sv));
 	}
 
-	std::filesystem::remove(Project2FilePath);
+	RemoveFile(Project2FilePath);
 	{
 		GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(24),
 							   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24),
@@ -8035,7 +8033,7 @@ TEST_CASE("project.store.rpc.getchunks")
 			CompositeBuffer Buffer = Attachment->AsCompositeBinary();
 			CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)),
 					 IoHash::HashBuffer(Buffer));
-			CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second));
+			CHECK_EQ(Chunk["Size"sv].AsUInt64(), FileSizeFromPath(FilesOpIdAttachments[0].second));
 			CHECK(!Chunk.FindView("RawSize"));
 		}
 		{
@@ -8516,12 +8514,7 @@ TEST_CASE("project.store.partial.read")
 		uint64_t ModificationTag = 0;
 		IoBuffer Chunk;
 		CHECK(ProjectStore
-				  .GetChunk("proj1"sv,
-							"oplog1"sv,
-							Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
-							HttpContentType::kCompressedBinary,
-							Chunk,
-							&ModificationTag)
+				  .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
 				  .first == HttpResponseCode::OK);
 		IoHash			 RawHash;
 		uint64_t		 RawSize;
@@ -8530,12 +8523,7 @@ TEST_CASE("project.store.partial.read")
 
 		CHECK(ModificationTag != 0);
 		CHECK(ProjectStore
-				  .GetChunk("proj1"sv,
-							"oplog1"sv,
-							Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
-							HttpContentType::kCompressedBinary,
-							Chunk,
-							&ModificationTag)
+				  .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
 				  .first == HttpResponseCode::NotModified);
 	}
 
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 8f2d3ce0d..368da5ea4 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -449,7 +449,6 @@ public:
 	std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId,
 													  const std::string_view OplogId,
 													  const std::string_view Cid,
-													  ZenContentType		 AcceptType,
 													  IoBuffer&				 OutChunk,
 													  uint64_t*				 OptionalInOutModificationTag);
 
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index a7263da83..f96b3e185 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -1212,7 +1212,7 @@ BuildContainer(CidStore&																	ChunkStore,
 					{
 						std::string_view	  ServerPath = View["serverpath"sv].AsString();
 						std::filesystem::path FilePath	 = Project.RootDir / ServerPath;
-						if (!std::filesystem::is_regular_file(FilePath))
+						if (!IsFile(FilePath))
 						{
 							remotestore_impl::ReportMessage(
 								OptionalContext,
@@ -3083,9 +3083,9 @@ LoadOplog(CidStore&			   ChunkStore,
 										 OptionalContext]() {
 					auto _ = MakeGuard([&DechunkLatch, &TempFileName] {
 						std::error_code Ec;
-						if (std::filesystem::exists(TempFileName, Ec))
+						if (IsFile(TempFileName, Ec))
 						{
-							std::filesystem::remove(TempFileName, Ec);
+							RemoveFile(TempFileName, Ec);
 							if (Ec)
 							{
 								ZEN_INFO("Failed to remove temporary file '{}'. Reason: {}", TempFileName, Ec.message());
diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp
index e438a840a..744b861dd 100644
--- a/src/zenserver/upstream/upstreamcache.cpp
+++ b/src/zenserver/upstream/upstreamcache.cpp
@@ -134,7 +134,7 @@ namespace detail {
 					return {.State = UpstreamEndpointState::kOk};
 				}
 
-				JupiterSession		Session(m_Client->Logger(), m_Client->Client());
+				JupiterSession		Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect);
 				const JupiterResult Result = Session.Authenticate();
 
 				if (Result.Success)
@@ -181,7 +181,7 @@ namespace detail {
 
 			try
 			{
-				JupiterSession Session(m_Client->Logger(), m_Client->Client());
+				JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect);
 				JupiterResult  Result;
 
 				std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace);
@@ -301,7 +301,7 @@ namespace detail {
 		{
 			ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheRecords");
 
-			JupiterSession		   Session(m_Client->Logger(), m_Client->Client());
+			JupiterSession		   Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect);
 			GetUpstreamCacheResult Result;
 
 			for (CacheKeyRequest* Request : Requests)
@@ -365,7 +365,7 @@ namespace detail {
 
 			try
 			{
-				JupiterSession		Session(m_Client->Logger(), m_Client->Client());
+				JupiterSession		Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect);
 				std::string_view	BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace);
 				const JupiterResult Result			   = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId);
 
@@ -398,7 +398,7 @@ namespace detail {
 		{
 			ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheChunks");
 
-			JupiterSession		   Session(m_Client->Logger(), m_Client->Client());
+			JupiterSession		   Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect);
 			GetUpstreamCacheResult Result;
 
 			for (CacheChunkRequest* RequestPtr : CacheChunkRequests)
@@ -453,7 +453,7 @@ namespace detail {
 		{
 			ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheValues");
 
-			JupiterSession		   Session(m_Client->Logger(), m_Client->Client());
+			JupiterSession		   Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect);
 			GetUpstreamCacheResult Result;
 
 			for (CacheValueRequest* RequestPtr : CacheValueRequests)
@@ -533,7 +533,7 @@ namespace detail {
 
 			try
 			{
-				JupiterSession Session(m_Client->Logger(), m_Client->Client());
+				JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect);
 
 				if (CacheRecord.Type == ZenContentType::kBinary)
 				{
@@ -756,6 +756,7 @@ namespace detail {
 		UpstreamStatus		  m_Status;
 		UpstreamEndpointStats m_Stats;
 		RefPtr<JupiterClient> m_Client;
+		const bool			  m_AllowRedirect = false;
 	};
 
 	class ZenUpstreamEndpoint final : public UpstreamEndpoint
diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp
index d302a10ec..bf761f8d1 100644
--- a/src/zenserver/vfs/vfsservice.cpp
+++ b/src/zenserver/vfs/vfsservice.cpp
@@ -61,7 +61,7 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb)
 // echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @-
 // echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @-
 
-VfsService::VfsService()
+VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
 {
 	m_Impl = new Impl;
 
@@ -136,10 +136,12 @@ VfsService::VfsService()
 			}
 		},
 		HttpVerb::kPost);
+	m_StatusService.RegisterHandler("vfs", *this);
 }
 
 VfsService::~VfsService()
 {
+	m_StatusService.UnregisterHandler("vfs", *this);
 	delete m_Impl;
 }
 
@@ -169,8 +171,9 @@ VfsService::AddService(Ref<ZenCacheStore>&& Z$)
 
 #else
 
-VfsService::VfsService()
+VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
 {
+	ZEN_UNUSED(StatusService);
 }
 
 VfsService::~VfsService()
@@ -209,6 +212,14 @@ VfsService::BaseUri() const
 }
 
 void
+VfsService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest)
 {
 	m_Router.HandleRequest(HttpServiceRequest);
diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h
index dcdc71e81..0d0168e23 100644
--- a/src/zenserver/vfs/vfsservice.h
+++ b/src/zenserver/vfs/vfsservice.h
@@ -4,6 +4,7 @@
 
 #include <zenbase/refcount.h>
 #include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
 #include <zenvfs/vfs.h>
 
 #include <memory>
@@ -24,10 +25,10 @@ class ZenCacheStore;
 
  */
 
-class VfsService : public HttpService
+class VfsService : public HttpService, public IHttpStatusProvider
 {
 public:
-	VfsService();
+	explicit VfsService(HttpStatusService& StatusService);
 	~VfsService();
 
 	void Mount(std::string_view MountPoint);
@@ -39,12 +40,14 @@ public:
 protected:
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& HttpServiceRequest) override;
+	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	struct Impl;
 	Impl* m_Impl = nullptr;
 
-	HttpRequestRouter m_Router;
+	HttpStatusService& m_StatusService;
+	HttpRequestRouter  m_Router;
 
 	friend struct VfsServiceDataSource;
 };
diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp
index 8a4b977ad..7ef84743e 100644
--- a/src/zenserver/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/workspaces/httpworkspaces.cpp
@@ -73,8 +73,12 @@ namespace {
 
 }  // namespace
 
-HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces)
+HttpWorkspacesService::HttpWorkspacesService(HttpStatusService&			  StatusService,
+											 HttpStatsService&			  StatsService,
+											 const WorkspacesServeConfig& Cfg,
+											 Workspaces&				  Workspaces)
 : m_Log(logging::Get("workspaces"))
+, m_StatusService(StatusService)
 , m_StatsService(StatsService)
 , m_Config(Cfg)
 , m_Workspaces(Workspaces)
@@ -84,7 +88,8 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con
 
 HttpWorkspacesService::~HttpWorkspacesService()
 {
-	m_StatsService.UnregisterHandler("prj", *this);
+	m_StatsService.UnregisterHandler("ws", *this);
+	m_StatusService.UnregisterHandler("ws", *this);
 }
 
 const char*
@@ -149,14 +154,21 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
 }
 
 void
+HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpWorkspacesService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpWorkspacesService::Initialize()
 {
 	using namespace std::literals;
 
 	ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service");
 
-	m_StatsService.RegisterHandler("ws", *this);
-
 	m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})");
 	m_Router.AddPattern("share_id", "([[:xdigit:]]{24})");
 	m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
@@ -238,6 +250,9 @@ HttpWorkspacesService::Initialize()
 		HttpVerb::kGet);
 
 	RefreshState();
+
+	m_StatsService.RegisterHandler("ws", *this);
+	m_StatusService.RegisterHandler("ws", *this);
 }
 
 std::filesystem::path
@@ -1100,7 +1115,7 @@ HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& Workspace
 					}
 				}
 
-				if (!std::filesystem::is_directory(Workspace.RootPath / NewConfig.SharePath))
+				if (!IsDir(Workspace.RootPath / NewConfig.SharePath))
 				{
 					return ServerRequest.WriteResponse(HttpResponseCode::NotFound,
 													   HttpContentType::kText,
diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h
index f01f58b86..89a8e8bdc 100644
--- a/src/zenserver/workspaces/httpworkspaces.h
+++ b/src/zenserver/workspaces/httpworkspaces.h
@@ -5,6 +5,7 @@
 #include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
 
 namespace zen {
 
@@ -16,16 +17,20 @@ struct WorkspacesServeConfig
 	bool				  AllowConfigurationChanges = false;
 };
 
-class HttpWorkspacesService final : public HttpService, public IHttpStatsProvider
+class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
-	HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces);
+	HttpWorkspacesService(HttpStatusService&		   StatusService,
+						  HttpStatsService&			   StatsService,
+						  const WorkspacesServeConfig& Cfg,
+						  Workspaces&				   Workspaces);
 	virtual ~HttpWorkspacesService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
 	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual void HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	struct WorkspacesStats
@@ -80,6 +85,7 @@ private:
 	void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId);
 	void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId);
 
+	HttpStatusService&			m_StatusService;
 	HttpStatsService&			m_StatsService;
 	const WorkspacesServeConfig m_Config;
 	HttpRequestRouter			m_Router;
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index a3d7aa124..470fbd24e 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -28,8 +28,6 @@ target("zenserver")
         add_cxxflags("/bigobj")
         add_links("delayimp", "projectedfslib")
         add_ldflags("/delayload:ProjectedFSLib.dll")
-
-        add_links("dbghelp", "winhttp", "version")    -- for Sentry
     else
         remove_files("windows/**")
     end
@@ -41,7 +39,6 @@ target("zenserver")
         add_ldflags("-framework Foundation")
         add_ldflags("-framework Security")
         add_ldflags("-framework SystemConfiguration")
-        add_syslinks("bsm")
     end
 
     add_options("compute")
@@ -57,18 +54,6 @@ target("zenserver")
         "vcpkg::sol2"
     )
 
-    if has_config("zensentry") then
-        add_packages("vcpkg::sentry-native")
-    end
-
-    if is_plat("linux") then
-        -- As sentry_native uses symbols from breakpad_client, the latter must
-        -- be specified after the former with GCC-like toolchains. xmake however
-        -- is unaware of this and simply globs files from vcpkg's output. The
-        -- line below forces breakpad_client to be to the right of sentry_native
-        add_syslinks("breakpad_client")
-    end
-
     -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to 
     -- our use of setsid() at startup we pass in `--no-detach` to zenserver
     -- ensure that it recieves signals when the user requests termination
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index f84bc0b00..27ec4c690 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -2,8 +2,6 @@
 
 #include "zenserver.h"
 
-#include "sentryintegration.h"
-
 #include <zenbase/refcount.h>
 #include <zencore/basicfile.h>
 #include <zencore/compactbinarybuilder.h>
@@ -16,6 +14,7 @@
 #include <zencore/jobqueue.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
+#include <zencore/sentryintegration.h>
 #include <zencore/session.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
@@ -23,6 +22,7 @@
 #include <zencore/trace.h>
 #include <zencore/workthreadpool.h>
 #include <zenhttp/httpserver.h>
+#include <zenstore/buildstore/buildstore.h>
 #include <zenstore/cidstore.h>
 #include <zenstore/scrubcontext.h>
 #include <zenstore/workspaces.h>
@@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 		return -1;
 	}
 
-	m_UseSentry				 = ServerOptions.NoSentry == false;
+	m_UseSentry				 = ServerOptions.SentryConfig.Disable == false;
 	m_ServerEntry			 = ServerEntry;
 	m_DebugOptionForcedCrash = ServerOptions.ShouldCrash;
 	m_IsPowerCycle			 = ServerOptions.IsPowerCycle;
@@ -250,18 +250,27 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 									  *m_JobQueue,
 									  *m_OpenProcessCache,
 									  ProjectStore::Configuration{});
-	m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr});
+	m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr});
 
 	if (ServerOptions.WorksSpacesConfig.Enabled)
 	{
 		m_Workspaces.reset(new Workspaces());
 		m_HttpWorkspacesService.reset(
-			new HttpWorkspacesService(m_StatsService,
+			new HttpWorkspacesService(m_StatusService,
+									  m_StatsService,
 									  {.SystemRootDir			  = ServerOptions.SystemRootDir,
 									   .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges},
 									  *m_Workspaces));
 	}
 
+	if (ServerOptions.BuildStoreConfig.Enabled)
+	{
+		BuildStoreConfig BuildsCfg;
+		BuildsCfg.RootDirectory		= m_DataRoot / "builds";
+		BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit;
+		m_BuildStore				= std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager);
+	}
+
 	if (ServerOptions.StructuredCacheConfig.Enabled)
 	{
 		InitializeStructuredCache(ServerOptions);
@@ -287,7 +296,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 		m_Http->RegisterService(*m_HttpWorkspacesService);
 	}
 
-	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot);
+	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
 
 	if (m_FrontendService)
 	{
@@ -306,12 +315,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 			ObjCfg.Buckets.push_back(std::move(NewBucket));
 		}
 
-		m_ObjStoreService = std::make_unique<HttpObjectStoreService>(std::move(ObjCfg));
+		m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg));
 		m_Http->RegisterService(*m_ObjStoreService);
 	}
 
+	if (ServerOptions.BuildStoreConfig.Enabled)
+	{
+		m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore);
+		m_Http->RegisterService(*m_BuildStoreService);
+	}
+
 #if ZEN_WITH_VFS
-	m_VfsService = std::make_unique<VfsService>();
+	m_VfsService = std::make_unique<VfsService>(m_StatusService);
 	m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore));
 	m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore));
 	m_Http->RegisterService(*m_VfsService);
@@ -327,6 +342,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 							   .Interval				= std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds),
 							   .MaxCacheDuration		= std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds),
 							   .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds),
+							   .MaxBuildStoreDuration	= std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds),
 							   .CollectSmallObjects		= ServerOptions.GcConfig.CollectSmallObjects,
 							   .Enabled					= ServerOptions.GcConfig.Enabled,
 							   .DiskReserveSize			= ServerOptions.GcConfig.DiskReserveSize,
@@ -347,6 +363,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 										   m_CacheStore.Get(),
 										   m_CidStore.get(),
 										   m_ProjectStore,
+										   m_BuildStore.get(),
 										   HttpAdminService::LogPaths{.AbsLogPath	= ServerOptions.AbsLogFile,
 																	  .HttpLogPath	= ServerOptions.DataDir / "logs" / "http.log",
 																	  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
@@ -418,7 +435,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 				if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION)
 				{
 					std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch";
-					if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath))
+					if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath))
 					{
 						ZEN_INFO(
 							"Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating "
@@ -467,7 +484,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 			{
 				ZEN_INFO("Deleting '{}'", DirEntry.path());
 
-				std::filesystem::remove_all(DirEntry.path(), Ec);
+				DeleteDirectories(DirEntry.path(), Ec);
 
 				if (Ec)
 				{
@@ -530,10 +547,26 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 	Config.AllowAutomaticCreationOfNamespaces = true;
 	Config.Logging							  = {.EnableWriteLog  = ServerOptions.StructuredCacheConfig.WriteLogEnabled,
 												 .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled};
-	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold,
-	Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes		  = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes;
-	Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds		  = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds;
-	Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds			  = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds;
+	for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs)
+	{
+		const std::string&					   BucketName	   = It.first;
+		const ZenStructuredCacheBucketConfig&  ZenBucketConfig = It.second;
+		ZenCacheDiskLayer::BucketConfiguration BucketConfig	   = {.MaxBlockSize			 = ZenBucketConfig.MaxBlockSize,
+																  .PayloadAlignment		 = ZenBucketConfig.PayloadAlignment,
+																  .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold,
+																  .LargeObjectThreshold	 = ZenBucketConfig.LargeObjectThreshold};
+		Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig);
+	}
+	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize,
+	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.PayloadAlignment =
+		ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment,
+	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold =
+		ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold,
+	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold =
+		ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold,
+	Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes;
+	Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds	= ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds;
+	Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds		= ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds;
 
 	if (ServerOptions.IsDedicated)
 	{
@@ -801,6 +834,9 @@ ZenServer::Cleanup()
 		m_ObjStoreService.reset();
 		m_FrontendService.reset();
 
+		m_BuildStoreService.reset();
+		m_BuildStore = {};
+
 		m_StructuredCacheService.reset();
 		m_UpstreamService.reset();
 		m_UpstreamCache.reset();
@@ -895,7 +931,7 @@ ZenServer::CheckStateMarker()
 	std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker";
 	try
 	{
-		if (!std::filesystem::exists(StateMarkerPath))
+		if (!IsFile(StateMarkerPath))
 		{
 			ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath);
 			RequestExit(1);
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 80054dc35..5cfa04ba1 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -25,6 +25,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #include <zenstore/cache/structuredcachestore.h>
 #include <zenstore/gc.h>
 #include "admin/admin.h"
+#include "buildstore/httpbuildstore.h"
 #include "cache/httpstructuredcache.h"
 #include "diag/diagsvcs.h"
 #include "frontend/frontend.h"
@@ -127,6 +128,8 @@ private:
 	Ref<ZenCacheStore>				  m_CacheStore;
 	std::unique_ptr<OpenProcessCache> m_OpenProcessCache;
 	HttpTestService					  m_TestService;
+	std::unique_ptr<BuildStore>		  m_BuildStore;
+
 #if ZEN_WITH_TESTS
 	HttpTestingService m_TestingService;
 #endif
@@ -140,6 +143,7 @@ private:
 	HttpHealthService							m_HealthService;
 	std::unique_ptr<HttpFrontendService>		m_FrontendService;
 	std::unique_ptr<HttpObjectStoreService>		m_ObjStoreService;
+	std::unique_ptr<HttpBuildStoreService>		m_BuildStoreService;
 	std::unique_ptr<VfsService>					m_VfsService;
 	std::unique_ptr<JobQueue>					m_JobQueue;
 	std::unique_ptr<HttpAdminService>			m_AdminService;
diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp
index 32fc69783..6ae0b84b0 100644
--- a/src/zenstore-test/zenstore-test.cpp
+++ b/src/zenstore-test/zenstore-test.cpp
@@ -2,6 +2,7 @@
 
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 #include <zenstore/zenstore.h>
 
 #include <zencore/memory/newdelete.h>
@@ -22,6 +23,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 	zen::IgnoreChildSignals();
 #	endif
 
+	zen::TraceInit("zenstore-test");
 	zen::logging::InitializeLogging();
 	zen::MaximizeOpenFileCount();
 
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index e976c061d..7b56c64bd 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -70,7 +70,7 @@ BlockStoreFile::Open()
 			return false;
 		}
 		ZEN_WARN("Failed to open cas block '{}', reason: '{}', retries left: {}.", m_Path, Ec.message(), RetriesLeft);
-		Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+		Sleep(100 + (3 - RetriesLeft) * 100);  // Total 600 ms
 		RetriesLeft--;
 		return true;
 	});
@@ -85,7 +85,7 @@ BlockStoreFile::Create(uint64_t InitialSize)
 	ZEN_TRACE_CPU("BlockStoreFile::Create");
 
 	auto ParentPath = m_Path.parent_path();
-	if (!std::filesystem::is_directory(ParentPath))
+	if (!IsDir(ParentPath))
 	{
 		CreateDirectories(ParentPath);
 	}
@@ -153,14 +153,28 @@ void
 BlockStoreFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
 {
 	ZEN_TRACE_CPU("BlockStoreFile::Write");
+#if ZEN_BUILD_DEBUG
+	if (uint64_t CachedFileSize = m_CachedFileSize.load(); CachedFileSize > 0)
+	{
+		ZEN_ASSERT(FileOffset + Size <= CachedFileSize);
+	}
+#endif	// ZEN_BUILD_DEBUG
 	m_File.Write(Data, Size, FileOffset);
 }
 
 void
-BlockStoreFile::Flush()
+BlockStoreFile::Flush(uint64_t FinalSize)
 {
 	ZEN_TRACE_CPU("BlockStoreFile::Flush");
 	m_File.Flush();
+	if (FinalSize != (uint64_t)-1)
+	{
+		uint64_t ExpectedSize = 0;
+		if (!m_CachedFileSize.compare_exchange_weak(ExpectedSize, FinalSize))
+		{
+			ZEN_ASSERT(m_CachedFileSize.load() == FinalSize);
+		}
+	}
 }
 
 BasicFile&
@@ -215,7 +229,7 @@ IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::p
 	}
 	if (MetaWriteTime < BlockWriteTime)
 	{
-		std::filesystem::remove(MetaPath, Ec);
+		RemoveFile(MetaPath, Ec);
 		return false;
 	}
 	return true;
@@ -239,7 +253,7 @@ BlockStoreFile::MetaSize() const
 	if (IsMetaDataValid(m_Path, MetaPath))
 	{
 		std::error_code DummyEc;
-		if (uint64_t Size = std::filesystem::file_size(MetaPath, DummyEc); !DummyEc)
+		if (uint64_t Size = FileSizeFromPath(MetaPath, DummyEc); !DummyEc)
 		{
 			return Size;
 		}
@@ -252,7 +266,7 @@ BlockStoreFile::RemoveMeta()
 {
 	std::filesystem::path MetaPath = GetMetaPath();
 	std::error_code		  DummyEc;
-	std::filesystem::remove(MetaPath, DummyEc);
+	RemoveFile(MetaPath, DummyEc);
 }
 
 std::filesystem::path
@@ -272,6 +286,14 @@ BlockStore::BlockStore()
 
 BlockStore::~BlockStore()
 {
+	try
+	{
+		Close();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("~BlockStore() failed with: ", Ex.what());
+	}
 }
 
 void
@@ -291,8 +313,9 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 	m_MaxBlockSize	 = MaxBlockSize;
 	m_MaxBlockCount	 = MaxBlockCount;
 
-	if (std::filesystem::is_directory(m_BlocksBasePath))
+	if (IsDir(m_BlocksBasePath))
 	{
+		std::vector<std::filesystem::path> EmptyBlockFiles;
 		uint32_t						   NextBlockIndex = 0;
 		std::vector<std::filesystem::path> FoldersToScan;
 		FoldersToScan.push_back(m_BlocksBasePath);
@@ -320,6 +343,12 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 					{
 						continue;
 					}
+					if (Entry.file_size() == 0)
+					{
+						EmptyBlockFiles.push_back(Path);
+						continue;
+					}
+
 					Ref<BlockStoreFile> BlockFile{new BlockStoreFile(Path)};
 					BlockFile->Open();
 					m_TotalSize.fetch_add(BlockFile->TotalSize(), std::memory_order::relaxed);
@@ -333,6 +362,17 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 			}
 			++FolderOffset;
 		}
+
+		for (const std::filesystem::path& EmptyBlockFile : EmptyBlockFiles)
+		{
+			std::error_code Ec;
+			RemoveFile(EmptyBlockFile, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Unable to remove empty block file {}. Reason: {}", EmptyBlockFile, Ec.message());
+			}
+		}
+
 		m_WriteBlockIndex.store(NextBlockIndex, std::memory_order_release);
 	}
 	else
@@ -341,7 +381,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 	}
 }
 
-void
+BlockStore::BlockIndexSet
 BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 {
 	ZEN_MEMSCOPE(GetBlocksTag());
@@ -349,8 +389,8 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 
 	RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
 
-	tsl::robin_set<uint32_t> MissingBlocks;
-	tsl::robin_set<uint32_t> DeleteBlocks;
+	BlockIndexSet MissingBlocks;
+	BlockIndexSet DeleteBlocks;
 	DeleteBlocks.reserve(m_ChunkBlocks.size());
 	for (auto It : m_ChunkBlocks)
 	{
@@ -369,13 +409,6 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 			MissingBlocks.insert(BlockIndex);
 		}
 	}
-	for (std::uint32_t BlockIndex : MissingBlocks)
-	{
-		std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex);
-		Ref<BlockStoreFile>	  NewBlockFile(new BlockStoreFile(BlockPath));
-		NewBlockFile->Create(0);
-		m_ChunkBlocks[BlockIndex] = NewBlockFile;
-	}
 	for (std::uint32_t BlockIndex : DeleteBlocks)
 	{
 		std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex);
@@ -386,6 +419,7 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 		}
 		m_ChunkBlocks.erase(BlockIndex);
 	}
+	return MissingBlocks;
 }
 
 BlockStore::BlockEntryCountMap
@@ -500,7 +534,7 @@ BlockStore::GetFreeBlockIndex(uint32_t ProbeIndex, RwLock::ExclusiveLockScope&,
 		{
 			OutBlockPath = GetBlockPath(m_BlocksBasePath, ProbeIndex);
 			std::error_code Ec;
-			bool			Exists = std::filesystem::exists(OutBlockPath, Ec);
+			bool			Exists = IsFile(OutBlockPath, Ec);
 			if (Ec)
 			{
 				ZEN_WARN("Failed to probe existence of file '{}' when trying to allocate a new block. Reason: '{}'",
@@ -540,7 +574,7 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, cons
 	{
 		if (m_WriteBlock)
 		{
-			m_WriteBlock->Flush();
+			m_WriteBlock->Flush(m_CurrentInsertOffset);
 			m_WriteBlock = nullptr;
 		}
 
@@ -578,7 +612,7 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, cons
 }
 
 void
-BlockStore::WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback)
+BlockStore::WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback)
 {
 	ZEN_MEMSCOPE(GetBlocksTag());
 	ZEN_TRACE_CPU("BlockStore::WriteChunks");
@@ -674,6 +708,27 @@ BlockStore::WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const Wri
 	}
 }
 
+bool
+BlockStore::HasChunk(const BlockStoreLocation& Location) const
+{
+	ZEN_TRACE_CPU("BlockStore::TryGetChunk");
+	RwLock::SharedLockScope InsertLock(m_InsertLock);
+	if (auto BlockIt = m_ChunkBlocks.find(Location.BlockIndex); BlockIt != m_ChunkBlocks.end())
+	{
+		if (const Ref<BlockStoreFile>& Block = BlockIt->second; Block)
+		{
+			InsertLock.ReleaseNow();
+
+			const uint64_t BlockSize = Block->FileSize();
+			if (Location.Offset + Location.Size <= BlockSize)
+			{
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
 IoBuffer
 BlockStore::TryGetChunk(const BlockStoreLocation& Location) const
 {
@@ -706,7 +761,7 @@ BlockStore::Flush(bool ForceNewBlock)
 		{
 			if (m_WriteBlock)
 			{
-				m_WriteBlock->Flush();
+				m_WriteBlock->Flush(m_CurrentInsertOffset);
 			}
 			m_WriteBlock		  = nullptr;
 			m_CurrentInsertOffset = 0;
@@ -735,6 +790,8 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation>   ChunkLocations,
 		return true;
 	}
 
+	ZEN_ASSERT(ChunkLocations.size() >= InChunkIndexes.size());
+
 	if (LargeSizeLimit == 0)
 	{
 		LargeSizeLimit = DefaultIterateSmallChunkWindowSize;
@@ -746,7 +803,10 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation>   ChunkLocations,
 
 	IterateSmallChunkWindowSize = Min((LargeSizeLimit + IterateSmallChunkMaxGapSize) * ChunkLocations.size(), IterateSmallChunkWindowSize);
 
-	uint32_t			BlockIndex = ChunkLocations[InChunkIndexes[0]].BlockIndex;
+	const size_t FirstLocationIndex = InChunkIndexes[0];
+	ZEN_ASSERT(FirstLocationIndex < ChunkLocations.size());
+
+	const uint32_t		BlockIndex = ChunkLocations[FirstLocationIndex].BlockIndex;
 	std::vector<size_t> ChunkIndexes(InChunkIndexes.begin(), InChunkIndexes.end());
 	std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&](size_t IndexA, size_t IndexB) -> bool {
 		return ChunkLocations[IndexA].Offset < ChunkLocations[IndexB].Offset;
@@ -756,8 +816,9 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation>   ChunkLocations,
 						 IterateSmallChunkWindowSize,
 						 IterateSmallChunkMaxGapSize,
 						 &ChunkLocations](uint64_t BlockFileSize, std::span<const size_t> ChunkIndexes, size_t StartIndexOffset) -> size_t {
-		size_t					  ChunkCount	= 0;
-		size_t					  StartIndex	= ChunkIndexes[StartIndexOffset];
+		size_t ChunkCount = 0;
+		size_t StartIndex = ChunkIndexes[StartIndexOffset];
+		ZEN_ASSERT(StartIndex < ChunkLocations.size());
 		const BlockStoreLocation& StartLocation = ChunkLocations[StartIndex];
 		uint64_t				  StartOffset	= StartLocation.Offset;
 		uint64_t				  LastEnd		= StartOffset + StartLocation.Size;
@@ -810,22 +871,26 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation>   ChunkLocations,
 		ZEN_ASSERT(BlockFile);
 		InsertLock.ReleaseNow();
 
+		const size_t BlockSize = BlockFile->FileSize();
+
 		IoBuffer ReadBuffer;
 		void*	 BufferBase = nullptr;
 
 		size_t LocationIndexOffset = 0;
 		while (LocationIndexOffset < ChunkIndexes.size())
 		{
-			size_t					  ChunkIndex	= ChunkIndexes[LocationIndexOffset];
+			size_t ChunkIndex = ChunkIndexes[LocationIndexOffset];
+			ZEN_ASSERT(ChunkIndex < ChunkLocations.size());
 			const BlockStoreLocation& FirstLocation = ChunkLocations[ChunkIndex];
+			ZEN_ASSERT(FirstLocation.BlockIndex == BlockIndex);
 
-			const size_t BlockSize	= BlockFile->FileSize();
 			const size_t RangeCount = GetNextRange(BlockSize, ChunkIndexes, LocationIndexOffset);
 			if (RangeCount > 1)
 			{
-				size_t					  LastChunkIndex = ChunkIndexes[LocationIndexOffset + RangeCount - 1];
-				const BlockStoreLocation& LastLocation	 = ChunkLocations[LastChunkIndex];
-				uint64_t				  Size			 = LastLocation.Offset + LastLocation.Size - FirstLocation.Offset;
+				size_t LastChunkIndex = ChunkIndexes[LocationIndexOffset + RangeCount - 1];
+				ZEN_ASSERT(LastChunkIndex < ChunkLocations.size());
+				const BlockStoreLocation& LastLocation = ChunkLocations[LastChunkIndex];
+				uint64_t				  Size		   = LastLocation.Offset + LastLocation.Size - FirstLocation.Offset;
 				if (ReadBuffer.GetSize() < Size)
 				{
 					ReadBuffer = IoBuffer(Min(Size * 2, IterateSmallChunkWindowSize));
@@ -834,8 +899,9 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation>   ChunkLocations,
 				BlockFile->Read(BufferBase, Size, FirstLocation.Offset);
 				for (size_t RangeIndex = 0; RangeIndex < RangeCount; ++RangeIndex)
 				{
-					size_t					  NextChunkIndex = ChunkIndexes[LocationIndexOffset + RangeIndex];
-					const BlockStoreLocation& ChunkLocation	 = ChunkLocations[NextChunkIndex];
+					size_t NextChunkIndex = ChunkIndexes[LocationIndexOffset + RangeIndex];
+					ZEN_ASSERT(NextChunkIndex < ChunkLocations.size());
+					const BlockStoreLocation& ChunkLocation = ChunkLocations[NextChunkIndex];
 					if (ChunkLocation.Size == 0 || ((ChunkLocation.Offset + ChunkLocation.Size) > BlockSize))
 					{
 						ZEN_LOG_SCOPE("chunk [{},{}] out of bounds (block #{} file size = {})",
@@ -958,6 +1024,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 
 	uint32_t		 NewBlockIndex = 0;
 	MovedChunksArray MovedChunks;
+	ChunkIndexArray	 ScrubbedChunks;
 
 	uint64_t AddedSize	 = 0;
 	uint64_t RemovedSize = 0;
@@ -986,14 +1053,16 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 
 	auto ReportChanges = [&]() -> bool {
 		bool Continue = true;
-		if (!MovedChunks.empty() || RemovedSize > 0)
+		if (!MovedChunks.empty() || !ScrubbedChunks.empty() || RemovedSize > 0)
 		{
-			Continue = ChangeCallback(MovedChunks, RemovedSize > AddedSize ? RemovedSize - AddedSize : 0);
+			Continue = ChangeCallback(MovedChunks, ScrubbedChunks, RemovedSize > AddedSize ? RemovedSize - AddedSize : 0);
 			DeletedSize += RemovedSize;
+			m_TotalSize.fetch_add(AddedSize);
 			RemovedSize = 0;
 			AddedSize	= 0;
 			MovedCount += MovedChunks.size();
 			MovedChunks.clear();
+			ScrubbedChunks.clear();
 		}
 		return Continue;
 	};
@@ -1022,6 +1091,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 						 LogPrefix,
 						 m_BlocksBasePath,
 						 BlockIndex);
+				ScrubbedChunks.insert(ScrubbedChunks.end(), KeepChunkIndexes.begin(), KeepChunkIndexes.end());
 				return true;
 			}
 			if (!It->second)
@@ -1030,6 +1100,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 						 LogPrefix,
 						 m_BlocksBasePath,
 						 BlockIndex);
+				ScrubbedChunks.insert(ScrubbedChunks.end(), KeepChunkIndexes.begin(), KeepChunkIndexes.end());
 				return true;
 			}
 			OldBlockFile = It->second;
@@ -1051,11 +1122,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 			std::sort(SortedChunkIndexes.begin(), SortedChunkIndexes.end(), [&ChunkLocations](size_t Lhs, size_t Rhs) {
 				return ChunkLocations[Lhs].Offset < ChunkLocations[Rhs].Offset;
 			});
-			BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(65536u, OldBlockSize));
+			BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(256u * 1024u, OldBlockSize));
 
-			uint64_t			 WrittenBytesToBlock = 0;
-			uint64_t			 MovedFromBlock		 = 0;
-			std::vector<uint8_t> Chunk;
+			uint64_t			 MovedFromBlock = 0;
+			std::vector<uint8_t> ChunkBuffer;
 			for (const size_t& ChunkIndex : SortedChunkIndexes)
 			{
 				const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
@@ -1070,19 +1140,29 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 						ChunkLocation.Size,
 						OldBlockFile->GetPath(),
 						OldBlockSize);
+					ScrubbedChunks.push_back(ChunkIndex);
 					continue;
 				}
 
-				Chunk.resize(ChunkLocation.Size);
-				SourceFileBuffer.Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
+				MemoryView ChunkView = SourceFileBuffer.MakeView(ChunkLocation.Size, ChunkLocation.Offset);
+				if (ChunkView.GetSize() != ChunkLocation.Size)
+				{
+					ChunkBuffer.resize(ChunkLocation.Size);
+					SourceFileBuffer.Read(ChunkBuffer.data(), ChunkLocation.Size, ChunkLocation.Offset);
+					ChunkView = MemoryView(ChunkBuffer.data(), ChunkLocation.Size);
+				}
 
-				if ((WriteOffset + Chunk.size()) > m_MaxBlockSize)
+				if ((WriteOffset + ChunkView.GetSize()) > m_MaxBlockSize)
 				{
-					TargetFileBuffer.reset();
+					if (TargetFileBuffer)
+					{
+						TargetFileBuffer->Flush();
+						TargetFileBuffer.reset();
+					}
 					if (NewBlockFile)
 					{
 						ZEN_ASSERT_SLOW(NewBlockFile->IsOpen());
-						NewBlockFile->Flush();
+						NewBlockFile->Flush(WriteOffset);
 						uint64_t NewBlockSize = NewBlockFile->FileSize();
 						MovedSize += NewBlockSize;
 						NewBlockFile = nullptr;
@@ -1161,22 +1241,23 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 								 NiceBytes(Space.Free + ReclaimedSpace));
 					}
 					NewBlockFile->Create(m_MaxBlockSize);
-					NewBlockIndex = NextBlockIndex;
-					WriteOffset	  = 0;
-					AddedSize += WrittenBytesToBlock;
-					WrittenBytesToBlock = 0;
-					TargetFileBuffer	= std::make_unique<BasicFileWriter>(NewBlockFile->GetBasicFile(), Min(65536u, m_MaxBlockSize));
+					NewBlockIndex	 = NextBlockIndex;
+					WriteOffset		 = 0;
+					TargetFileBuffer = std::make_unique<BasicFileWriter>(NewBlockFile->GetBasicFile(), Min(256u * 1024u, m_MaxBlockSize));
 				}
 
-				TargetFileBuffer->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
+				const uint64_t OldWriteOffset = WriteOffset;
+				WriteOffset					  = TargetFileBuffer->AlignTo(PayloadAlignment);
+
+				TargetFileBuffer->Write(ChunkView.GetData(), ChunkLocation.Size, WriteOffset);
 				MovedChunks.push_back(
 					{ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
-				WrittenBytesToBlock = WriteOffset + ChunkLocation.Size;
+				WriteOffset += ChunkLocation.Size;
 
 				MovedFromBlock += RoundUp(ChunkLocation.Offset + ChunkLocation.Size, PayloadAlignment) - ChunkLocation.Offset;
-				WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
+				uint64_t WrittenBytes = WriteOffset - OldWriteOffset;
+				AddedSize += WrittenBytes;
 			}
-			AddedSize += WrittenBytesToBlock;
 			ZEN_INFO("{}moved {} chunks ({}) from '{}' to new block, freeing {}",
 					 LogPrefix,
 					 KeepChunkIndexes.size(),
@@ -1209,10 +1290,16 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 		return true;
 	});
 
+	if (TargetFileBuffer)
+	{
+		TargetFileBuffer->Flush();
+		TargetFileBuffer.reset();
+	}
+
 	if (NewBlockFile)
 	{
 		ZEN_ASSERT_SLOW(NewBlockFile->IsOpen());
-		NewBlockFile->Flush();
+		NewBlockFile->Flush(WriteOffset);
 		uint64_t NewBlockSize = NewBlockFile->FileSize();
 		MovedSize += NewBlockSize;
 		NewBlockFile = nullptr;
@@ -1343,6 +1430,8 @@ TEST_CASE("blockstore.blockfile")
 		CHECK(std::string(Boop) == "boop");
 		File1.Flush();
 		CHECK(File1.FileSize() == 10);
+		File1.Flush(10);
+		CHECK(File1.FileSize() == 10);
 	}
 	{
 		BlockStoreFile File1(RootDirectory / "1");
@@ -1375,14 +1464,14 @@ TEST_CASE("blockstore.blockfile")
 			BoopChunk = File1.GetChunk(5, 5);
 		}
 
-		CHECK(std::filesystem::exists(RootDirectory / "1"));
+		CHECK(IsFile(RootDirectory / "1"));
 
 		const char* Data = static_cast<const char*>(DataChunk.GetData());
 		CHECK(std::string(Data) == "data");
 		const char* Boop = static_cast<const char*>(BoopChunk.GetData());
 		CHECK(std::string(Boop) == "boop");
 	}
-	CHECK(std::filesystem::exists(RootDirectory / "1"));
+	CHECK(IsFile(RootDirectory / "1"));
 
 	{
 		IoBuffer DataChunk;
@@ -1401,7 +1490,7 @@ TEST_CASE("blockstore.blockfile")
 		const char* Boop = static_cast<const char*>(BoopChunk.GetData());
 		CHECK(std::string(Boop) == "boop");
 	}
-	CHECK(!std::filesystem::exists(RootDirectory / "1"));
+	CHECK(!IsFile(RootDirectory / "1"));
 }
 
 namespace blockstore::impl {
@@ -1800,7 +1889,7 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray&, uint64_t) {
+			[&](const BlockStore::MovedChunksArray&, const BlockStore::ChunkIndexArray&, uint64_t) {
 				CHECK(false);
 				return true;
 			},
@@ -1825,9 +1914,10 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+			[&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) {
 				RemovedSize += Removed;
 				CHECK(Moved.empty());
+				CHECK(Scrubbed.empty());
 				return true;
 			},
 			[]() { return 0; });
@@ -1850,9 +1940,10 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+			[&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) {
 				RemovedSize += Removed;
 				CHECK(Moved.empty());
+				CHECK(Scrubbed.empty());
 				return true;
 			},
 			[]() { return 0; });
@@ -1860,7 +1951,7 @@ TEST_CASE("blockstore.compact.blocks")
 		CHECK_LE(Store.TotalSize(), 1088);
 		CHECK_GT(Store.TotalSize(), 0);
 	}
-	SUBCASE("keep everthing")
+	SUBCASE("keep everything")
 	{
 		Store.Flush(true);
 
@@ -1873,7 +1964,7 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray&, uint64_t) {
+			[&](const BlockStore::MovedChunksArray&, const BlockStore::ChunkIndexArray&, uint64_t) {
 				CHECK(false);
 				return true;
 			},
@@ -1904,8 +1995,9 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+			[&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) {
 				CHECK(Moved.empty());
+				CHECK(Scrubbed.empty());
 				RemovedSize += Removed;
 				return true;
 			},
@@ -1939,7 +2031,8 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+			[&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) {
+				CHECK(Scrubbed.empty());
 				for (const auto& Move : Moved)
 				{
 					const BlockStoreLocation& OldLocation = State.GetLocation(Move.first);
@@ -2016,7 +2109,8 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+			[&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) {
+				CHECK(Scrubbed.empty());
 				for (const auto& Move : Moved)
 				{
 					const BlockStoreLocation& OldLocation = State.GetLocation(Move.first);
@@ -2051,6 +2145,42 @@ TEST_CASE("blockstore.compact.blocks")
 		}
 		CHECK_LT(Store.TotalSize(), PreSize);
 	}
+	SUBCASE("scrub")
+	{
+		Store.Flush(true);
+
+		BlockStoreCompactState State;
+		for (const BlockStoreLocation& Location : ChunkLocations)
+		{
+			State.IncludeBlock(Location.BlockIndex);
+			CHECK(State.AddKeepLocation(Location));
+		}
+		State.IncludeBlock(0);
+		State.IncludeBlock(999);
+		std::vector<size_t> ExpectedScrubbedIndexes;
+		ExpectedScrubbedIndexes.push_back(ChunkLocations.size() + 0);
+		State.AddKeepLocation(BlockStoreLocation{.BlockIndex = 0, .Offset = 2000, .Size = 322});
+		ExpectedScrubbedIndexes.push_back(ChunkLocations.size() + 1);
+		State.AddKeepLocation(BlockStoreLocation{.BlockIndex = 0, .Offset = 10, .Size = 3220});
+		ExpectedScrubbedIndexes.push_back(ChunkLocations.size() + 2);
+		State.AddKeepLocation(BlockStoreLocation{.BlockIndex = 999, .Offset = 2, .Size = 40});
+
+		std::vector<size_t> ScrubbedIndexes;
+
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray&, const BlockStore::ChunkIndexArray& ScrubbedArray, uint64_t) {
+				ScrubbedIndexes.insert(ScrubbedIndexes.end(), ScrubbedArray.begin(), ScrubbedArray.end());
+				return true;
+			},
+			[]() {
+				CHECK(false);
+				return 0;
+			});
+		std::sort(ScrubbedIndexes.begin(), ScrubbedIndexes.end());
+		CHECK_EQ(ExpectedScrubbedIndexes, ScrubbedIndexes);
+	}
 }
 
 #endif
diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp
new file mode 100644
index 000000000..20dc55bca
--- /dev/null
+++ b/src/zenstore/buildstore/buildstore.cpp
@@ -0,0 +1,2053 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenstore/buildstore/buildstore.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/memory/llm.h>
+#include <zencore/scopeguard.h>
+#include <zencore/trace.h>
+#include <zencore/workthreadpool.h>
+#include <zenutil/parallelwork.h>
+
+#include <zencore/uid.h>
+#include <zencore/xxhash.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#if ZEN_WITH_TESTS
+#	include <zencore/compactbinarybuilder.h>
+#	include <zencore/compress.h>
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#	include <zenutil/workerpools.h>
+#endif	// ZEN_WITH_TESTS
+
+namespace zen {
+const FLLMTag&
+GetBuildstoreTag()
+{
+	static FLLMTag _("store", FLLMTag("builds"));
+
+	return _;
+}
+
+using namespace std::literals;
+
+namespace blobstore::impl {
+
+	const std::string BaseName			  = "builds";
+	const std::string ManifestExtension	  = ".cbo";
+	const char*		  IndexExtension	  = ".uidx";
+	const char*		  LogExtension		  = ".slog";
+	const char*		  AccessTimeExtension = ".zacs";
+
+	const uint32_t ManifestVersion = (1 << 16) | (0 << 8) | (0);
+
+	std::filesystem::path GetManifestPath(const std::filesystem::path& RootDirectory)
+	{
+		return RootDirectory / (BaseName + ManifestExtension);
+	}
+
+	std::filesystem::path GetBlobIndexPath(const std::filesystem::path& RootDirectory)
+	{
+		return RootDirectory / (BaseName + IndexExtension);
+	}
+
+	std::filesystem::path GetBlobLogPath(const std::filesystem::path& RootDirectory) { return RootDirectory / (BaseName + LogExtension); }
+
+	std::filesystem::path GetMetaIndexPath(const std::filesystem::path& RootDirectory)
+	{
+		return RootDirectory / (BaseName + "_meta" + IndexExtension);
+	}
+
+	std::filesystem::path GetMetaLogPath(const std::filesystem::path& RootDirectory)
+	{
+		return RootDirectory / (BaseName + "_meta" + LogExtension);
+	}
+
+	std::filesystem::path GetAccessTimesPath(const std::filesystem::path& RootDirectory)
+	{
+		return RootDirectory / (BaseName + AccessTimeExtension);
+	}
+
+	struct AccessTimeRecord
+	{
+		IoHash		  Key;
+		std::uint32_t SecondsSinceEpoch = 0;
+	};
+
+	static_assert(sizeof(AccessTimeRecord) == 24);
+
+#pragma pack(push)
+#pragma pack(1)
+	struct AccessTimesHeader
+	{
+		static constexpr uint32_t ExpectedMagic	 = 0x7363617a;	// 'zacs';
+		static constexpr uint32_t CurrentVersion = 1;
+		static constexpr uint64_t DataAlignment	 = 8;
+
+		uint32_t Magic			 = ExpectedMagic;
+		uint32_t Version		 = CurrentVersion;
+		uint32_t AccessTimeCount = 0;
+		uint32_t Checksum		 = 0;
+
+		static uint32_t ComputeChecksum(const AccessTimesHeader& Header)
+		{
+			return XXH32(&Header.Magic, sizeof(AccessTimesHeader) - sizeof(uint32_t), 0xC0C0'BABA);
+		}
+	};
+#pragma pack(pop)
+
+	static_assert(sizeof(AccessTimesHeader) == 16);
+
+}  // namespace blobstore::impl
+
+BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc)
+: m_Log(logging::Get("builds"))
+, m_Config(Config)
+, m_Gc(Gc)
+, m_LargeBlobStore(m_Gc)
+, m_SmallBlobStore(Gc)
+, m_MetadataBlockStore()
+{
+	ZEN_TRACE_CPU("BuildStore::BuildStore");
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+	try
+	{
+		bool	   IsNew = true;
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			ZEN_INFO("{} build store at {} in {}",
+					 IsNew ? "Initialized" : "Read",
+					 m_Config.RootDirectory,
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		std::filesystem::path BlobLogPath	  = blobstore::impl::GetBlobLogPath(Config.RootDirectory);
+		std::filesystem::path MetaLogPath	  = blobstore::impl::GetMetaLogPath(Config.RootDirectory);
+		std::filesystem::path ManifestPath	  = blobstore::impl::GetManifestPath(Config.RootDirectory);
+		std::filesystem::path AccessTimesPath = blobstore::impl::GetAccessTimesPath(Config.RootDirectory);
+		if (IsFile(ManifestPath) && IsFile(BlobLogPath) && IsFile(MetaLogPath))
+		{
+			IsNew = false;
+		}
+
+		if (!IsNew)
+		{
+			RwLock::ExclusiveLockScope Lock(m_Lock);
+
+			CbObject ManifestReader = LoadCompactBinaryObject(ReadFile(ManifestPath).Flatten());
+			Oid		 ManifestId		= ManifestReader["id"].AsObjectId();
+			uint32_t Version		= ManifestReader["version"].AsUInt32();
+			DateTime CreationDate	= ManifestReader["createdAt"].AsDateTime();
+			ZEN_UNUSED(CreationDate);
+			if (ManifestId == Oid::Zero || Version != blobstore::impl::ManifestVersion)
+			{
+				ZEN_WARN("Invalid manifest at {}, wiping state", ManifestPath);
+				IsNew = true;
+			}
+			else
+			{
+				m_BlobLogFlushPosition = ReadPayloadLog(Lock, BlobLogPath, 0);
+				m_MetaLogFlushPosition = ReadMetadataLog(Lock, MetaLogPath, 0);
+				if (IsFile(AccessTimesPath))
+				{
+					ReadAccessTimes(Lock, AccessTimesPath);
+				}
+			}
+		}
+
+		if (IsNew)
+		{
+			CleanDirectory(Config.RootDirectory, false);
+			CbObjectWriter ManifestWriter;
+			ManifestWriter.AddObjectId("id", Oid::NewOid());
+			ManifestWriter.AddInteger("version", blobstore::impl::ManifestVersion);
+			ManifestWriter.AddDateTime("createdAt", DateTime::Now());
+			TemporaryFile::SafeWriteFile(ManifestPath, ManifestWriter.Save().GetBuffer().AsIoBuffer());
+		}
+		m_LargeBlobStore.Initialize(Config.RootDirectory / "file_cas", IsNew);
+		m_SmallBlobStore.Initialize(Config.RootDirectory,
+									"blob_cas",
+									m_Config.SmallBlobBlockStoreMaxBlockSize,
+									m_Config.SmallBlobBlockStoreAlignement,
+									IsNew);
+		m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20);
+
+		BlockStore::BlockIndexSet KnownBlocks;
+		for (const BlobEntry& Blob : m_BlobEntries)
+		{
+			if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex)
+			{
+				const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex];
+				KnownBlocks.insert(Metadata.Location.BlockIndex);
+			}
+		}
+		BlockStore::BlockIndexSet MissingBlocks = m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
+
+		m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite);
+		m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite);
+
+		if (!MissingBlocks.empty())
+		{
+			std::vector<MetadataDiskEntry> MissingMetadatas;
+			for (auto& It : m_BlobLookup)
+			{
+				const IoHash&	 BlobHash	   = It.first;
+				const BlobIndex	 ReadBlobIndex = It.second;
+				const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+				if (ReadBlobEntry.Metadata)
+				{
+					const MetadataEntry& MetaData = m_MetadataEntries[ReadBlobEntry.Metadata];
+					if (MissingBlocks.contains(MetaData.Location.BlockIndex))
+					{
+						MissingMetadatas.push_back(
+							MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = BlobHash});
+						MissingMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone;
+						m_MetadataEntries[ReadBlobEntry.Metadata] = {};
+						m_BlobEntries[ReadBlobIndex].Metadata	  = {};
+					}
+				}
+			}
+			ZEN_ASSERT(!MissingMetadatas.empty());
+
+			for (const MetadataDiskEntry& Entry : MissingMetadatas)
+			{
+				auto It = m_BlobLookup.find(Entry.BlobHash);
+				ZEN_ASSERT(It != m_BlobLookup.end());
+
+				const BlobIndex	 ReadBlobIndex = It->second;
+				const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+				if (!ReadBlobEntry.Payload)
+				{
+					m_BlobLookup.erase(It);
+				}
+			}
+			m_MetadatalogFile.Append(MissingMetadatas);
+			CompactState();
+		}
+
+		m_Gc.AddGcReferencer(*this);
+		m_Gc.AddGcReferenceLocker(*this);
+		m_Gc.AddGcStorage(this);
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what());
+		m_Gc.RemoveGcStorage(this);
+		m_Gc.RemoveGcReferenceLocker(*this);
+		m_Gc.RemoveGcReferencer(*this);
+	}
+}
+
+BuildStore::~BuildStore()
+{
+	try
+	{
+		ZEN_TRACE_CPU("BuildStore::~BuildStore");
+		m_Gc.RemoveGcStorage(this);
+		m_Gc.RemoveGcReferenceLocker(*this);
+		m_Gc.RemoveGcReferencer(*this);
+		Flush();
+		m_MetadatalogFile.Close();
+		m_PayloadlogFile.Close();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("~BuildStore() threw exception: {}", Ex.what());
+	}
+}
+
+void
+BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload)
+{
+	ZEN_TRACE_CPU("BuildStore::PutBlob");
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCompressedBinary);
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
+		{
+			const BlobIndex BlobIndex = It->second;
+			if (m_BlobEntries[BlobIndex].Payload)
+			{
+				return;
+			}
+		}
+	}
+
+	uint64_t	 PayloadSize = Payload.GetSize();
+	PayloadEntry Entry;
+	if (Payload.GetSize() > m_Config.SmallBlobBlockStoreMaxBlockEmbedSize)
+	{
+		CasStore::InsertResult Result = m_LargeBlobStore.InsertChunk(Payload, BlobHash);
+		ZEN_UNUSED(Result);
+		Entry = PayloadEntry(PayloadEntry::kStandalone, PayloadSize);
+	}
+	else
+	{
+		CasStore::InsertResult Result = m_SmallBlobStore.InsertChunk(Payload, BlobHash);
+		ZEN_UNUSED(Result);
+		Entry = PayloadEntry(0, PayloadSize);
+	}
+
+	{
+		RwLock::ExclusiveLockScope _(m_Lock);
+		if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
+		{
+			const BlobIndex ExistingBlobIndex = It->second;
+			BlobEntry&		Blob			  = m_BlobEntries[ExistingBlobIndex];
+			if (Blob.Payload)
+			{
+				m_PayloadEntries[Blob.Payload] = Entry;
+			}
+			else
+			{
+				Blob.Payload = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
+				m_PayloadEntries.push_back(Entry);
+			}
+			Blob.LastAccessTime = GcClock::TickCount();
+		}
+		else
+		{
+			PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
+			m_PayloadEntries.push_back(Entry);
+
+			const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
+			// we only remove during GC and compact this then...
+			m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
+			m_BlobLookup.insert({BlobHash, NewBlobIndex});
+		}
+	}
+	m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
+	m_LastAccessTimeUpdateCount++;
+}
+
+IoBuffer
+BuildStore::GetBlob(const IoHash& BlobHash)
+{
+	ZEN_TRACE_CPU("BuildStore::GetBlob");
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+	RwLock::SharedLockScope Lock(m_Lock);
+	if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
+	{
+		const BlobIndex ExistingBlobIndex = It->second;
+		BlobEntry&		Blob			  = m_BlobEntries[ExistingBlobIndex];
+		Blob.LastAccessTime				  = GcClock::TickCount();
+		if (Blob.Payload)
+		{
+			const PayloadEntry& Entry		 = m_PayloadEntries[Blob.Payload];
+			const bool			IsStandalone = (Entry.GetFlags() & PayloadEntry::kStandalone) != 0;
+			Lock.ReleaseNow();
+
+			IoBuffer Chunk;
+			if (IsStandalone)
+			{
+				ZEN_TRACE_CPU("GetLarge");
+				Chunk = m_LargeBlobStore.FindChunk(BlobHash);
+			}
+			else
+			{
+				ZEN_TRACE_CPU("GetSmall");
+				Chunk = m_SmallBlobStore.FindChunk(BlobHash);
+			}
+			if (Chunk)
+			{
+				Chunk.SetContentType(ZenContentType::kCompressedBinary);
+				return Chunk;
+			}
+			else
+			{
+				ZEN_WARN("Inconsistencies in build store, {} is in index but not {}", BlobHash, IsStandalone ? "on disk" : "in block");
+			}
+		}
+	}
+	return {};
+}
+
+std::vector<BuildStore::BlobExistsResult>
+BuildStore::BlobsExists(std::span<const IoHash> BlobHashes)
+{
+	ZEN_TRACE_CPU("BuildStore::BlobsExists");
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+	std::vector<BuildStore::BlobExistsResult> Result;
+	Result.reserve(BlobHashes.size());
+	RwLock::SharedLockScope _(m_Lock);
+	for (const IoHash& BlobHash : BlobHashes)
+	{
+		if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
+		{
+			const BlobIndex ExistingBlobIndex = It->second;
+			BlobEntry&		Blob			  = m_BlobEntries[ExistingBlobIndex];
+			bool			HasPayload		  = !!Blob.Payload;
+			bool			HasMetadata		  = !!Blob.Metadata;
+			Result.push_back(BlobExistsResult{.HasBody = HasPayload, .HasMetadata = HasMetadata});
+		}
+		else
+		{
+			Result.push_back({});
+		}
+	}
+	return Result;
+}
+
+void
+BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas)
+{
+	ZEN_TRACE_CPU("BuildStore::PutMetadatas");
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+	size_t WriteBlobIndex = 0;
+	m_MetadataBlockStore.WriteChunks(MetaDatas, m_Config.MetadataBlockStoreAlignement, [&](std::span<BlockStoreLocation> Locations) {
+		RwLock::ExclusiveLockScope _(m_Lock);
+		for (size_t LocationIndex = 0; LocationIndex < Locations.size(); LocationIndex++)
+		{
+			const IoBuffer&			  Data	   = MetaDatas[WriteBlobIndex];
+			const IoHash&			  BlobHash = BlobHashes[WriteBlobIndex];
+			const BlockStoreLocation& Location = Locations[LocationIndex];
+
+			MetadataEntry Entry = {.Location = Location, .ContentType = Data.GetContentType(), .Flags = 0};
+
+			if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
+			{
+				const BlobIndex ExistingBlobIndex = It->second;
+				BlobEntry&		Blob			  = m_BlobEntries[ExistingBlobIndex];
+				if (Blob.Metadata)
+				{
+					m_MetadataEntries[Blob.Metadata] = Entry;
+				}
+				else
+				{
+					Blob.Metadata = MetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size()));
+					m_MetadataEntries.push_back(Entry);
+				}
+				Blob.LastAccessTime = GcClock::TickCount();
+			}
+			else
+			{
+				MetadataIndex NewMetadataIndex = MetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size()));
+				m_MetadataEntries.push_back(Entry);
+
+				const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
+				m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
+				m_BlobLookup.insert({BlobHash, NewBlobIndex});
+			}
+
+			m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash});
+
+			m_LastAccessTimeUpdateCount++;
+			WriteBlobIndex++;
+			if (m_TrackedCacheKeys)
+			{
+				m_TrackedCacheKeys->insert(BlobHash);
+			}
+		}
+	});
+}
+
+std::vector<IoBuffer>
+BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool)
+{
+	ZEN_TRACE_CPU("BuildStore::GetMetadatas");
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+	std::vector<BlockStoreLocation> MetaLocations;
+	std::vector<size_t>				MetaLocationResultIndexes;
+	MetaLocations.reserve(BlobHashes.size());
+	MetaLocationResultIndexes.reserve(BlobHashes.size());
+	tsl::robin_set<uint32_t> ReferencedBlocks;
+
+	std::vector<IoBuffer>		Result;
+	std::vector<ZenContentType> ResultContentTypes;
+	Result.resize(BlobHashes.size());
+	ResultContentTypes.resize(BlobHashes.size(), ZenContentType::kUnknownContentType);
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		for (size_t Index = 0; Index < BlobHashes.size(); Index++)
+		{
+			const IoHash& BlobHash = BlobHashes[Index];
+			if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end())
+			{
+				const BlobIndex ExistingBlobIndex = It->second;
+				BlobEntry&		ExistingBlobEntry = m_BlobEntries[ExistingBlobIndex];
+				if (ExistingBlobEntry.Metadata)
+				{
+					const MetadataEntry& ExistingMetadataEntry = m_MetadataEntries[ExistingBlobEntry.Metadata];
+					MetaLocations.push_back(ExistingMetadataEntry.Location);
+					MetaLocationResultIndexes.push_back(Index);
+					ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex);
+					ResultContentTypes[Index] = ExistingMetadataEntry.ContentType;
+				}
+				ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount());
+				m_LastAccessTimeUpdateCount++;
+			}
+		}
+	}
+
+	auto DoOneBlock = [this](std::span<const BlockStoreLocation> MetaLocations,
+							 std::span<const size_t>			 MetaLocationResultIndexes,
+							 std::span<const size_t>			 ChunkIndexes,
+							 std::vector<IoBuffer>&				 Result) {
+		if (ChunkIndexes.size() < 4)
+		{
+			for (size_t ChunkIndex : ChunkIndexes)
+			{
+				IoBuffer Chunk = m_MetadataBlockStore.TryGetChunk(MetaLocations[ChunkIndex]);
+				if (Chunk)
+				{
+					size_t ResultIndex	= MetaLocationResultIndexes[ChunkIndex];
+					Result[ResultIndex] = std::move(Chunk);
+				}
+			}
+			return true;
+		}
+		return m_MetadataBlockStore.IterateBlock(
+			MetaLocations,
+			ChunkIndexes,
+			[&MetaLocationResultIndexes, &Result](size_t ChunkIndex, const void* Data, uint64_t Size) {
+				if (Data != nullptr)
+				{
+					size_t ResultIndex	= MetaLocationResultIndexes[ChunkIndex];
+					Result[ResultIndex] = IoBuffer(IoBuffer::Clone, Data, Size);
+				}
+				return true;
+			},
+			[&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+				size_t ResultIndex	= MetaLocationResultIndexes[ChunkIndex];
+				Result[ResultIndex] = File.GetChunk(Offset, Size);
+				return true;
+			},
+			8u * 1024u);
+	};
+
+	if (!MetaLocations.empty())
+	{
+		std::atomic<bool> AbortFlag;
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
+
+		try
+		{
+			m_MetadataBlockStore.IterateChunks(
+				MetaLocations,
+				[this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock](
+					uint32_t				BlockIndex,
+					std::span<const size_t> ChunkIndexes) -> bool {
+					ZEN_UNUSED(BlockIndex);
+					if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1)
+					{
+						return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result);
+					}
+					else
+					{
+						ZEN_ASSERT(OptionalWorkerPool != nullptr);
+						std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end());
+						Work.ScheduleWork(
+							*OptionalWorkerPool,
+							[this,
+							 &Result,
+							 &MetaLocations,
+							 &MetaLocationResultIndexes,
+							 DoOneBlock,
+							 ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic<bool>& AbortFlag) {
+								if (AbortFlag)
+								{
+									return;
+								}
+								try
+								{
+									if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result))
+									{
+										AbortFlag.store(true);
+									}
+								}
+								catch (const std::exception& Ex)
+								{
+									ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what());
+								}
+							});
+						return !Work.IsAborted();
+					}
+				});
+		}
+		catch (const std::exception& Ex)
+		{
+			AbortFlag.store(true);
+			ZEN_WARN("Failed iterating block metadata chunks in {}. Reason: '{}'", m_Config.RootDirectory, Ex.what());
+		}
+
+		Work.Wait();
+	}
+	for (size_t Index = 0; Index < Result.size(); Index++)
+	{
+		if (Result[Index])
+		{
+			Result[Index].SetContentType(ResultContentTypes[Index]);
+		}
+	}
+	return Result;
+}
+
+void
+BuildStore::Flush()
+{
+	ZEN_TRACE_CPU("BuildStore::Flush");
+	try
+	{
+		Stopwatch  Timer;
+		const auto _ = MakeGuard(
+			[&] { ZEN_INFO("Flushed build store at {} in {}", m_Config.RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+
+		m_LargeBlobStore.Flush();
+		m_SmallBlobStore.Flush();
+		m_MetadataBlockStore.Flush(false);
+
+		m_PayloadlogFile.Flush();
+		m_MetadatalogFile.Flush();
+
+		if (uint64_t LastAccessTimeUpdateCount = m_LastAccessTimeUpdateCount.load(); LastAccessTimeUpdateCount > 0)
+		{
+			m_LastAccessTimeUpdateCount -= LastAccessTimeUpdateCount;
+			RwLock::ExclusiveLockScope UpdateLock(m_Lock);
+			WriteAccessTimes(UpdateLock, blobstore::impl::GetAccessTimesPath(m_Config.RootDirectory));
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("BuildStore::Flush failed. Reason: {}", Ex.what());
+	}
+}
+
+BuildStore::StorageStats
+BuildStore::GetStorageStats() const
+{
+	StorageStats Result;
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		Result.EntryCount = m_BlobLookup.size();
+
+		for (auto LookupIt : m_BlobLookup)
+		{
+			const BlobIndex	 ReadBlobIndex = LookupIt.second;
+			const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+			if (ReadBlobEntry.Payload)
+			{
+				const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload];
+				uint64_t			Size	= Payload.GetSize();
+				if ((Payload.GetFlags() & PayloadEntry::kStandalone) != 0)
+				{
+					Result.LargeBlobCount++;
+					Result.LargeBlobBytes += Size;
+				}
+				else
+				{
+					Result.SmallBlobCount++;
+					Result.SmallBlobBytes += Size;
+				}
+			}
+			if (ReadBlobEntry.Metadata)
+			{
+				const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata];
+				Result.MetadataCount++;
+				Result.MetadataByteCount += Metadata.Location.Size;
+			}
+		}
+	}
+	return Result;
+}
+
+#if ZEN_WITH_TESTS
+std::optional<AccessTime>
+BuildStore::GetLastAccessTime(const IoHash& Key) const
+{
+	RwLock::SharedLockScope _(m_Lock);
+	if (auto It = m_BlobLookup.find(Key); It != m_BlobLookup.end())
+	{
+		const BlobIndex	 Index = It->second;
+		const BlobEntry& Entry = m_BlobEntries[Index];
+		return Entry.LastAccessTime;
+	}
+	return {};
+}
+
+bool
+BuildStore::SetLastAccessTime(const IoHash& Key, const AccessTime& Time)
+{
+	RwLock::SharedLockScope _(m_Lock);
+	if (auto It = m_BlobLookup.find(Key); It != m_BlobLookup.end())
+	{
+		const BlobIndex Index = It->second;
+		BlobEntry&		Entry = m_BlobEntries[Index];
+		Entry.LastAccessTime  = Time;
+		return true;
+	}
+	return false;
+}
+#endif	// ZEN_WITH_TESTS
+
+void
+BuildStore::CompactState()
+{
+	ZEN_TRACE_CPU("BuildStore::CompactState");
+
+	std::vector<BlobEntry>	   BlobEntries;
+	std::vector<PayloadEntry>  PayloadEntries;
+	std::vector<MetadataEntry> MetadataEntries;
+
+	tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> BlobLookup;
+
+	RwLock::ExclusiveLockScope _(m_Lock);
+	const size_t			   EntryCount = m_BlobLookup.size();
+	BlobLookup.reserve(EntryCount);
+	const size_t PayloadCount = m_PayloadEntries.size();
+	PayloadEntries.reserve(PayloadCount);
+	const size_t MetadataCount = m_MetadataEntries.size();
+	MetadataEntries.reserve(MetadataCount);
+
+	for (auto LookupIt : m_BlobLookup)
+	{
+		const IoHash&	 BlobHash	   = LookupIt.first;
+		const BlobIndex	 ReadBlobIndex = LookupIt.second;
+		const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+
+		const BlobIndex WriteBlobIndex(gsl::narrow<uint32_t>(BlobEntries.size()));
+		BlobEntries.push_back(ReadBlobEntry);
+		BlobEntry& WriteBlobEntry = BlobEntries.back();
+
+		if (WriteBlobEntry.Payload)
+		{
+			const PayloadEntry& ReadPayloadEntry = m_PayloadEntries[ReadBlobEntry.Payload];
+			WriteBlobEntry.Payload				 = PayloadIndex(gsl::narrow<uint32_t>(PayloadEntries.size()));
+			PayloadEntries.push_back(ReadPayloadEntry);
+		}
+		if (ReadBlobEntry.Metadata)
+		{
+			const MetadataEntry& ReadMetadataEntry = m_MetadataEntries[ReadBlobEntry.Metadata];
+			WriteBlobEntry.Metadata				   = MetadataIndex(gsl::narrow<uint32_t>(MetadataEntries.size()));
+			MetadataEntries.push_back(ReadMetadataEntry);
+		}
+
+		BlobLookup.insert({BlobHash, WriteBlobIndex});
+	}
+	m_BlobEntries.swap(BlobEntries);
+	m_PayloadEntries.swap(PayloadEntries);
+	m_MetadataEntries.swap(MetadataEntries);
+	m_BlobLookup.swap(BlobLookup);
+}
+
+uint64_t
+BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount)
+{
+	ZEN_TRACE_CPU("BuildStore::ReadPayloadLog");
+	if (!IsFile(LogPath))
+	{
+		return 0;
+	}
+
+	uint64_t   LogEntryCount = 0;
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		ZEN_INFO("read build store '{}' payload log containing {} entries in {}",
+				 LogPath,
+				 LogEntryCount,
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
+
+	TCasLogFile<PayloadDiskEntry> CasLog;
+	if (!CasLog.IsValid(LogPath))
+	{
+		RemoveFile(LogPath);
+		return 0;
+	}
+	CasLog.Open(LogPath, CasLogFile::Mode::kRead);
+	if (!CasLog.Initialize())
+	{
+		return 0;
+	}
+
+	const uint64_t EntryCount = CasLog.GetLogCount();
+	if (EntryCount < SkipEntryCount)
+	{
+		ZEN_WARN("reading full payload log at '{}', reason: Log position from index snapshot is out of range", LogPath);
+		SkipEntryCount = 0;
+	}
+
+	LogEntryCount			   = EntryCount - SkipEntryCount;
+	uint64_t InvalidEntryCount = 0;
+
+	CasLog.Replay(
+		[&](const PayloadDiskEntry& Record) {
+			std::string InvalidEntryReason;
+			if (Record.Entry.GetFlags() & PayloadEntry::kTombStone)
+			{
+				// Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
+				if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end())
+				{
+					if (!m_BlobEntries[ExistingIt->second].Metadata)
+					{
+						m_BlobLookup.erase(ExistingIt);
+					}
+					else
+					{
+						m_BlobEntries[ExistingIt->second].Payload = {};
+					}
+				}
+				return;
+			}
+
+			if (!ValidatePayloadDiskEntry(Record, InvalidEntryReason))
+			{
+				ZEN_WARN("skipping invalid payload entry in '{}', reason: '{}'", LogPath, InvalidEntryReason);
+				++InvalidEntryCount;
+				return;
+			}
+			if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end())
+			{
+				const BlobIndex ExistingBlobIndex = It->second;
+				BlobEntry&		ExistingBlob	  = m_BlobEntries[ExistingBlobIndex];
+				if (ExistingBlob.Payload)
+				{
+					const PayloadIndex ExistingPayloadIndex = ExistingBlob.Payload;
+					m_PayloadEntries[ExistingPayloadIndex]	= Record.Entry;
+				}
+				else
+				{
+					const PayloadIndex NewPayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
+					m_PayloadEntries.push_back(Record.Entry);
+					ExistingBlob.Payload = NewPayloadIndex;
+				}
+			}
+			else
+			{
+				const PayloadIndex NewPayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size()));
+				m_PayloadEntries.push_back(Record.Entry);
+
+				const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
+				m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
+				m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex);
+			}
+		},
+		SkipEntryCount);
+
+	if (InvalidEntryCount)
+	{
+		ZEN_WARN("found {} invalid payload entries in '{}'", InvalidEntryCount, LogPath);
+	}
+
+	return LogEntryCount;
+}
+
+uint64_t
+BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount)
+{
+	ZEN_TRACE_CPU("BuildStore::ReadMetadataLog");
+	if (!IsFile(LogPath))
+	{
+		return 0;
+	}
+
+	uint64_t   LogEntryCount = 0;
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		ZEN_INFO("read build store '{}' metadata log containing {} entries in {}",
+				 LogPath,
+				 LogEntryCount,
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
+
+	TCasLogFile<MetadataDiskEntry> CasLog;
+	if (!CasLog.IsValid(LogPath))
+	{
+		RemoveFile(LogPath);
+		return 0;
+	}
+	CasLog.Open(LogPath, CasLogFile::Mode::kRead);
+	if (!CasLog.Initialize())
+	{
+		return 0;
+	}
+
+	const uint64_t EntryCount = CasLog.GetLogCount();
+	if (EntryCount < SkipEntryCount)
+	{
+		ZEN_WARN("reading full metadata log at '{}', reason: Log position from index snapshot is out of range", LogPath);
+		SkipEntryCount = 0;
+	}
+
+	LogEntryCount			   = EntryCount - SkipEntryCount;
+	uint64_t InvalidEntryCount = 0;
+
+	CasLog.Replay(
+		[&](const MetadataDiskEntry& Record) {
+			std::string InvalidEntryReason;
+			if (Record.Entry.Flags & MetadataEntry::kTombStone)
+			{
+				// Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
+				// Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
+				if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end())
+				{
+					if (!m_BlobEntries[ExistingIt->second].Payload)
+					{
+						m_BlobLookup.erase(ExistingIt);
+					}
+					else
+					{
+						m_BlobEntries[ExistingIt->second].Metadata = {};
+					}
+				}
+				return;
+			}
+
+			if (!ValidateMetadataDiskEntry(Record, InvalidEntryReason))
+			{
+				ZEN_WARN("skipping invalid metadata entry in '{}', reason: '{}'", LogPath, InvalidEntryReason);
+				++InvalidEntryCount;
+				return;
+			}
+			if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end())
+			{
+				const BlobIndex ExistingBlobIndex = It->second;
+				BlobEntry&		ExistingBlob	  = m_BlobEntries[ExistingBlobIndex];
+				if (ExistingBlob.Metadata)
+				{
+					const MetadataIndex ExistingMetadataIndex = ExistingBlob.Metadata;
+					m_MetadataEntries[ExistingMetadataIndex]  = Record.Entry;
+				}
+				else
+				{
+					const MetadataIndex NewMetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size()));
+					m_MetadataEntries.push_back(Record.Entry);
+					ExistingBlob.Metadata = NewMetadataIndex;
+				}
+			}
+			else
+			{
+				const MetadataIndex NewMetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size()));
+				m_MetadataEntries.push_back(Record.Entry);
+
+				const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
+				m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
+				m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex);
+			}
+		},
+		SkipEntryCount);
+
+	if (InvalidEntryCount)
+	{
+		ZEN_WARN("found {} invalid metadata entries in '{}'", InvalidEntryCount, LogPath);
+	}
+
+	return LogEntryCount;
+}
+
+void
+BuildStore::ReadAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath)
+{
+	ZEN_TRACE_CPU("BuildStore::ReadAccessTimes");
+
+	using namespace blobstore::impl;
+
+	BasicFile AccessTimesFile;
+	AccessTimesFile.Open(AccessTimesPath, BasicFile::Mode::kRead);
+	uint64_t Size = AccessTimesFile.FileSize();
+	if (Size >= sizeof(AccessTimesHeader))
+	{
+		AccessTimesHeader Header;
+		uint64_t		  Offset = 0;
+		AccessTimesFile.Read(&Header, sizeof(Header), 0);
+		Offset += sizeof(AccessTimesHeader);
+		Offset = RoundUp(Offset, AccessTimesHeader::DataAlignment);
+		if ((Header.Magic == AccessTimesHeader::ExpectedMagic) && (Header.Version == AccessTimesHeader::CurrentVersion) &&
+			(Header.Checksum == AccessTimesHeader::ComputeChecksum(Header)))
+		{
+			uint64_t RecordsSize = sizeof(AccessTimeRecord) * Header.AccessTimeCount;
+			if (AccessTimesFile.FileSize() >= Offset + RecordsSize)
+			{
+				std::vector<AccessTimeRecord> AccessRecords(Header.AccessTimeCount);
+				AccessTimesFile.Read(AccessRecords.data(), RecordsSize, Offset);
+				for (const AccessTimeRecord& Record : AccessRecords)
+				{
+					const IoHash&  Key				 = Record.Key;
+					const uint32_t SecondsSinceEpoch = Record.SecondsSinceEpoch;
+					if (auto It = m_BlobLookup.find(Key); It != m_BlobLookup.end())
+					{
+						const BlobIndex Index = It->second;
+						BlobEntry&		Entry = m_BlobEntries[Index];
+						Entry.LastAccessTime.SetSecondsSinceEpoch(SecondsSinceEpoch);
+					}
+					else
+					{
+						m_LastAccessTimeUpdateCount++;
+					}
+				}
+			}
+			else
+			{
+				m_LastAccessTimeUpdateCount++;
+			}
+		}
+		else
+		{
+			m_LastAccessTimeUpdateCount++;
+		}
+	}
+	else
+	{
+		m_LastAccessTimeUpdateCount++;
+	}
+}
+
+void
+BuildStore::WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath)
+{
+	ZEN_TRACE_CPU("BuildStore::WriteAccessTimes");
+
+	using namespace blobstore::impl;
+
+	uint32_t		  Count	 = gsl::narrow<uint32_t>(m_BlobLookup.size());
+	AccessTimesHeader Header = {.AccessTimeCount = Count};
+	Header.Checksum			 = AccessTimesHeader::ComputeChecksum(Header);
+
+	TemporaryFile	TempFile;
+	std::error_code Ec;
+	if (TempFile.CreateTemporary(AccessTimesPath.parent_path(), Ec); Ec)
+	{
+		throw std::runtime_error(fmt::format("Failed to create temporary file {} to write access times. Reason ({}) {}",
+											 TempFile.GetPath(),
+											 Ec.value(),
+											 Ec.message()));
+	}
+	{
+		uint64_t Offset = 0;
+		TempFile.Write(&Header, sizeof(AccessTimesHeader), Offset);
+		Offset += sizeof(AccessTimesHeader);
+		Offset = RoundUp(Offset, AccessTimesHeader::DataAlignment);
+
+		std::vector<AccessTimeRecord> AccessRecords;
+		AccessRecords.reserve(Header.AccessTimeCount);
+
+		for (auto It : m_BlobLookup)
+		{
+			const IoHash&	 Key			   = It.first;
+			const BlobIndex	 Index			   = It.second;
+			const BlobEntry& Entry			   = m_BlobEntries[Index];
+			const uint32_t	 SecondsSinceEpoch = Entry.LastAccessTime.GetSecondsSinceEpoch();
+			AccessRecords.emplace_back(AccessTimeRecord{.Key = Key, .SecondsSinceEpoch = SecondsSinceEpoch});
+		}
+		uint64_t RecordsSize = sizeof(AccessTimeRecord) * Header.AccessTimeCount;
+		TempFile.Write(AccessRecords.data(), RecordsSize, Offset);
+		Offset += sizeof(AccessTimesHeader) * Header.AccessTimeCount;
+	}
+	if (TempFile.MoveTemporaryIntoPlace(AccessTimesPath, Ec); Ec)
+	{
+		throw std::runtime_error(fmt::format("Failed to move temporary file {} to {} when write access times. Reason ({}) {}",
+											 TempFile.GetPath(),
+											 AccessTimesPath,
+											 Ec.value(),
+											 Ec.message()));
+	}
+}
+
+bool
+BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason)
+{
+	if (Entry.BlobHash == IoHash::Zero)
+	{
+		OutReason = fmt::format("Invalid blob hash {}", Entry.BlobHash.ToHexString());
+		return false;
+	}
+	if (Entry.Entry.GetFlags() & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone))
+	{
+		OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.GetFlags(), Entry.BlobHash.ToHexString());
+		return false;
+	}
+	if (Entry.Entry.GetFlags() & PayloadEntry::kTombStone)
+	{
+		return true;
+	}
+	if (Entry.Entry.GetSize() == 0 || Entry.Entry.GetSize() == 0x00ffffffffffffffu)
+	{
+		OutReason = fmt::format("Invalid size field {} for meta entry {}", Entry.Entry.GetSize(), Entry.BlobHash.ToHexString());
+		return false;
+	}
+	return true;
+}
+
+bool
+BuildStore::ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason)
+{
+	if (Entry.BlobHash == IoHash::Zero)
+	{
+		OutReason = fmt::format("Invalid blob hash {} for meta entry", Entry.BlobHash.ToHexString());
+		return false;
+	}
+	if (Entry.Entry.Location.Size == 0)
+	{
+		OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.Location.Size);
+		return false;
+	}
+	if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0)
+	{
+		OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString());
+		return false;
+	}
+	if (Entry.Entry.Flags & MetadataEntry::kTombStone)
+	{
+		return true;
+	}
+	if (Entry.Entry.ContentType == ZenContentType::kCOUNT)
+	{
+		OutReason = fmt::format("Invalid content type for meta entry {}", Entry.BlobHash.ToHexString());
+		return false;
+	}
+	if (Entry.Reserved1 != 0 || Entry.Reserved2 != 0 || Entry.Reserved3 != 0 || Entry.Reserved4 != 0)
+	{
+		OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString());
+		return false;
+	}
+	return true;
+}
+
+class BuildStoreGcReferenceChecker : public GcReferenceChecker
+{
+public:
+	BuildStoreGcReferenceChecker(BuildStore& Store) : m_Store(Store) {}
+	virtual std::string GetGcName(GcCtx& Ctx) override
+	{
+		ZEN_UNUSED(Ctx);
+		return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string());
+	}
+
+	virtual void PreCache(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); }
+
+	virtual void UpdateLockedState(GcCtx& Ctx) override
+	{
+		ZEN_TRACE_CPU("Builds::UpdateLockedState");
+		ZEN_MEMSCOPE(GetBuildstoreTag());
+
+		auto Log = [&Ctx]() { return Ctx.Logger; };
+
+		m_References.reserve(m_Store.m_BlobLookup.size());
+		for (const auto& It : m_Store.m_BlobLookup)
+		{
+			const BuildStore::BlobIndex ExistingBlobIndex = It.second;
+			if (m_Store.m_BlobEntries[ExistingBlobIndex].Payload)
+			{
+				m_References.push_back(It.first);
+			}
+		}
+		FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", "buildstore"), m_References);
+	}
+
+	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
+	{
+		ZEN_UNUSED(Ctx);
+		ZEN_TRACE_CPU("Builds::GetUnusedReferences");
+		ZEN_MEMSCOPE(GetBuildstoreTag());
+
+		auto Log = [&Ctx]() { return Ctx.Logger; };
+
+		size_t InitialCount = IoCids.size();
+		size_t UsedCount	= InitialCount;
+
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: buildstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}",
+					 "buildstore",
+					 UsedCount,
+					 InitialCount,
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids);
+		UsedCount						   = IoCids.size() - UnusedReferences.size();
+		return UnusedReferences;
+	}
+
+private:
+	BuildStore&			m_Store;
+	std::vector<IoHash> m_References;
+};
+
+std::string
+BuildStore::GetGcName(GcCtx& Ctx)
+{
+	ZEN_UNUSED(Ctx);
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+
+	return fmt::format("buildstore: '{}'", m_Config.RootDirectory.string());
+}
+
+class BuildStoreGcCompator : public GcStoreCompactor
+{
+	using BlobEntry			= BuildStore::BlobEntry;
+	using PayloadEntry		= BuildStore::PayloadEntry;
+	using MetadataEntry		= BuildStore::MetadataEntry;
+	using MetadataDiskEntry = BuildStore::MetadataDiskEntry;
+	using BlobIndex			= BuildStore::BlobIndex;
+	using PayloadIndex		= BuildStore::PayloadIndex;
+	using MetadataIndex		= BuildStore::MetadataIndex;
+
+public:
+	BuildStoreGcCompator(BuildStore& Store, std::vector<IoHash>&& RemovedBlobs) : m_Store(Store), m_RemovedBlobs(std::move(RemovedBlobs)) {}
+
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override
+	{
+		ZEN_UNUSED(ClaimDiskReserveCallback);
+		ZEN_TRACE_CPU("Builds::CompactStore");
+		ZEN_MEMSCOPE(GetBuildstoreTag());
+
+		auto Log = [&Ctx]() { return Ctx.Logger; };
+
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: buildstore [COMPACT] '{}': RemovedDisk: {} in {}",
+					 m_Store.m_Config.RootDirectory,
+					 NiceBytes(Stats.RemovedDisk),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		const auto __ = MakeGuard([&] { m_Store.Flush(); });
+
+		if (!m_RemovedBlobs.empty())
+		{
+			if (Ctx.Settings.CollectSmallObjects)
+			{
+				m_Store.m_Lock.WithExclusiveLock([this]() { m_Store.m_TrackedCacheKeys = std::make_unique<HashSet>(); });
+				auto __ = MakeGuard([this]() { m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedCacheKeys.reset(); }); });
+
+				BlockStore::BlockUsageMap BlockUsage;
+				{
+					RwLock::SharedLockScope __(m_Store.m_Lock);
+
+					for (auto LookupIt : m_Store.m_BlobLookup)
+					{
+						const BlobIndex	 ReadBlobIndex = LookupIt.second;
+						const BlobEntry& ReadBlobEntry = m_Store.m_BlobEntries[ReadBlobIndex];
+
+						if (ReadBlobEntry.Metadata)
+						{
+							const MetadataEntry& ReadMetadataEntry = m_Store.m_MetadataEntries[ReadBlobEntry.Metadata];
+
+							uint32_t BlockIndex = ReadMetadataEntry.Location.BlockIndex;
+							uint64_t ChunkSize	= RoundUp(ReadMetadataEntry.Location.Size, m_Store.m_Config.MetadataBlockStoreAlignement);
+
+							if (auto BlockUsageIt = BlockUsage.find(BlockIndex); BlockUsageIt != BlockUsage.end())
+							{
+								BlockStore::BlockUsageInfo& Info = BlockUsageIt.value();
+								Info.EntryCount++;
+								Info.DiskUsage += ChunkSize;
+							}
+							else
+							{
+								BlockUsage.insert_or_assign(BlockIndex,
+															BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
+							}
+						}
+					}
+				}
+
+				BlockStore::BlockEntryCountMap BlocksToCompact = m_Store.m_MetadataBlockStore.GetBlocksToCompact(BlockUsage, 90);
+				BlockStoreCompactState		   BlockCompactState;
+				std::vector<IoHash>			   BlockCompactStateKeys;
+				BlockCompactState.IncludeBlocks(BlocksToCompact);
+
+				if (BlocksToCompact.size() > 0)
+				{
+					{
+						RwLock::SharedLockScope ___(m_Store.m_Lock);
+						for (const auto& Entry : m_Store.m_BlobLookup)
+						{
+							BlobIndex Index = Entry.second;
+
+							if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta)
+							{
+								if (BlockCompactState.AddKeepLocation(m_Store.m_MetadataEntries[Meta].Location))
+								{
+									BlockCompactStateKeys.push_back(Entry.first);
+								}
+							}
+						}
+					}
+
+					if (Ctx.Settings.IsDeleteMode)
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: buildstore [COMPACT] '{}': compacting {} blocks",
+									 m_Store.m_Config.RootDirectory,
+									 BlocksToCompact.size());
+						}
+
+						m_Store.m_MetadataBlockStore.CompactBlocks(
+							BlockCompactState,
+							m_Store.m_Config.MetadataBlockStoreAlignement,
+							[&](const BlockStore::MovedChunksArray& MovedArray,
+								const BlockStore::ChunkIndexArray&	ScrubbedArray,
+								uint64_t							FreedDiskSpace) {
+								std::vector<MetadataDiskEntry> MovedEntries;
+								MovedEntries.reserve(MovedArray.size());
+								RwLock::ExclusiveLockScope _(m_Store.m_Lock);
+								for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+								{
+									size_t		  ChunkIndex = Moved.first;
+									const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
+
+									ZEN_ASSERT(m_Store.m_TrackedCacheKeys);
+									if (m_Store.m_TrackedCacheKeys->contains(Key))
+									{
+										continue;
+									}
+
+									if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end())
+									{
+										const BlobIndex Index = It->second;
+
+										if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta)
+										{
+											m_Store.m_MetadataEntries[Meta].Location = Moved.second;
+											MovedEntries.push_back(
+												MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key});
+										}
+									}
+								}
+
+								for (size_t Scrubbed : ScrubbedArray)
+								{
+									const IoHash& Key = BlockCompactStateKeys[Scrubbed];
+									if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end())
+									{
+										const BlobIndex Index = It->second;
+
+										if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta)
+										{
+											MovedEntries.push_back(
+												MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key});
+											MovedEntries.back().Entry.Flags |= MetadataEntry::kTombStone;
+											m_Store.m_MetadataEntries[Meta]		  = {};
+											m_Store.m_BlobEntries[Index].Metadata = {};
+										}
+									}
+								}
+
+								m_Store.m_MetadatalogFile.Append(MovedEntries);
+
+								Stats.RemovedDisk += FreedDiskSpace;
+								if (Ctx.IsCancelledFlag.load())
+								{
+									return false;
+								}
+								return true;
+							},
+							ClaimDiskReserveCallback,
+							fmt::format("GCV2: buildstore [COMPACT] '{}': ", m_Store.m_Config.RootDirectory));
+					}
+					else
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: buildstore [COMPACT] '{}': skipped compacting of {} eligible blocks",
+									 m_Store.m_Config.RootDirectory,
+									 BlocksToCompact.size());
+						}
+					}
+				}
+			}
+		}
+	}
+
+	virtual std::string GetGcName(GcCtx& Ctx) override
+	{
+		ZEN_UNUSED(Ctx);
+		ZEN_MEMSCOPE(GetBuildstoreTag());
+
+		return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string());
+	}
+
+private:
+	BuildStore&				  m_Store;
+	const std::vector<IoHash> m_RemovedBlobs;
+};
+
+GcStoreCompactor*
+BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
+{
+	ZEN_TRACE_CPU("Builds::RemoveExpiredData");
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+
+	auto Log = [&Ctx]() { return Ctx.Logger; };
+
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		if (Ctx.Settings.Verbose)
+		{
+			ZEN_INFO("GCV2: buildstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}",
+					 m_Config.RootDirectory,
+					 Stats.CheckedCount,
+					 Stats.FoundCount,
+					 Stats.DeletedCount,
+					 NiceBytes(Stats.FreedMemory),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		}
+	});
+
+	const GcClock::Tick					   ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count();
+	std::vector<IoHash>					   ExpiredBlobs;
+	tsl::robin_set<IoHash, IoHash::Hasher> SizeDroppedBlobs;
+
+	{
+		struct SizeInfo
+		{
+			const IoHash Key;
+			uint32_t	 SecondsSinceEpoch = 0;
+			uint64_t	 BlobSize		   = 0;
+		};
+
+		bool		   DiskSizeExceeded = false;
+		const uint64_t CurrentDiskSize =
+			m_LargeBlobStore.StorageSize().DiskSize + m_SmallBlobStore.StorageSize().DiskSize + m_MetadataBlockStore.TotalSize();
+		if (CurrentDiskSize > m_Config.MaxDiskSpaceLimit)
+		{
+			DiskSizeExceeded = true;
+		}
+
+		uint64_t ExpiredDataSize = 0;
+
+		std::vector<SizeInfo> NonExpiredBlobSizeInfos;
+
+		{
+			RwLock::SharedLockScope __(m_Lock);
+			if (DiskSizeExceeded)
+			{
+				NonExpiredBlobSizeInfos.reserve(m_BlobLookup.size());
+			}
+			for (const auto& It : m_BlobLookup)
+			{
+				const BlobIndex	 ReadBlobIndex = It.second;
+				const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+				uint64_t		 Size		   = 0;
+				if (ReadBlobEntry.Payload)
+				{
+					const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload];
+					Size += Payload.GetSize();
+				}
+				if (ReadBlobEntry.Metadata)
+				{
+					const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata];
+					Size += Metadata.Location.Size;
+				}
+
+				const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime;
+				if (AccessTick < ExpireTicks)
+				{
+					ExpiredBlobs.push_back(It.first);
+					ExpiredDataSize += ExpiredDataSize;
+				}
+				else if (DiskSizeExceeded)
+				{
+					NonExpiredBlobSizeInfos.emplace_back(SizeInfo{.Key				 = It.first,
+																  .SecondsSinceEpoch = ReadBlobEntry.LastAccessTime.GetSecondsSinceEpoch(),
+																  .BlobSize			 = Size});
+				}
+			}
+			Stats.CheckedCount += m_BlobLookup.size();
+			Stats.FoundCount += ExpiredBlobs.size();
+		}
+
+		if (DiskSizeExceeded)
+		{
+			const uint64_t NewSizeLimit =
+				m_Config.MaxDiskSpaceLimit -
+				(m_Config.MaxDiskSpaceLimit >> 4);	// Remove a bit more than just below the limit so we have some space to grow
+			if ((CurrentDiskSize - ExpiredDataSize) > NewSizeLimit)
+			{
+				std::vector<size_t> NonExpiredOrder;
+				NonExpiredOrder.resize(NonExpiredBlobSizeInfos.size());
+				for (size_t Index = 0; Index < NonExpiredOrder.size(); Index++)
+				{
+					NonExpiredOrder[Index] = Index;
+				}
+				std::sort(NonExpiredOrder.begin(), NonExpiredOrder.end(), [&NonExpiredBlobSizeInfos](const size_t Lhs, const size_t Rhs) {
+					const SizeInfo& LhsInfo = NonExpiredBlobSizeInfos[Lhs];
+					const SizeInfo& RhsInfo = NonExpiredBlobSizeInfos[Rhs];
+					return LhsInfo.SecondsSinceEpoch < RhsInfo.SecondsSinceEpoch;
+				});
+
+				auto It = NonExpiredOrder.begin();
+				while (It != NonExpiredOrder.end())
+				{
+					const SizeInfo& Info = NonExpiredBlobSizeInfos[*It];
+					if ((CurrentDiskSize - ExpiredDataSize) < NewSizeLimit)
+					{
+						break;
+					}
+					ExpiredDataSize += Info.BlobSize;
+					ExpiredBlobs.push_back(Info.Key);
+					SizeDroppedBlobs.insert(Info.Key);
+					It++;
+				}
+			}
+		}
+	}
+
+	std::vector<IoHash> RemovedBlobs;
+	if (!ExpiredBlobs.empty())
+	{
+		if (Ctx.Settings.IsDeleteMode)
+		{
+			RemovedBlobs.reserve(ExpiredBlobs.size());
+
+			std::vector<PayloadDiskEntry>  RemovedPayloads;
+			std::vector<MetadataDiskEntry> RemoveMetadatas;
+
+			RwLock::ExclusiveLockScope __(m_Lock);
+			if (Ctx.IsCancelledFlag.load())
+			{
+				return nullptr;
+			}
+
+			for (const IoHash& ExpiredBlob : ExpiredBlobs)
+			{
+				if (auto It = m_BlobLookup.find(ExpiredBlob); It != m_BlobLookup.end())
+				{
+					const BlobIndex	 ReadBlobIndex = It->second;
+					const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+
+					const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime;
+
+					if (SizeDroppedBlobs.contains(ExpiredBlob) || (AccessTick < ExpireTicks))
+					{
+						if (ReadBlobEntry.Payload)
+						{
+							RemovedPayloads.push_back(
+								PayloadDiskEntry{.Entry = m_PayloadEntries[ReadBlobEntry.Payload], .BlobHash = ExpiredBlob});
+							RemovedPayloads.back().Entry.AddFlag(PayloadEntry::kTombStone);
+							m_PayloadEntries[ReadBlobEntry.Payload] = {};
+							m_BlobEntries[ReadBlobIndex].Payload	= {};
+						}
+						if (ReadBlobEntry.Metadata)
+						{
+							RemoveMetadatas.push_back(
+								MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = ExpiredBlob});
+							RemoveMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone;
+							m_MetadataEntries[ReadBlobEntry.Metadata] = {};
+							m_BlobEntries[ReadBlobIndex].Metadata	  = {};
+						}
+
+						m_BlobLookup.erase(It);
+						m_LastAccessTimeUpdateCount++;
+
+						RemovedBlobs.push_back(ExpiredBlob);
+						Stats.DeletedCount++;
+					}
+				}
+			}
+			if (!RemovedPayloads.empty())
+			{
+				m_PayloadlogFile.Append(RemovedPayloads);
+			}
+			if (!RemoveMetadatas.empty())
+			{
+				m_MetadatalogFile.Append(RemoveMetadatas);
+			}
+		}
+	}
+
+	if (!RemovedBlobs.empty())
+	{
+		CompactState();
+	}
+
+	return new BuildStoreGcCompator(*this, std::move(RemovedBlobs));
+}
+
+std::vector<GcReferenceChecker*>
+BuildStore::CreateReferenceCheckers(GcCtx& Ctx)
+{
+	ZEN_UNUSED(Ctx);
+	ZEN_MEMSCOPE(GetBuildstoreTag());
+	return {new BuildStoreGcReferenceChecker(*this)};
+}
+
+std::vector<GcReferenceValidator*>
+BuildStore::CreateReferenceValidators(GcCtx& Ctx)
+{
+	ZEN_UNUSED(Ctx);
+	return {};
+}
+
+std::vector<RwLock::SharedLockScope>
+BuildStore::LockState(GcCtx& Ctx)
+{
+	ZEN_UNUSED(Ctx);
+	std::vector<RwLock::SharedLockScope> Locks;
+	Locks.emplace_back(RwLock::SharedLockScope(m_Lock));
+	return Locks;
+}
+
+void
+BuildStore::ScrubStorage(ScrubContext& ScrubCtx)
+{
+	ZEN_UNUSED(ScrubCtx);
+	// TODO
+}
+
+GcStorageSize
+BuildStore::StorageSize() const
+{
+	GcStorageSize Result;
+	Result.DiskSize = m_MetadataBlockStore.TotalSize();
+	return Result;
+}
+
+/*
+		___________              __
+		\__    ___/___   _______/  |_  ______
+		  |    |_/ __ \ /  ___/\   __\/  ___/
+		  |    |\  ___/ \___ \  |  |  \___ \
+		  |____| \___  >____  > |__| /____  >
+					 \/     \/            \/
+*/
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("BuildStore.Blobs")
+{
+	ScopedTemporaryDirectory _;
+
+	BuildStoreConfig Config;
+	Config.RootDirectory = _.Path() / "build_store";
+
+	std::vector<IoHash> CompressedBlobsHashes;
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+
+		for (size_t I = 0; I < 5; I++)
+		{
+			IoBuffer		 Blob			= CreateSemiRandomBlob(4711 + I * 7);
+			CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+			CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+			IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+			Store.PutBlob(CompressedBlobsHashes.back(), Payload);
+		}
+
+		for (const IoHash& RawHash : CompressedBlobsHashes)
+		{
+			IoBuffer Payload = Store.GetBlob(RawHash);
+			CHECK(Payload);
+			CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary);
+			IoHash			 VerifyRawHash;
+			uint64_t		 VerifyRawSize;
+			CompressedBuffer CompressedBlob =
+				CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize);
+			CHECK(CompressedBlob);
+			CHECK(VerifyRawHash == RawHash);
+			IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer();
+			CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
+		}
+	}
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+		for (const IoHash& RawHash : CompressedBlobsHashes)
+		{
+			IoBuffer Payload = Store.GetBlob(RawHash);
+			CHECK(Payload);
+			CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary);
+			IoHash			 VerifyRawHash;
+			uint64_t		 VerifyRawSize;
+			CompressedBuffer CompressedBlob =
+				CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize);
+			CHECK(CompressedBlob);
+			CHECK(VerifyRawHash == RawHash);
+			IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer();
+			CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
+		}
+
+		for (size_t I = 0; I < 5; I++)
+		{
+			IoBuffer		 Blob			= CreateSemiRandomBlob(5713 + I * 7);
+			CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+			CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+			IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+			Store.PutBlob(CompressedBlobsHashes.back(), Payload);
+		}
+	}
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+		for (const IoHash& RawHash : CompressedBlobsHashes)
+		{
+			IoBuffer Payload = Store.GetBlob(RawHash);
+			CHECK(Payload);
+			CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary);
+			IoHash			 VerifyRawHash;
+			uint64_t		 VerifyRawSize;
+			CompressedBuffer CompressedBlob =
+				CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize);
+			CHECK(CompressedBlob);
+			CHECK(VerifyRawHash == RawHash);
+			IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer();
+			CHECK(IoHash::HashBuffer(Decompressed) == RawHash);
+		}
+	}
+}
+
+namespace blockstore::testing {
+	IoBuffer MakeMetaData(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues)
+	{
+		CbObjectWriter Writer;
+		Writer.AddHash("rawHash"sv, BlobHash);
+		Writer.BeginObject("values");
+		{
+			for (const auto& V : KeyValues)
+			{
+				Writer.AddString(V.first, V.second);
+			}
+		}
+		Writer.EndObject();	 // values
+		return Writer.Save().GetBuffer().AsIoBuffer();
+	};
+
+}  // namespace blockstore::testing
+
+TEST_CASE("BuildStore.Metadata")
+{
+	using namespace blockstore::testing;
+
+	ScopedTemporaryDirectory _;
+
+	WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst);
+
+	BuildStoreConfig Config;
+	Config.RootDirectory = _.Path() / "build_store";
+
+	std::vector<IoHash>	  BlobHashes;
+	std::vector<IoBuffer> MetaPayloads;
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+
+		for (size_t I = 0; I < 5; I++)
+		{
+			BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I)));
+			MetaPayloads.push_back(MakeMetaData(BlobHashes.back(), {{"index", fmt::format("{}", I)}}));
+			MetaPayloads.back().SetContentType(ZenContentType::kCbObject);
+		}
+		Store.PutMetadatas(BlobHashes, MetaPayloads);
+
+		std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool);
+		CHECK(ValidateMetaPayloads.size() == MetaPayloads.size());
+		for (size_t I = 0; I < ValidateMetaPayloads.size(); I++)
+		{
+			const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]);
+			CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash);
+		}
+	}
+	{
+		GcManager			  Gc;
+		BuildStore			  Store(Config, Gc);
+		std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool);
+		CHECK(ValidateMetaPayloads.size() == MetaPayloads.size());
+		for (size_t I = 0; I < ValidateMetaPayloads.size(); I++)
+		{
+			const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]);
+			CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash);
+		}
+		for (const IoHash& BlobHash : BlobHashes)
+		{
+			CHECK(!Store.GetBlob(BlobHash));
+		}
+	}
+	std::vector<IoHash> CompressedBlobsHashes;
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+		for (size_t I = 0; I < 5; I++)
+		{
+			IoBuffer		 Blob			= CreateSemiRandomBlob(4711 + I * 7);
+			CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+			CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+			IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+			Store.PutBlob(CompressedBlobsHashes.back(), Payload);
+		}
+		std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool);
+		for (const auto& MetadataIt : MetadataPayloads)
+		{
+			CHECK(!MetadataIt);
+		}
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			IoBuffer Blob = Store.GetBlob(BlobHash);
+			CHECK(Blob);
+			IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer();
+			CHECK(DecompressedBlob);
+			CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash);
+		}
+	}
+
+	std::vector<IoBuffer> BlobMetaPayloads;
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
+			BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
+		}
+		Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+
+		std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool);
+		CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
+		for (size_t I = 0; I < MetadataPayloads.size(); I++)
+		{
+			const IoBuffer& MetadataPayload = MetadataPayloads[I];
+			CHECK_EQ(IoHash::HashBuffer(MetadataPayload), IoHash::HashBuffer(BlobMetaPayloads[I]));
+		}
+	}
+
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+
+		std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool);
+		CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
+		for (size_t I = 0; I < MetadataPayloads.size(); I++)
+		{
+			const IoBuffer& MetadataPayload = MetadataPayloads[I];
+			CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I]));
+		}
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			IoBuffer Blob = Store.GetBlob(BlobHash);
+			CHECK(Blob);
+			IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer();
+			CHECK(DecompressedBlob);
+			CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash);
+		}
+
+		BlobMetaPayloads.clear();
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			BlobMetaPayloads.push_back(
+				MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}}));
+			BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
+		}
+		Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+	}
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+
+		std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool);
+		CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
+		for (size_t I = 0; I < MetadataPayloads.size(); I++)
+		{
+			const IoBuffer& MetadataPayload = MetadataPayloads[I];
+			CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I]));
+		}
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			IoBuffer Blob = Store.GetBlob(BlobHash);
+			CHECK(Blob);
+			IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer();
+			CHECK(DecompressedBlob);
+			CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash);
+		}
+	}
+}
+
+TEST_CASE("BuildStore.GC")
+{
+	using namespace blockstore::testing;
+
+	ScopedTemporaryDirectory _;
+
+	BuildStoreConfig Config;
+	Config.RootDirectory = _.Path() / "build_store";
+
+	std::vector<IoHash>	  CompressedBlobsHashes;
+	std::vector<IoBuffer> BlobMetaPayloads;
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+		for (size_t I = 0; I < 5; I++)
+		{
+			IoBuffer		 Blob			= CreateSemiRandomBlob(4711 + I * 7);
+			CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob)));
+			CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+			IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+			Store.PutBlob(CompressedBlobsHashes.back(), Payload);
+		}
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
+			BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
+		}
+		Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+	}
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+
+		{
+			GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
+														   .CollectSmallObjects	 = false,
+														   .IsDeleteMode		 = false,
+														   .Verbose				 = true});
+			CHECK(!Result.WasCancelled);
+			for (const IoHash& BlobHash : CompressedBlobsHashes)
+			{
+				IoBuffer Blob = Store.GetBlob(BlobHash);
+				CHECK(Blob);
+				IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer();
+				CHECK(DecompressedBlob);
+				CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash);
+			}
+
+			std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr);
+			CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
+			for (size_t I = 0; I < MetadataPayloads.size(); I++)
+			{
+				const IoBuffer& MetadataPayload = MetadataPayloads[I];
+				CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I]));
+			}
+		}
+		{
+			GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
+														   .CollectSmallObjects	 = true,
+														   .IsDeleteMode		 = true,
+														   .Verbose				 = true});
+			CHECK(!Result.WasCancelled);
+			for (const IoHash& BlobHash : CompressedBlobsHashes)
+			{
+				IoBuffer Blob = Store.GetBlob(BlobHash);
+				CHECK(!Blob);
+			}
+
+			std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr);
+			CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
+			for (size_t I = 0; I < MetadataPayloads.size(); I++)
+			{
+				const IoBuffer& MetadataPayload = MetadataPayloads[I];
+				CHECK(!MetadataPayload);
+			}
+		}
+	}
+}
+
+TEST_CASE("BuildStore.SizeLimit")
+{
+	using namespace blockstore::testing;
+
+	ScopedTemporaryDirectory _;
+
+	BuildStoreConfig Config = {.MaxDiskSpaceLimit = 1024u * 1024u};
+	Config.RootDirectory	= _.Path() / "build_store";
+
+	std::vector<IoHash>	  CompressedBlobsHashes;
+	std::vector<IoBuffer> BlobMetaPayloads;
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+		for (size_t I = 0; I < 64; I++)
+		{
+			IoBuffer		 Blob = CreateSemiRandomBlob(65537 + I * 7);
+			CompressedBuffer CompressedBlob =
+				CompressedBuffer::Compress(SharedBuffer(std::move(Blob)), OodleCompressor::Mermaid, OodleCompressionLevel::None);
+			CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+			IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+			Store.PutBlob(CompressedBlobsHashes.back(), Payload);
+		}
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
+			BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
+		}
+		Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+
+		{
+			for (size_t I = 0; I < 64; I++)
+			{
+				const IoHash& Key		 = CompressedBlobsHashes[I];
+				GcClock::Tick AccessTick = (GcClock::Now() + std::chrono::minutes(I)).time_since_epoch().count();
+
+				Store.SetLastAccessTime(Key, AccessTime(AccessTick));
+			}
+		}
+	}
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+
+		{
+			GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
+														   .CollectSmallObjects	 = true,
+														   .IsDeleteMode		 = true,
+														   .Verbose				 = true});
+
+			uint32_t DeletedBlobs = 0;
+
+			CHECK(!Result.WasCancelled);
+			for (const IoHash& BlobHash : CompressedBlobsHashes)
+			{
+				IoBuffer Blob = Store.GetBlob(BlobHash);
+				if (!Blob)
+				{
+					DeletedBlobs++;
+				}
+				else
+				{
+					IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer();
+					CHECK(DecompressedBlob);
+					CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash);
+				}
+			}
+			CHECK(DeletedBlobs == 50);
+
+			std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr);
+			CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
+			for (size_t I = 0; I < MetadataPayloads.size(); I++)
+			{
+				const IoBuffer& MetadataPayload = MetadataPayloads[I];
+				if (I < DeletedBlobs)
+				{
+					CHECK(!MetadataPayload);
+				}
+				else
+				{
+					CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I]));
+				}
+			}
+		}
+	}
+}
+
+void
+buildstore_forcelink()
+{
+}
+
+#endif
+
+}  // namespace zen
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 61552fafc..15a1c9650 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -14,6 +14,7 @@
 #include <zencore/trace.h>
 #include <zencore/workthreadpool.h>
 #include <zencore/xxhash.h>
+#include <zenutil/parallelwork.h>
 #include <zenutil/referencemetadata.h>
 #include <zenutil/workerpools.h>
 
@@ -195,34 +196,33 @@ namespace cache::impl {
 		return true;
 	}
 
-	bool MoveAndDeleteDirectory(const std::filesystem::path& Dir)
+	std::filesystem::path MoveDroppedDirectory(const std::filesystem::path& Dir)
 	{
 		int DropIndex = 0;
 		do
 		{
-			if (!std::filesystem::exists(Dir))
+			if (!IsDir(Dir))
 			{
-				return false;
+				return {};
 			}
 
 			std::string			  DroppedName		= fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex);
 			std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName;
-			if (std::filesystem::exists(DroppedBucketPath))
+			if (IsDir(DroppedBucketPath))
 			{
 				DropIndex++;
 				continue;
 			}
 
 			std::error_code Ec;
-			std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+			RenameDirectory(Dir, DroppedBucketPath, Ec);
 			if (!Ec)
 			{
-				DeleteDirectories(DroppedBucketPath);
-				return true;
+				return DroppedBucketPath;
 			}
-			// TODO: Do we need to bail at some point?
 			zen::Sleep(100);
-		} while (true);
+		} while (DropIndex < 10);
+		return {};
 	}
 }  // namespace cache::impl
 
@@ -318,10 +318,10 @@ private:
 #pragma pack(4)
 	struct ManifestData
 	{
-		uint32_t   RawSize;	   // 4
-		AccessTime Timestamp;  // 4
-		IoHash	   RawHash;	   // 20
-		IoHash	   Key;		   // 20
+		uint32_t RawSize;			 // 4
+		uint32_t SecondsSinceEpoch;	 // 4
+		IoHash	 RawHash;			 // 20
+		IoHash	 Key;				 // 20
 	};
 #pragma pack(pop)
 
@@ -603,7 +603,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope&								  B
 
 			ZenCacheDiskLayer::CacheBucket::BucketPayload& PayloadEntry = Payloads[PlIndex];
 
-			AccessTimes[PlIndex] = Entry->Timestamp;
+			AccessTimes[PlIndex].SetSecondsSinceEpoch(Entry->SecondsSinceEpoch);
 
 			if (Entry->RawSize && Entry->RawHash != IoHash::Zero)
 			{
@@ -630,6 +630,16 @@ BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&,
 {
 	ZEN_TRACE_CPU("Z$::WriteSidecarFile");
 
+	ZEN_DEBUG("writing store sidecar for '{}'", SidecarPath);
+	const uint64_t EntryCount = Index.size();
+	Stopwatch	   Timer;
+	const auto	   _ = MakeGuard([&] {
+		ZEN_INFO("wrote store sidecar for '{}' containing {} entries in {}",
+				 SidecarPath,
+				 EntryCount,
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
+
 	BucketMetaHeader Header;
 	Header.EntryCount  = m_ManifestEntryCount;
 	Header.LogPosition = SnapshotLogPosition;
@@ -647,43 +657,44 @@ BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&,
 
 	SidecarFile.Write(&Header, sizeof Header, 0);
 
-	// TODO: make this batching for better performance
 	{
 		uint64_t WriteOffset = sizeof Header;
 
-		//		BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024);
+		const size_t MaxManifestDataBufferCount = (512u * 1024u) / sizeof(ManifestData);
 
-		std::vector<ManifestData> ManifestDataBuffer;
-		const size_t			  MaxManifestDataBufferCount = Min(Index.size(), 8192u);  // 512 Kb
-		ManifestDataBuffer.reserve(MaxManifestDataBufferCount);
+		std::vector<ManifestData> ManifestDataBuffer(Min(m_ManifestEntryCount, MaxManifestDataBufferCount));
+		auto					  WriteIt = ManifestDataBuffer.begin();
 		for (auto& Kv : Index)
 		{
-			const IoHash&	   Key	   = Kv.first;
-			const PayloadIndex PlIndex = Kv.second;
+			ManifestData& Data = *WriteIt++;
 
-			IoHash	 RawHash = IoHash::Zero;
-			uint32_t RawSize = 0;
+			const PayloadIndex PlIndex = Kv.second;
+			Data.Key				   = Kv.first;
+			Data.SecondsSinceEpoch	   = AccessTimes[PlIndex].GetSecondsSinceEpoch();
 
 			if (const MetaDataIndex MetaIndex = Payloads[PlIndex].MetaData)
 			{
-				RawHash = MetaDatas[MetaIndex].RawHash;
-				RawSize = MetaDatas[MetaIndex].RawSize;
+				Data.RawHash = MetaDatas[MetaIndex].RawHash;
+				Data.RawSize = MetaDatas[MetaIndex].RawSize;
+			}
+			else
+			{
+				Data.RawHash = IoHash::Zero;
+				Data.RawSize = 0;
 			}
 
-			ManifestDataBuffer.emplace_back(
-				ManifestData{.RawSize = RawSize, .Timestamp = AccessTimes[PlIndex], .RawHash = RawHash, .Key = Key});
-			if (ManifestDataBuffer.size() == MaxManifestDataBufferCount)
+			if (WriteIt == ManifestDataBuffer.end())
 			{
-				const uint64_t WriteSize = sizeof(ManifestData) * ManifestDataBuffer.size();
+				uint64_t WriteSize = std::distance(ManifestDataBuffer.begin(), WriteIt) * sizeof(ManifestData);
 				SidecarFile.Write(ManifestDataBuffer.data(), WriteSize, WriteOffset);
 				WriteOffset += WriteSize;
-				ManifestDataBuffer.clear();
-				ManifestDataBuffer.reserve(MaxManifestDataBufferCount);
+				WriteIt = ManifestDataBuffer.begin();
 			}
 		}
-		if (ManifestDataBuffer.size() > 0)
+		if (WriteIt != ManifestDataBuffer.begin())
 		{
-			SidecarFile.Write(ManifestDataBuffer.data(), sizeof(ManifestData) * ManifestDataBuffer.size(), WriteOffset);
+			uint64_t WriteSize = std::distance(ManifestDataBuffer.begin(), WriteIt) * sizeof(ManifestData);
+			SidecarFile.Write(ManifestDataBuffer.data(), WriteSize, WriteOffset);
 		}
 	}
 
@@ -740,6 +751,16 @@ ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager&				   Gc,
 
 ZenCacheDiskLayer::CacheBucket::~CacheBucket()
 {
+	try
+	{
+		m_SlogFile.Flush();
+		m_SlogFile.Close();
+		m_BlockStore.Close();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("~CacheBucket() failed with: ", Ex.what());
+	}
 	m_Gc.RemoveGcReferencer(*this);
 }
 
@@ -813,12 +834,13 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, const std::function<uint64_t()>& ClaimDiskReserveFunc)
+ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(uint64_t						  LogPosition,
+														 bool							  ResetLog,
+														 const std::function<uint64_t()>& ClaimDiskReserveFunc)
 {
 	ZEN_TRACE_CPU("Z$::Bucket::WriteIndexSnapshot");
 
-	const uint64_t LogCount = FlushLockPosition ? 0 : m_SlogFile.GetLogCount();
-	if (m_LogFlushPosition == LogCount)
+	if (m_LogFlushPosition == LogPosition)
 	{
 		return;
 	}
@@ -835,7 +857,7 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition,
 
 	namespace fs = std::filesystem;
 
-	fs::path IndexPath = cache::impl::GetIndexPath(m_BucketDir, m_BucketName);
+	const fs::path IndexPath = cache::impl::GetIndexPath(m_BucketDir, m_BucketName);
 
 	try
 	{
@@ -867,66 +889,70 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition,
 			throw std::system_error(Ec, fmt::format("failed to create new snapshot file in '{}'", m_BucketDir));
 		}
 
-		{
-			// This is in a separate scope just to ensure IndexWriter goes out
-			// of scope before the file is flushed/closed, in order to ensure
-			// all data is written to the file
-			BasicFileWriter IndexWriter(ObjectIndexFile, 128 * 1024);
+		const uint64_t IndexLogPosition = ResetLog ? 0 : LogPosition;
 
-			cache::impl::CacheBucketIndexHeader Header = {.EntryCount		= EntryCount,
-														  .LogPosition		= LogCount,
-														  .PayloadAlignment = gsl::narrow<uint32_t>(m_Configuration.PayloadAlignment)};
+		cache::impl::CacheBucketIndexHeader Header = {.EntryCount		= EntryCount,
+													  .LogPosition		= IndexLogPosition,
+													  .PayloadAlignment = gsl::narrow<uint32_t>(m_Configuration.PayloadAlignment)};
 
-			Header.Checksum = cache::impl::CacheBucketIndexHeader::ComputeChecksum(Header);
-			IndexWriter.Write(&Header, sizeof(cache::impl::CacheBucketIndexHeader), 0);
+		Header.Checksum = cache::impl::CacheBucketIndexHeader::ComputeChecksum(Header);
+		ObjectIndexFile.Write(&Header, sizeof(cache::impl::CacheBucketIndexHeader), 0);
 
+		if (EntryCount > 0)
+		{
 			uint64_t IndexWriteOffset = sizeof(cache::impl::CacheBucketIndexHeader);
 
+			size_t						MaxWriteEntryCount = (512u * 1024u) / sizeof(DiskIndexEntry);
+			std::vector<DiskIndexEntry> DiskEntryBuffer(Min(m_Index.size(), MaxWriteEntryCount));
+
+			auto WriteIt = DiskEntryBuffer.begin();
 			for (auto& Entry : m_Index)
 			{
-				DiskIndexEntry IndexEntry;
-				IndexEntry.Key		= Entry.first;
-				IndexEntry.Location = m_Payloads[Entry.second].Location;
-				IndexWriter.Write(&IndexEntry, sizeof(DiskIndexEntry), IndexWriteOffset);
-
-				IndexWriteOffset += sizeof(DiskIndexEntry);
+				*WriteIt++ = {.Key = Entry.first, .Location = m_Payloads[Entry.second].Location};
+				if (WriteIt == DiskEntryBuffer.end())
+				{
+					uint64_t WriteSize = std::distance(DiskEntryBuffer.begin(), WriteIt) * sizeof(DiskIndexEntry);
+					ObjectIndexFile.Write(DiskEntryBuffer.data(), WriteSize, IndexWriteOffset);
+					IndexWriteOffset += WriteSize;
+					WriteIt = DiskEntryBuffer.begin();
+				}
 			}
 
-			IndexWriter.Flush();
+			if (WriteIt != DiskEntryBuffer.begin())
+			{
+				uint64_t WriteSize = std::distance(DiskEntryBuffer.begin(), WriteIt) * sizeof(DiskIndexEntry);
+				ObjectIndexFile.Write(DiskEntryBuffer.data(), WriteSize, IndexWriteOffset);
+			}
 		}
 
 		ObjectIndexFile.Flush();
 		ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec);
 		if (Ec)
 		{
-			std::filesystem::path TempFilePath = ObjectIndexFile.GetPath();
-			ZEN_WARN("snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", TempFilePath, IndexPath, Ec.message());
+			throw std::system_error(Ec,
+									fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'",
+												ObjectIndexFile.GetPath(),
+												IndexPath,
+												Ec.message()));
 		}
-		else
+
+		if (ResetLog)
 		{
-			// We must only update the log flush position once the snapshot write succeeds
-			if (FlushLockPosition)
-			{
-				std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName);
+			const std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName);
 
-				if (std::filesystem::is_regular_file(LogPath))
+			if (IsFile(LogPath))
+			{
+				m_SlogFile.Close();
+				if (!RemoveFile(LogPath, Ec) || Ec)
 				{
-					if (!std::filesystem::remove(LogPath, Ec) || Ec)
-					{
-						ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'",
-								 LogPath,
-								 IndexPath,
-								 Ec.message());
-						std::error_code RemoveIndexEc;
-						std::filesystem::remove(IndexPath, RemoveIndexEc);
-					}
+					// This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in
+					// the end it will be the same result
+					ZEN_WARN("snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message());
 				}
-			}
-			if (!Ec)
-			{
-				m_LogFlushPosition = LogCount;
+				m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite);
 			}
 		}
+		m_LogFlushPosition = IndexLogPosition;
 	}
 	catch (const std::exception& Err)
 	{
@@ -939,7 +965,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const
 {
 	ZEN_TRACE_CPU("Z$::Bucket::ReadIndexFile");
 
-	if (!std::filesystem::is_regular_file(IndexPath))
+	if (!IsFile(IndexPath))
 	{
 		return 0;
 	}
@@ -1023,7 +1049,7 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std::
 {
 	ZEN_TRACE_CPU("Z$::Bucket::ReadLog");
 
-	if (!std::filesystem::is_regular_file(LogPath))
+	if (!IsFile(LogPath))
 	{
 		return 0;
 	}
@@ -1103,47 +1129,40 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco
 
 	if (IsNew)
 	{
-		fs::remove(LogPath);
-		fs::remove(IndexPath);
-		fs::remove_all(m_BlocksBasePath);
+		RemoveFile(LogPath);
+		RemoveFile(IndexPath);
+		DeleteDirectories(m_BlocksBasePath);
 	}
 
 	CreateDirectories(m_BucketDir);
 
 	m_BlockStore.Initialize(m_BlocksBasePath, m_Configuration.MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1);
 
-	if (std::filesystem::is_regular_file(IndexPath))
+	if (IsFile(IndexPath))
 	{
 		uint32_t IndexVersion = 0;
 		m_LogFlushPosition	  = ReadIndexFile(IndexLock, IndexPath, IndexVersion);
 		if (IndexVersion == 0)
 		{
 			ZEN_WARN("removing invalid index file at '{}'", IndexPath);
-			std::filesystem::remove(IndexPath);
+			RemoveFile(IndexPath);
 		}
 	}
 
 	uint64_t LogEntryCount = 0;
-	if (std::filesystem::is_regular_file(LogPath))
+	if (IsFile(LogPath))
 	{
 		if (TCasLogFile<DiskIndexEntry>::IsValid(LogPath))
 		{
 			LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition);
 		}
-		else if (fs::is_regular_file(LogPath))
+		else if (IsFile(LogPath))
 		{
 			ZEN_WARN("removing invalid log at '{}'", LogPath);
-			std::filesystem::remove(LogPath);
+			RemoveFile(LogPath);
 		}
 	}
 
-	if (IsNew || LogEntryCount > 0 || m_LogFlushPosition != 0)
-	{
-		WriteIndexSnapshot(IndexLock, /*Flush log*/ true);
-	}
-
-	m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite);
-
 	BlockStore::BlockIndexSet KnownBlocks;
 	for (const auto& Entry : m_Index)
 	{
@@ -1161,7 +1180,53 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco
 			KnownBlocks.insert(BlockIndex);
 		}
 	}
-	m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
+	BlockStore::BlockIndexSet MissingBlocks = m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
+	m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite);
+
+	bool RemovedEntries = false;
+	if (!MissingBlocks.empty())
+	{
+		std::vector<DiskIndexEntry> MissingEntries;
+
+		for (auto& It : m_Index)
+		{
+			BucketPayload& Payload	= m_Payloads[It.second];
+			DiskLocation   Location = Payload.Location;
+			if (!Location.IsFlagSet(DiskLocation::kStandaloneFile))
+			{
+				if (MissingBlocks.contains(Location.Location.BlockLocation.GetBlockIndex()))
+				{
+					RemoveMemCachedData(IndexLock, Payload);
+					RemoveMetaData(IndexLock, Payload);
+				}
+			}
+			Location.Flags |= DiskLocation::kTombStone;
+			MissingEntries.push_back(DiskIndexEntry{.Key = It.first, .Location = Location});
+		}
+
+		ZEN_ASSERT(!MissingEntries.empty());
+
+		for (const DiskIndexEntry& Entry : MissingEntries)
+		{
+			m_Index.erase(Entry.Key);
+		}
+		m_SlogFile.Append(MissingEntries);
+		m_SlogFile.Flush();
+		{
+			std::vector<BucketPayload>	Payloads;
+			std::vector<AccessTime>		AccessTimes;
+			std::vector<BucketMetaData> MetaDatas;
+			std::vector<MemCacheData>	MemCachedPayloads;
+			IndexMap					Index;
+			CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
+		}
+		RemovedEntries = true;
+	}
+
+	if (IsNew || LogEntryCount > 0 || m_LogFlushPosition != 0 || RemovedEntries)
+	{
+		WriteIndexSnapshot(IndexLock, m_SlogFile.GetLogCount(), /*Flush log*/ true);
+	}
 }
 
 void
@@ -1424,6 +1489,13 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept
 							}
 						}
 					}
+					else
+					{
+						if (m_Configuration.MemCacheSizeThreshold > 0)
+						{
+							m_MemoryMissCount++;
+						}
+					}
 				}
 			}
 
@@ -1432,7 +1504,7 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept
 			// Often we will find the metadata due to the thread setting the mem cached part doing it before us so it is worth
 			// checking if it is present once more before spending time fetching and setting the RawHash and RawSize in metadata
 
-			auto FillOne = [&](const DiskLocation& Location, size_t KeyIndex, IoBuffer&& Value) {
+			auto FillOne = [&](const DiskLocation& Location, size_t KeyIndex, IoBuffer&& Value, bool UsesTemporaryMemory) {
 				if (!Value)
 				{
 					return;
@@ -1455,6 +1527,12 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept
 					}
 				}
 
+				if (AddToMemCache || UsesTemporaryMemory)
+				{
+					// We need to own it if we want to add it to the memcache or the buffer is just a range of the block iteration buffer
+					OutValue.Value.MakeOwned();
+				}
+
 				if (SetMetaInfo)
 				{
 					// See ZenCacheDiskLayer::CacheBucket::Get - it sets the memcache part first and then if it needs to it set the
@@ -1526,35 +1604,42 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept
 			if (!InlineDiskLocations.empty())
 			{
 				ZEN_TRACE_CPU("Z$::Bucket::EndGetBatch::ReadInline");
-				m_BlockStore.IterateChunks(
-					std::span{begin(InlineBlockLocations), end(InlineBlockLocations)},
-					[&](uint32_t, std::span<const size_t> ChunkIndexes) -> bool {
-						// Only read into memory the IoBuffers we could potentially add to memcache
-						const uint64_t LargeChunkSizeLimit = Max(m_Configuration.MemCacheSizeThreshold, 1u * 1024u);
-						m_BlockStore.IterateBlock(
-							std::span{begin(InlineBlockLocations), end(InlineBlockLocations)},
-							ChunkIndexes,
-							[this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t	  ChunkIndex,
-																					  const void* Data,
-																					  uint64_t	  Size) -> bool {
-								if (Data != nullptr)
-								{
-									FillOne(InlineDiskLocations[ChunkIndex],
-											InlineKeyIndexes[ChunkIndex],
-											IoBufferBuilder::MakeCloneFromMemory(Data, Size));
-								}
-								return true;
-							},
-							[this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t		  ChunkIndex,
-																					  BlockStoreFile& File,
-																					  uint64_t		  Offset,
-																					  uint64_t		  Size) -> bool {
-								FillOne(InlineDiskLocations[ChunkIndex], InlineKeyIndexes[ChunkIndex], File.GetChunk(Offset, Size));
-								return true;
-							},
-							LargeChunkSizeLimit);
-						return true;
-					});
+				m_BlockStore.IterateChunks(std::span{begin(InlineBlockLocations), end(InlineBlockLocations)},
+										   [&](uint32_t, std::span<const size_t> ChunkIndexes) -> bool {
+											   // Up to 8KB or m_Configuration.MemCacheSizeThreshold depending on configuration
+											   const uint64_t LargeChunkSizeLimit =
+												   m_Configuration.MemCacheSizeThreshold == 0
+													   ? Min(m_Configuration.LargeObjectThreshold, 8u * 1024u)
+													   : Max(m_Configuration.MemCacheSizeThreshold, 8u * 1024u);
+
+											   m_BlockStore.IterateBlock(
+												   std::span{begin(InlineBlockLocations), end(InlineBlockLocations)},
+												   ChunkIndexes,
+												   [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t		 ChunkIndex,
+																											 const void* Data,
+																											 uint64_t	 Size) -> bool {
+													   if (Data != nullptr)
+													   {
+														   FillOne(InlineDiskLocations[ChunkIndex],
+																   InlineKeyIndexes[ChunkIndex],
+																   IoBufferBuilder::MakeFromMemory(MemoryView(Data, Size)),
+																   /*UsesTemporaryMemory*/ true);
+													   }
+													   return true;
+												   },
+												   [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t			 ChunkIndex,
+																											 BlockStoreFile& File,
+																											 uint64_t		 Offset,
+																											 uint64_t		 Size) -> bool {
+													   FillOne(InlineDiskLocations[ChunkIndex],
+															   InlineKeyIndexes[ChunkIndex],
+															   File.GetChunk(Offset, Size),
+															   /*UsesTemporaryMemory*/ false);
+													   return true;
+												   },
+												   LargeChunkSizeLimit);
+											   return true;
+										   });
 			}
 
 			if (!StandaloneDiskLocations.empty())
@@ -1564,7 +1649,7 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept
 				{
 					size_t				KeyIndex = StandaloneKeyIndexes[Index];
 					const DiskLocation& Location = StandaloneDiskLocations[Index];
-					FillOne(Location, KeyIndex, GetStandaloneCacheValue(Location, Batch->Keys[KeyIndex]));
+					FillOne(Location, KeyIndex, GetStandaloneCacheValue(Location, Batch->Keys[KeyIndex]), /*UsesTemporaryMemory*/ false);
 				}
 			}
 
@@ -1644,10 +1729,6 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept
 				else
 				{
 					m_DiskMissCount++;
-					if (m_Configuration.MemCacheSizeThreshold > 0)
-					{
-						m_MemoryMissCount++;
-					}
 				}
 			}
 		}
@@ -1918,11 +1999,13 @@ ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock
 	}
 }
 
-bool
+std::function<void()>
 ZenCacheDiskLayer::CacheBucket::Drop()
 {
 	ZEN_TRACE_CPU("Z$::Bucket::Drop");
 
+	m_Gc.RemoveGcReferencer(*this);
+
 	RwLock::ExclusiveLockScope _(m_IndexLock);
 
 	std::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks;
@@ -1934,7 +2017,7 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_BlockStore.Close();
 	m_SlogFile.Close();
 
-	const bool Deleted = cache::impl::MoveAndDeleteDirectory(m_BucketDir);
+	std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(m_BucketDir);
 
 	m_Index.clear();
 	m_Payloads.clear();
@@ -1947,7 +2030,21 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_OuterCacheMemoryUsage.fetch_sub(m_MemCachedSize.load());
 	m_MemCachedSize.store(0);
 
-	return Deleted;
+	if (DroppedPath.empty())
+	{
+		return {};
+	}
+	else
+	{
+		return [DroppedPath = std::move(DroppedPath)]() {
+			std::error_code Ec;
+			(void)DeleteDirectories(DroppedPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message());
+			}
+		};
+	}
 }
 
 void
@@ -1982,6 +2079,9 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 	ZEN_TRACE_CPU("Z$::Bucket::SaveSnapshot");
 	try
 	{
+		// Be defensive regarding log position as it is written to without acquiring m_LocationMapLock
+		const uint64_t LogPosition = m_SlogFile.GetLogCount();
+
 		bool UseLegacyScheme = false;
 
 		IoBuffer							  Buffer;
@@ -1996,7 +2096,7 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 
 			{
 				RwLock::SharedLockScope IndexLock(m_IndexLock);
-				WriteIndexSnapshot(IndexLock, /*Flush log*/ false);
+				WriteIndexSnapshot(IndexLock, LogPosition, /*Flush log*/ false);
 				// Note: this copy could be eliminated on shutdown to
 				// reduce memory usage and execution time
 				Index		= m_Index;
@@ -2036,7 +2136,7 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 		else
 		{
 			RwLock::SharedLockScope IndexLock(m_IndexLock);
-			WriteIndexSnapshot(IndexLock, /*Flush log*/ false);
+			WriteIndexSnapshot(IndexLock, LogPosition, /*Flush log*/ false);
 			const uint64_t EntryCount = m_Index.size();
 			Buffer					  = ManifestWriter.MakeSidecarManifest(m_BucketId, EntryCount);
 			uint64_t SidecarSize	  = ManifestWriter.GetSidecarSize();
@@ -2146,7 +2246,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 					RwLock::SharedLockScope ValueLock(LockForHash(HashKey));
 
 					std::error_code Ec;
-					uintmax_t		size = std::filesystem::file_size(DataFilePath.ToPath(), Ec);
+					uintmax_t		size = FileSizeFromPath(DataFilePath.ToPath(), Ec);
 					if (Ec)
 					{
 						ReportBadKey(HashKey);
@@ -2287,11 +2387,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 					BuildPath(Path, Entry.Key);
 					fs::path				   FilePath = Path.ToPath();
 					RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key));
-					if (fs::is_regular_file(FilePath))
+					if (IsFile(FilePath))
 					{
 						ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8());
 						std::error_code Ec;
-						fs::remove(FilePath, Ec);  // We don't care if we fail, we are no longer tracking this file...
+						RemoveFile(FilePath, Ec);  // We don't care if we fail, we are no longer tracking this file...
 					}
 				}
 			}
@@ -2424,7 +2524,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 		 if (CleanUpTempFile)
 		 {
 			 std::error_code Ec;
-			 std::filesystem::remove(DataFile.GetPath(), Ec);
+			 RemoveFile(DataFile.GetPath(), Ec);
 			 if (Ec)
 			 {
 				 ZEN_WARN("Failed to clean up temporary file '{}' for put in '{}', reason '{}'",
@@ -2452,7 +2552,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 	RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey));
 
 	// We do a speculative remove of the file instead of probing with a exists call and check the error code instead
-	std::filesystem::remove(FsPath, Ec);
+	RemoveFile(FsPath, Ec);
 	if (Ec)
 	{
 		if (Ec.value() != ENOENT)
@@ -2460,7 +2560,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 			ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message());
 			Sleep(100);
 			Ec.clear();
-			std::filesystem::remove(FsPath, Ec);
+			RemoveFile(FsPath, Ec);
 			if (Ec && Ec.value() != ENOENT)
 			{
 				throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir));
@@ -2685,7 +2785,6 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash&		 HashKey,
 								ZEN_MEMSCOPE(GetCacheDiskTag());
 								ZEN_TRACE_CPU("Z$::Bucket::UpdateLocation");
 								DiskLocation Location(BlockStoreLocation, m_Configuration.PayloadAlignment, EntryFlags);
-								m_SlogFile.Append({.Key = HashKey, .Location = Location});
 
 								RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 								if (m_TrackedCacheKeys)
@@ -2715,6 +2814,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash&		 HashKey,
 									m_AccessTimes.emplace_back(GcClock::TickCount());
 									m_Index.insert_or_assign(HashKey, EntryIndex);
 								}
+								m_SlogFile.Append({.Key = HashKey, .Location = Location});
 							});
 }
 
@@ -2731,9 +2831,10 @@ class DiskBucketStoreCompactor : public GcStoreCompactor
 	using CacheBucket	= ZenCacheDiskLayer::CacheBucket;
 
 public:
-	DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
+	DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys, bool FlushBucket)
 	: m_Bucket(Bucket)
 	, m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys))
+	, m_FlushBucket(FlushBucket)
 	{
 		m_ExpiredStandaloneKeys.shrink_to_fit();
 	}
@@ -2791,7 +2892,7 @@ public:
 					ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8());
 
 					std::error_code Ec;
-					if (!fs::remove(FilePath, Ec))
+					if (!RemoveFile(FilePath, Ec))
 					{
 						continue;
 					}
@@ -2812,7 +2913,7 @@ public:
 					ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': checking standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8());
 
 					std::error_code Ec;
-					bool			Existed = std::filesystem::is_regular_file(FilePath, Ec);
+					bool			Existed = IsFile(FilePath, Ec);
 					if (Ec)
 					{
 						ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'",
@@ -2912,10 +3013,12 @@ public:
 						m_Bucket.m_BlockStore.CompactBlocks(
 							BlockCompactState,
 							m_Bucket.m_Configuration.PayloadAlignment,
-							[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
+							[&](const BlockStore::MovedChunksArray& MovedArray,
+								const BlockStore::ChunkIndexArray&	ScrubbedArray,
+								uint64_t							FreedDiskSpace) {
 								std::vector<DiskIndexEntry> MovedEntries;
 								MovedEntries.reserve(MovedArray.size());
-								RwLock::ExclusiveLockScope _(m_Bucket.m_IndexLock);
+								RwLock::ExclusiveLockScope IndexLock(m_Bucket.m_IndexLock);
 								for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
 								{
 									size_t		  ChunkIndex = Moved.first;
@@ -2937,6 +3040,24 @@ public:
 										MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
 									}
 								}
+
+								for (size_t ScrubbedIndex : ScrubbedArray)
+								{
+									const IoHash& Key = BlockCompactStateKeys[ScrubbedIndex];
+
+									if (auto It = m_Bucket.m_Index.find(Key); It != m_Bucket.m_Index.end())
+									{
+										BucketPayload& Payload	= m_Bucket.m_Payloads[It->second];
+										DiskLocation   Location = Payload.Location;
+
+										m_Bucket.RemoveMemCachedData(IndexLock, Payload);
+										m_Bucket.RemoveMetaData(IndexLock, Payload);
+
+										Location.Flags |= DiskLocation::kTombStone;
+										MovedEntries.push_back(DiskIndexEntry{.Key = Key, .Location = Location});
+									}
+								}
+
 								m_Bucket.m_SlogFile.Append(MovedEntries);
 								Stats.RemovedDisk += FreedDiskSpace;
 								if (Ctx.IsCancelledFlag.load())
@@ -2960,6 +3081,10 @@ public:
 				}
 			}
 		}
+		if (m_FlushBucket)
+		{
+			m_Bucket.Flush();
+		}
 	}
 
 	virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); }
@@ -2967,6 +3092,7 @@ public:
 private:
 	ZenCacheDiskLayer::CacheBucket&			 m_Bucket;
 	std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys;
+	bool									 m_FlushBucket = false;
 };
 
 GcStoreCompactor*
@@ -2990,24 +3116,6 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 					 NiceBytes(Stats.FreedMemory),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		}
-		if (Stats.DeletedCount > 0)
-		{
-			bool Expected = false;
-			if (m_IsFlushing || !m_IsFlushing.compare_exchange_strong(Expected, true))
-			{
-				return;
-			}
-			auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); });
-
-			try
-			{
-				SaveSnapshot([]() { return 0; });
-			}
-			catch (const std::exception& Ex)
-			{
-				ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what());
-			}
-		}
 	});
 
 	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -3059,7 +3167,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 			return nullptr;
 		}
 
-		if (Ctx.Settings.IsDeleteMode)
+		if (Ctx.Settings.IsDeleteMode && !ExpiredEntries.empty())
 		{
 			for (const DiskIndexEntry& Entry : ExpiredEntries)
 			{
@@ -3094,7 +3202,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		return nullptr;
 	}
 
-	return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys));
+	return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys), /*FlushBucket*/ Stats.DeletedCount > 0);
 }
 
 bool
@@ -3284,7 +3392,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef&	   Logger,
 						CaptureAttachments(ChunkIndex, File.GetChunk(Offset, Size).GetView());
 						return !IsCancelledFlag.load();
 					},
-					0);
+					32u * 1024);
 
 				if (Continue)
 				{
@@ -3583,20 +3691,6 @@ ZenCacheDiskLayer::~ZenCacheDiskLayer()
 	}
 }
 
-template<typename T, typename U>
-struct equal_to_2 : public eastl::binary_function<T, U, bool>
-{
-	constexpr bool operator()(const T& a, const U& b) const { return a == b; }
-
-	template<typename T_ = T,
-			 typename U_ = U,
-			 typename	 = eastl::enable_if_t<!eastl::is_same_v<eastl::remove_const_t<T_>, eastl::remove_const_t<U_>>>>
-	constexpr bool operator()(const U& b, const T& a) const
-	{
-		return b == a;
-	}
-};
-
 ZenCacheDiskLayer::CacheBucket*
 ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket)
 {
@@ -3604,7 +3698,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket)
 
 	{
 		RwLock::SharedLockScope SharedLock(m_Lock);
-		if (auto It = m_Buckets.find_as(InBucket, std::hash<std::string_view>(), equal_to_2<std::string, std::string_view>());
+		if (auto It = m_Buckets.find_as(InBucket, std::hash<std::string_view>(), eastl::equal_to_2<std::string, std::string_view>());
 			It != m_Buckets.end())
 		{
 			return It->second.get();
@@ -3613,10 +3707,18 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket)
 
 	// We create the bucket without holding a lock since contructor calls GcManager::AddGcReferencer which takes an exclusive lock.
 	// This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock
-	std::unique_ptr<CacheBucket> Bucket(std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, InBucket, m_Configuration.BucketConfig));
+	BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig;
+	if (auto It = m_Configuration.BucketConfigMap.find_as(InBucket,
+														  std::hash<std::string_view>(),
+														  eastl::equal_to_2<std::string, std::string_view>());
+		It != m_Configuration.BucketConfigMap.end())
+	{
+		BucketConfig = &It->second;
+	}
+	std::unique_ptr<CacheBucket> Bucket(std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, InBucket, *BucketConfig));
 
 	RwLock::ExclusiveLockScope Lock(m_Lock);
-	if (auto It = m_Buckets.find_as(InBucket, std::hash<std::string_view>(), equal_to_2<std::string, std::string_view>());
+	if (auto It = m_Buckets.find_as(InBucket, std::hash<std::string_view>(), eastl::equal_to_2<std::string, std::string_view>());
 		It != m_Buckets.end())
 	{
 		return It->second.get();
@@ -3896,7 +3998,11 @@ ZenCacheDiskLayer::DiscoverBuckets()
 		if (IsKnownBadBucketName(BucketName))
 		{
 			BadBucketDirectories.push_back(BucketPath);
-
+			continue;
+		}
+		else if (BucketName.starts_with("[dropped]"))
+		{
+			BadBucketDirectories.push_back(BucketPath);
 			continue;
 		}
 
@@ -3929,50 +4035,66 @@ ZenCacheDiskLayer::DiscoverBuckets()
 	RwLock SyncLock;
 
 	WorkerThreadPool& Pool = GetLargeWorkerPool(EWorkloadType::Burst);
-	Latch			  WorkLatch(1);
-	for (auto& BucketPath : FoundBucketDirectories)
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	try
 	{
-		WorkLatch.AddCount(1);
-		Pool.ScheduleWork([this, &WorkLatch, &SyncLock, BucketPath]() {
-			ZEN_MEMSCOPE(GetCacheDiskTag());
-
-			auto			  _			 = MakeGuard([&]() { WorkLatch.CountDown(); });
-			const std::string BucketName = PathToUtf8(BucketPath.stem());
-			try
-			{
-				std::unique_ptr<CacheBucket> NewBucket =
-					std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig);
+		for (auto& BucketPath : FoundBucketDirectories)
+		{
+			Work.ScheduleWork(Pool, [this, &SyncLock, BucketPath](std::atomic<bool>&) {
+				ZEN_MEMSCOPE(GetCacheDiskTag());
 
-				CacheBucket* Bucket = nullptr;
+				const std::string BucketName = PathToUtf8(BucketPath.stem());
+				try
 				{
-					RwLock::ExclusiveLockScope __(SyncLock);
-					auto					   InsertResult = m_Buckets.emplace(BucketName, std::move(NewBucket));
-					Bucket									= InsertResult.first->second.get();
-				}
-				ZEN_ASSERT(Bucket);
+					BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig;
+					if (auto It = m_Configuration.BucketConfigMap.find_as(std::string_view(BucketName),
+																		  std::hash<std::string_view>(),
+																		  eastl::equal_to_2<std::string, std::string_view>());
+						It != m_Configuration.BucketConfigMap.end())
+					{
+						BucketConfig = &It->second;
+					}
 
-				if (!Bucket->OpenOrCreate(BucketPath, /* AllowCreate */ false))
-				{
-					ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
+					std::unique_ptr<CacheBucket> NewBucket =
+						std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, *BucketConfig);
 
+					CacheBucket* Bucket = nullptr;
 					{
 						RwLock::ExclusiveLockScope __(SyncLock);
-						m_Buckets.erase(BucketName);
+						auto					   InsertResult = m_Buckets.emplace(BucketName, std::move(NewBucket));
+						Bucket									= InsertResult.first->second.get();
+					}
+					ZEN_ASSERT(Bucket);
+
+					if (!Bucket->OpenOrCreate(BucketPath, /* AllowCreate */ false))
+					{
+						ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
+
+						{
+							RwLock::ExclusiveLockScope __(SyncLock);
+							m_Buckets.erase(BucketName);
+						}
 					}
 				}
-			}
-			catch (const std::exception& Err)
-			{
-				ZEN_ERROR("Opening bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
-				return;
-			}
-		});
+				catch (const std::exception& Err)
+				{
+					ZEN_ERROR("Opening bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
+					return;
+				}
+			});
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		AbortFlag.store(true);
+		ZEN_WARN("Failed discovering buckets in {}. Reason: '{}'", m_RootDir, Ex.what());
 	}
-	WorkLatch.CountDown();
-	WorkLatch.Wait();
+	Work.Wait();
 }
 
-bool
+std::function<void()>
 ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
 {
 	ZEN_TRACE_CPU("Z$::DropBucket");
@@ -3990,33 +4112,72 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
 		return Bucket.Drop();
 	}
 
-	// Make sure we remove the folder even if we don't know about the bucket
 	std::filesystem::path BucketPath = m_RootDir;
 	BucketPath /= std::string(InBucket);
-	return cache::impl::MoveAndDeleteDirectory(BucketPath);
+	std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(BucketPath);
+	if (DroppedPath.empty())
+	{
+		return {};
+	}
+	else
+	{
+		return [DroppedPath = std::move(DroppedPath)]() {
+			std::error_code Ec;
+			(void)DeleteDirectories(DroppedPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message());
+			}
+		};
+	}
 }
 
-bool
+std::function<void()>
 ZenCacheDiskLayer::Drop()
 {
 	ZEN_TRACE_CPU("Z$::Drop");
 
-	RwLock::ExclusiveLockScope _(m_Lock);
-
-	std::vector<std::unique_ptr<CacheBucket>> Buckets;
-	Buckets.reserve(m_Buckets.size());
-	while (!m_Buckets.empty())
+	std::vector<std::function<void()>> PostDropOps;
 	{
-		const auto&	 It		= m_Buckets.begin();
-		CacheBucket& Bucket = *It->second;
-		m_DroppedBuckets.push_back(std::move(It->second));
-		m_Buckets.erase(It->first);
-		if (!Bucket.Drop())
+		RwLock::ExclusiveLockScope _(m_Lock);
+		PostDropOps.reserve(m_Buckets.size());
+		while (!m_Buckets.empty())
 		{
-			return false;
+			const auto&	 It		= m_Buckets.begin();
+			CacheBucket& Bucket = *It->second;
+			m_DroppedBuckets.push_back(std::move(It->second));
+			m_Buckets.erase(It->first);
+			if (std::function<void()> PostDropOp = Bucket.Drop(); !PostDropOp)
+			{
+				return {};
+			}
+			else
+			{
+				PostDropOps.emplace_back(std::move(PostDropOp));
+			}
 		}
 	}
-	return cache::impl::MoveAndDeleteDirectory(m_RootDir);
+
+	std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(m_RootDir);
+	if (DroppedPath.empty())
+	{
+		return {};
+	}
+	else
+	{
+		return [DroppedPath = std::move(DroppedPath), PostDropOps = std::move(PostDropOps)]() {
+			for (auto& PostDropOp : PostDropOps)
+			{
+				PostDropOp();
+			}
+			std::error_code Ec;
+			(void)DeleteDirectories(DroppedPath, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message());
+			}
+		};
+	}
 }
 
 void
@@ -4046,16 +4207,16 @@ ZenCacheDiskLayer::Flush()
 	}
 	{
 		WorkerThreadPool& Pool = GetMediumWorkerPool(EWorkloadType::Burst);
-		Latch			  WorkLatch(1);
+		std::atomic<bool> AbortFlag;
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
 		try
 		{
 			for (auto& Bucket : Buckets)
 			{
-				WorkLatch.AddCount(1);
-				Pool.ScheduleWork([&WorkLatch, Bucket]() {
+				Work.ScheduleWork(Pool, [Bucket](std::atomic<bool>&) {
 					ZEN_MEMSCOPE(GetCacheDiskTag());
 
-					auto _ = MakeGuard([&]() { WorkLatch.CountDown(); });
 					try
 					{
 						Bucket->Flush();
@@ -4069,13 +4230,14 @@ ZenCacheDiskLayer::Flush()
 		}
 		catch (const std::exception& Ex)
 		{
+			AbortFlag.store(true);
 			ZEN_ERROR("Failed to flush buckets at '{}'. Reason: '{}'", m_RootDir, Ex.what());
 		}
-		WorkLatch.CountDown();
-		while (!WorkLatch.Wait(1000))
-		{
-			ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", WorkLatch.Remaining(), m_RootDir);
-		}
+
+		Work.Wait(1000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t RemainingWork) {
+			ZEN_UNUSED(IsAborted, IsPaused);
+			ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", RemainingWork, m_RootDir);
+		});
 	}
 }
 
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index 97e26a38d..de4b0a37c 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -220,6 +220,11 @@ CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context,
 				ZEN_WARN("Content format not supported, expected package message format");
 				return RpcResponseCode::BadRequest;
 			}
+			if (CbValidateError Error = ValidateCompactBinary(Object.GetView(), CbValidateMode::Default); Error != CbValidateError::None)
+			{
+				ZEN_WARN("Content format is corrupt, compact binary format validation failed. Reason: '{}'", ToString(Error));
+				return RpcResponseCode::BadRequest;
+			}
 		}
 
 		if (!UriNamespace.empty())
@@ -558,6 +563,13 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
 			{
 				FoundLocalInvalid = true;
 			}
+			else if (CbValidateError Error = ValidateCompactBinary(Request.RecordCacheValue.GetView(), CbValidateMode::Default);
+					 Error != CbValidateError::None)
+			{
+				ZEN_WARN("HandleRpcGetCacheRecords stored record is corrupt, compact binary format validation failed. Reason: '{}'",
+						 ToString(Error));
+				FoundLocalInvalid = true;
+			}
 			else
 			{
 				Request.RecordObject = CbObjectView(Request.RecordCacheValue.GetData());
@@ -641,7 +653,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
 				{
 					m_CidStore.IterateChunks(
 						CidHashes,
-						[this, &Request, ValueCount, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool {
+						[this, &Request, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool {
 							try
 							{
 								const size_t	  ValueIndex = RequestValueIndexes[Index];
@@ -1563,18 +1575,27 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext&				 Context,
 				Record.ValuesRead = true;
 				if (Record.CacheValue && Record.CacheValue.GetContentType() == ZenContentType::kCbObject)
 				{
-					CbObjectView RecordObject = CbObjectView(Record.CacheValue.GetData());
-					CbArrayView	 ValuesArray  = RecordObject["Values"sv].AsArrayView();
-					Record.Values.reserve(ValuesArray.Num());
-					for (CbFieldView ValueField : ValuesArray)
+					if (CbValidateError Error = ValidateCompactBinary(Record.CacheValue.GetView(), CbValidateMode::Default);
+						Error != CbValidateError::None)
+					{
+						ZEN_WARN("GetLocalCacheRecords stored record for is corrupt, compact binary format validation failed. Reason: '{}'",
+								 ToString(Error));
+					}
+					else
 					{
-						CbObjectView ValueObject  = ValueField.AsObjectView();
-						Oid			 ValueId	  = ValueObject["Id"sv].AsObjectId();
-						CbFieldView	 RawHashField = ValueObject["RawHash"sv];
-						IoHash		 RawHash	  = RawHashField.AsBinaryAttachment();
-						if (ValueId && !RawHashField.HasError())
+						CbObjectView RecordObject = CbObjectView(Record.CacheValue.GetData());
+						CbArrayView	 ValuesArray  = RecordObject["Values"sv].AsArrayView();
+						Record.Values.reserve(ValuesArray.Num());
+						for (CbFieldView ValueField : ValuesArray)
 						{
-							Record.Values.push_back({ValueId, RawHash, ValueObject["RawSize"sv].AsUInt64()});
+							CbObjectView ValueObject  = ValueField.AsObjectView();
+							Oid			 ValueId	  = ValueObject["Id"sv].AsObjectId();
+							CbFieldView	 RawHashField = ValueObject["RawHash"sv];
+							IoHash		 RawHash	  = RawHashField.AsBinaryAttachment();
+							if (ValueId && !RawHashField.HasError())
+							{
+								Record.Values.push_back({ValueId, RawHash, ValueObject["RawSize"sv].AsUInt64()});
+							}
 						}
 					}
 				}
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index 7d277329e..d956384ca 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -277,11 +277,14 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket)
 {
 	ZEN_INFO("dropping bucket '{}'", Bucket);
 
-	const bool Dropped = m_DiskLayer.DropBucket(Bucket);
-
-	ZEN_INFO("bucket '{}' was {}", Bucket, Dropped ? "dropped" : "not found");
-
-	return Dropped;
+	std::function<void()> PostDropOp = m_DiskLayer.DropBucket(Bucket);
+	if (!PostDropOp)
+	{
+		ZEN_INFO("bucket '{}' was not found in {}", Bucket, m_RootDir);
+		return false;
+	}
+	PostDropOp();
+	return true;
 }
 
 void
@@ -291,9 +294,10 @@ ZenCacheNamespace::EnumerateBucketContents(std::string_view
 	m_DiskLayer.EnumerateBucketContents(Bucket, Fn);
 }
 
-bool
+std::function<void()>
 ZenCacheNamespace::Drop()
 {
+	m_Gc.RemoveGcStorage(this);
 	return m_DiskLayer.Drop();
 }
 
@@ -786,16 +790,27 @@ ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket)
 bool
 ZenCacheStore::DropNamespace(std::string_view InNamespace)
 {
-	RwLock::SharedLockScope _(m_NamespacesLock);
-	if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end())
+	std::function<void()> PostDropOp;
+	{
+		RwLock::SharedLockScope _(m_NamespacesLock);
+		if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end())
+		{
+			ZenCacheNamespace& Namespace = *It->second;
+			m_DroppedNamespaces.push_back(std::move(It->second));
+			m_Namespaces.erase(It);
+			PostDropOp = Namespace.Drop();
+		}
+		else
+		{
+			ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace);
+			return false;
+		}
+	}
+	if (PostDropOp)
 	{
-		ZenCacheNamespace& Namespace = *It->second;
-		m_DroppedNamespaces.push_back(std::move(It->second));
-		m_Namespaces.erase(It);
-		return Namespace.Drop();
+		PostDropOp();
 	}
-	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace);
-	return false;
+	return true;
 }
 
 void
diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp
index 73c10a6db..460f0e10d 100644
--- a/src/zenstore/cas.cpp
+++ b/src/zenstore/cas.cpp
@@ -118,7 +118,7 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig)
 
 	// Ensure root directory exists - create if it doesn't exist already
 
-	std::filesystem::create_directories(m_Config.RootDirectory);
+	CreateDirectories(m_Config.RootDirectory);
 
 	// Open or create manifest
 
@@ -412,6 +412,7 @@ CasImpl::IterateChunks(std::span<IoHash>												 DecompressedIds,
 					   uint64_t															 LargeSizeLimit)
 {
 	ZEN_TRACE_CPU("CAS::IterateChunks");
+
 	if (!m_SmallStrategy.IterateChunks(
 			DecompressedIds,
 			[&](size_t Index, const IoBuffer& Payload) {
@@ -420,10 +421,11 @@ CasImpl::IterateChunks(std::span<IoHash>												 DecompressedIds,
 				return AsyncCallback(Index, Payload);
 			},
 			OptionalWorkerPool,
-			LargeSizeLimit))
+			LargeSizeLimit == 0 ? m_Config.HugeValueThreshold : Min(LargeSizeLimit, m_Config.HugeValueThreshold)))
 	{
 		return false;
 	}
+
 	if (!m_TinyStrategy.IterateChunks(
 			DecompressedIds,
 			[&](size_t Index, const IoBuffer& Payload) {
@@ -432,10 +434,11 @@ CasImpl::IterateChunks(std::span<IoHash>												 DecompressedIds,
 				return AsyncCallback(Index, Payload);
 			},
 			OptionalWorkerPool,
-			LargeSizeLimit))
+			LargeSizeLimit == 0 ? m_Config.TinyValueThreshold : Min(LargeSizeLimit, m_Config.TinyValueThreshold)))
 	{
 		return false;
 	}
+
 	if (!m_LargeStrategy.IterateChunks(
 			DecompressedIds,
 			[&](size_t Index, const IoBuffer& Payload) {
diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp
index 6c7b1b297..492ce9317 100644
--- a/src/zenstore/caslog.cpp
+++ b/src/zenstore/caslog.cpp
@@ -37,7 +37,7 @@ CasLogFile::~CasLogFile()
 bool
 CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize)
 {
-	if (!std::filesystem::is_regular_file(FileName))
+	if (!IsFile(FileName))
 	{
 		return false;
 	}
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 2be0542db..b00abb2cb 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -15,6 +15,7 @@
 #include <zencore/trace.h>
 #include <zencore/workthreadpool.h>
 #include <zenstore/scrubcontext.h>
+#include <zenutil/parallelwork.h>
 
 #include <gsl/gsl-lite.hpp>
 
@@ -144,6 +145,16 @@ CasContainerStrategy::CasContainerStrategy(GcManager& Gc) : m_Log(logging::Get("
 
 CasContainerStrategy::~CasContainerStrategy()
 {
+	try
+	{
+		m_BlockStore.Close();
+		m_CasLog.Flush();
+		m_CasLog.Close();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("~CasContainerStrategy failed with: ", Ex.what());
+	}
 	m_Gc.RemoveGcReferenceStore(*this);
 	m_Gc.RemoveGcStorage(this);
 }
@@ -203,12 +214,12 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
 		ZEN_TRACE_CPU("CasContainer::UpdateLocation");
 		BlockStoreDiskLocation	DiskLocation(Location, m_PayloadAlignment);
 		const CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = DiskLocation};
-		m_CasLog.Append(IndexEntry);
 		{
 			RwLock::ExclusiveLockScope _(m_LocationMapLock);
 			m_LocationMap.emplace(ChunkHash, m_Locations.size());
 			m_Locations.push_back(DiskLocation);
 		}
+		m_CasLog.Append(IndexEntry);
 	});
 
 	return CasStore::InsertResult{.New = true};
@@ -226,7 +237,7 @@ CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 }
 
 std::vector<CasStore::InsertResult>
-CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes)
+CasContainerStrategy::InsertChunks(std::span<const IoBuffer> Chunks, std::span<const IoHash> ChunkHashes)
 {
 	ZEN_MEMSCOPE(GetCasContainerTag());
 
@@ -272,7 +283,6 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash>
 			IndexEntries.emplace_back(
 				CasDiskIndexEntry{.Key = ChunkHashes[ChunkIndex], .Location = BlockStoreDiskLocation(Location, m_PayloadAlignment)});
 		}
-		m_CasLog.Append(IndexEntries);
 		{
 			RwLock::ExclusiveLockScope _(m_LocationMapLock);
 			for (const CasDiskIndexEntry& DiskIndexEntry : IndexEntries)
@@ -281,6 +291,7 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash>
 				m_Locations.push_back(DiskIndexEntry.Location);
 			}
 		}
+		m_CasLog.Append(IndexEntries);
 	});
 	return Result;
 }
@@ -306,7 +317,12 @@ bool
 CasContainerStrategy::HaveChunk(const IoHash& ChunkHash)
 {
 	RwLock::SharedLockScope _(m_LocationMapLock);
-	return m_LocationMap.contains(ChunkHash);
+	if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end())
+	{
+		const BlockStoreLocation& Location = m_Locations[KeyIt->second].Get(m_PayloadAlignment);
+		return m_BlockStore.HasChunk(Location);
+	}
+	return false;
 }
 
 void
@@ -323,7 +339,7 @@ CasContainerStrategy::FilterChunks(HashKeySet& InOutChunks)
 }
 
 bool
-CasContainerStrategy::IterateChunks(std::span<IoHash>												  ChunkHashes,
+CasContainerStrategy::IterateChunks(std::span<const IoHash>											  ChunkHashes,
 									const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
 									WorkerThreadPool*												  OptionalWorkerPool,
 									uint64_t														  LargeSizeLimit)
@@ -360,7 +376,11 @@ CasContainerStrategy::IterateChunks(std::span<IoHash>												  ChunkHashes,
 		return true;
 	}
 
-	auto DoOneBlock = [&](std::span<const size_t> ChunkIndexes) {
+	auto DoOneBlock = [this](const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
+							 uint64_t														   LargeSizeLimit,
+							 std::span<const size_t>										   FoundChunkIndexes,
+							 std::span<const BlockStoreLocation>							   FoundChunkLocations,
+							 std::span<const size_t>										   ChunkIndexes) {
 		if (ChunkIndexes.size() < 4)
 		{
 			for (size_t ChunkIndex : ChunkIndexes)
@@ -376,57 +396,96 @@ CasContainerStrategy::IterateChunks(std::span<IoHash>												  ChunkHashes,
 		return m_BlockStore.IterateBlock(
 			FoundChunkLocations,
 			ChunkIndexes,
-			[&](size_t ChunkIndex, const void* Data, uint64_t Size) {
+			[AsyncCallback, FoundChunkIndexes](size_t ChunkIndex, const void* Data, uint64_t Size) {
 				if (Data == nullptr)
 				{
 					return AsyncCallback(FoundChunkIndexes[ChunkIndex], IoBuffer());
 				}
 				return AsyncCallback(FoundChunkIndexes[ChunkIndex], IoBuffer(IoBuffer::Wrap, Data, Size));
 			},
-			[&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+			[AsyncCallback, FoundChunkIndexes](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
 				return AsyncCallback(FoundChunkIndexes[ChunkIndex], File.GetChunk(Offset, Size));
 			},
 			LargeSizeLimit);
 	};
 
-	Latch			 WorkLatch(1);
-	std::atomic_bool AsyncContinue = true;
-	bool Continue = m_BlockStore.IterateChunks(FoundChunkLocations, [&](uint32_t BlockIndex, std::span<const size_t> ChunkIndexes) {
-		if (OptionalWorkerPool && (ChunkIndexes.size() > 3))
+	std::atomic<bool> AbortFlag;
+	{
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
+		try
 		{
-			WorkLatch.AddCount(1);
-			OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() {
-				auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
-				if (!AsyncContinue)
-				{
-					return;
-				}
-				try
-				{
-					bool Continue = DoOneBlock(ChunkIndexes);
-					if (!Continue)
+			const bool Continue = m_BlockStore.IterateChunks(
+				FoundChunkLocations,
+				[this,
+				 &Work,
+				 &AbortFlag,
+				 &AsyncCallback,
+				 LargeSizeLimit,
+				 DoOneBlock,
+				 &FoundChunkIndexes,
+				 &FoundChunkLocations,
+				 OptionalWorkerPool](uint32_t BlockIndex, std::span<const size_t> ChunkIndexes) {
+					if (OptionalWorkerPool && (ChunkIndexes.size() > 3))
 					{
-						AsyncContinue.store(false);
-					}
-				}
-				catch (const std::exception& Ex)
-				{
-					ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'",
-							 m_RootDirectory,
+						std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end());
+						Work.ScheduleWork(
+							*OptionalWorkerPool,
+							[this,
+							 &AsyncCallback,
+							 LargeSizeLimit,
+							 DoOneBlock,
 							 BlockIndex,
-							 Ex.what());
-				}
-			});
-			return AsyncContinue.load();
+							 &FoundChunkIndexes,
+							 &FoundChunkLocations,
+							 ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic<bool>& AbortFlag) {
+								if (AbortFlag)
+								{
+									return;
+								}
+								try
+								{
+									bool Continue =
+										DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes);
+									if (!Continue)
+									{
+										AbortFlag.store(true);
+									}
+								}
+								catch (const std::exception& Ex)
+								{
+									ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'",
+											 m_RootDirectory,
+											 BlockIndex,
+											 Ex.what());
+									AbortFlag.store(true);
+								}
+							});
+						return !AbortFlag.load();
+					}
+					else
+					{
+						if (!DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes))
+						{
+							AbortFlag.store(true);
+						}
+						return !AbortFlag.load();
+					}
+				});
+			if (!Continue)
+			{
+				AbortFlag.store(true);
+			}
 		}
-		else
+		catch (const std::exception& Ex)
 		{
-			return DoOneBlock(ChunkIndexes);
+			AbortFlag.store(true);
+			ZEN_WARN("Failed iterating chunks for cas root path {}. Reason: '{}'", m_RootDirectory, Ex.what());
 		}
-	});
-	WorkLatch.CountDown();
-	WorkLatch.Wait();
-	return AsyncContinue.load() && Continue;
+
+		Work.Wait();
+	}
+	return !AbortFlag.load();
 }
 
 void
@@ -437,7 +496,7 @@ CasContainerStrategy::Flush()
 	ZEN_TRACE_CPU("CasContainer::Flush");
 	m_BlockStore.Flush(/*ForceNewBlock*/ false);
 	m_CasLog.Flush();
-	MakeIndexSnapshot();
+	MakeIndexSnapshot(/*ResetLog*/ false);
 }
 
 void
@@ -677,7 +736,9 @@ public:
 						m_CasContainerStrategy.m_BlockStore.CompactBlocks(
 							BlockCompactState,
 							m_CasContainerStrategy.m_PayloadAlignment,
-							[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
+							[&](const BlockStore::MovedChunksArray& MovedArray,
+								const BlockStore::ChunkIndexArray&	ScrubbedArray,
+								uint64_t							FreedDiskSpace) {
 								std::vector<CasDiskIndexEntry> MovedEntries;
 								RwLock::ExclusiveLockScope	   _(m_CasContainerStrategy.m_LocationMapLock);
 								for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
@@ -702,7 +763,27 @@ public:
 										MovedEntries.push_back(CasDiskIndexEntry{.Key = Key, .Location = Location});
 									}
 								}
+								for (size_t ChunkIndex : ScrubbedArray)
+								{
+									const IoHash& Key = BlockCompactStateKeys[ChunkIndex];
+									if (auto It = m_CasContainerStrategy.m_LocationMap.find(Key);
+										It != m_CasContainerStrategy.m_LocationMap.end())
+									{
+										BlockStoreDiskLocation&	  Location	  = m_CasContainerStrategy.m_Locations[It->second];
+										const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex);
+										if (Location.Get(m_CasContainerStrategy.m_PayloadAlignment) != OldLocation)
+										{
+											// Someone has moved our chunk so lets just skip the new location we were provided, it will be
+											// GC:d at a later time
+											continue;
+										}
+										MovedEntries.push_back(
+											CasDiskIndexEntry{.Key = Key, .Location = Location, .Flags = CasDiskIndexEntry::kTombstone});
+										m_CasContainerStrategy.m_LocationMap.erase(It);
+									}
+								}
 								m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
+								m_CasContainerStrategy.m_CasLog.Flush();
 								Stats.RemovedDisk += FreedDiskSpace;
 								if (Ctx.IsCancelledFlag.load())
 								{
@@ -900,13 +981,12 @@ CasContainerStrategy::StorageSize() const
 }
 
 void
-CasContainerStrategy::MakeIndexSnapshot()
+CasContainerStrategy::MakeIndexSnapshot(bool ResetLog)
 {
 	ZEN_MEMSCOPE(GetCasContainerTag());
 	ZEN_TRACE_CPU("CasContainer::MakeIndexSnapshot");
 
-	uint64_t LogCount = m_CasLog.GetLogCount();
-	if (m_LogFlushPosition == LogCount)
+	if (m_LogFlushPosition == m_CasLog.GetLogCount())
 	{
 		return;
 	}
@@ -923,34 +1003,17 @@ CasContainerStrategy::MakeIndexSnapshot()
 
 	namespace fs = std::filesystem;
 
-	fs::path IndexPath	   = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName);
-	fs::path TempIndexPath = cas::impl::GetTempIndexPath(m_RootDirectory, m_ContainerBaseName);
-
-	// Move index away, we keep it if something goes wrong
-	if (fs::is_regular_file(TempIndexPath))
-	{
-		std::error_code Ec;
-		if (!fs::remove(TempIndexPath, Ec) || Ec)
-		{
-			ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message());
-			return;
-		}
-	}
+	const fs::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName);
 
 	try
 	{
-		if (fs::is_regular_file(IndexPath))
-		{
-			fs::rename(IndexPath, TempIndexPath);
-		}
-
 		// Write the current state of the location map to a new index state
 		std::vector<CasDiskIndexEntry> Entries;
-		uint64_t					   IndexLogPosition = 0;
+		// Be defensive regarding log position as it is written to without acquiring m_LocationMapLock
+		const uint64_t IndexLogPosition = ResetLog ? 0 : m_CasLog.GetLogCount();
 
 		{
 			RwLock::SharedLockScope ___(m_LocationMapLock);
-			IndexLogPosition = m_CasLog.GetLogCount();
 			Entries.resize(m_LocationMap.size());
 
 			uint64_t EntryIndex = 0;
@@ -960,6 +1023,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 				IndexEntry.Key				  = Entry.first;
 				IndexEntry.Location			  = m_Locations[Entry.second];
 			}
+			EntryCount = m_LocationMap.size();
 		}
 
 		TemporaryFile	ObjectIndexFile;
@@ -969,7 +1033,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 		{
 			throw std::system_error(Ec, fmt::format("Failed to create temp file for index snapshot at '{}'", IndexPath));
 		}
-		CasDiskIndexHeader Header = {.EntryCount	   = Entries.size(),
+		CasDiskIndexHeader Header = {.EntryCount	   = EntryCount,
 									 .LogPosition	   = IndexLogPosition,
 									 .PayloadAlignment = gsl::narrow<uint32_t>(m_PayloadAlignment)};
 
@@ -981,35 +1045,34 @@ CasContainerStrategy::MakeIndexSnapshot()
 		ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec);
 		if (Ec)
 		{
-			throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", ObjectIndexFile.GetPath(), IndexPath));
+			throw std::system_error(Ec,
+									fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'",
+												ObjectIndexFile.GetPath(),
+												IndexPath,
+												Ec.message()));
 		}
-		EntryCount		   = Entries.size();
-		m_LogFlushPosition = IndexLogPosition;
-	}
-	catch (const std::exception& Err)
-	{
-		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
-
-		// Restore any previous snapshot
 
-		if (fs::is_regular_file(TempIndexPath))
+		if (ResetLog)
 		{
-			std::error_code Ec;
-			fs::remove(IndexPath, Ec);	// We don't care if this fails, we try to move the old temp file regardless
-			fs::rename(TempIndexPath, IndexPath, Ec);
-			if (Ec)
+			const std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName);
+
+			if (IsFile(LogPath))
 			{
-				ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message());
+				m_CasLog.Close();
+				if (!RemoveFile(LogPath, Ec) || Ec)
+				{
+					// This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in
+					// the end it will be the same result
+					ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message());
+				}
+				m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
 			}
 		}
+		m_LogFlushPosition = IndexLogPosition;
 	}
-	if (fs::is_regular_file(TempIndexPath))
+	catch (const std::exception& Err)
 	{
-		std::error_code Ec;
-		if (!fs::remove(TempIndexPath, Ec) || Ec)
-		{
-			ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message());
-		}
+		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
 	}
 }
 
@@ -1092,7 +1155,7 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski
 	if (!TCasLogFile<CasDiskIndexEntry>::IsValid(LogPath))
 	{
 		ZEN_WARN("removing invalid cas log at '{}'", LogPath);
-		std::filesystem::remove(LogPath);
+		RemoveFile(LogPath);
 		return 0;
 	}
 
@@ -1115,7 +1178,7 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski
 			ZEN_WARN("reading full log at '{}', reason: Log position from index snapshot is out of range", LogPath);
 			SkipEntryCount = 0;
 		}
-		LogEntryCount = EntryCount - SkipEntryCount;
+		LogEntryCount = SkipEntryCount;
 		CasLog.Replay(
 			[&](const CasDiskIndexEntry& Record) {
 				LogEntryCount++;
@@ -1134,7 +1197,6 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski
 				m_Locations.push_back(Record.Location);
 			},
 			SkipEntryCount);
-
 		return LogEntryCount;
 	}
 	return 0;
@@ -1155,7 +1217,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 
 	if (IsNewStore)
 	{
-		std::filesystem::remove_all(BasePath);
+		DeleteDirectories(BasePath);
 	}
 
 	CreateDirectories(BasePath);
@@ -1165,19 +1227,19 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 	std::filesystem::path LogPath	= cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName);
 	std::filesystem::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName);
 
-	if (std::filesystem::is_regular_file(IndexPath))
+	if (IsFile(IndexPath))
 	{
 		uint32_t IndexVersion = 0;
 		m_LogFlushPosition	  = ReadIndexFile(IndexPath, IndexVersion);
 		if (IndexVersion == 0)
 		{
 			ZEN_WARN("removing invalid index file at '{}'", IndexPath);
-			std::filesystem::remove(IndexPath);
+			RemoveFile(IndexPath);
 		}
 	}
 
 	uint64_t LogEntryCount = 0;
-	if (std::filesystem::is_regular_file(LogPath))
+	if (IsFile(LogPath))
 	{
 		if (TCasLogFile<CasDiskIndexEntry>::IsValid(LogPath))
 		{
@@ -1186,12 +1248,10 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		else
 		{
 			ZEN_WARN("removing invalid cas log at '{}'", LogPath);
-			std::filesystem::remove(LogPath);
+			RemoveFile(LogPath);
 		}
 	}
 
-	m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
-
 	BlockStore::BlockIndexSet KnownBlocks;
 
 	for (const auto& Entry : m_LocationMap)
@@ -1201,11 +1261,41 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		KnownBlocks.insert(BlockIndex);
 	}
 
-	m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
+	BlockStore::BlockIndexSet MissingBlocks = m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
+
+	m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
+
+	bool RemovedEntries = false;
+	if (!MissingBlocks.empty())
+	{
+		std::vector<CasDiskIndexEntry> MissingEntries;
+		for (auto& It : m_LocationMap)
+		{
+			const uint32_t BlockIndex = m_Locations[It.second].GetBlockIndex();
+			if (MissingBlocks.contains(BlockIndex))
+			{
+				MissingEntries.push_back({.Key = It.first, .Location = m_Locations[It.second], .Flags = CasDiskIndexEntry::kTombstone});
+			}
+		}
+		ZEN_ASSERT(!MissingEntries.empty());
+
+		for (const CasDiskIndexEntry& Entry : MissingEntries)
+		{
+			m_LocationMap.erase(Entry.Key);
+		}
+		m_CasLog.Append(MissingEntries);
+		m_CasLog.Flush();
+
+		{
+			RwLock::ExclusiveLockScope IndexLock(m_LocationMapLock);
+			CompactIndex(IndexLock);
+		}
+		RemovedEntries = true;
+	}
 
-	if (IsNewStore || (LogEntryCount > 0))
+	if (IsNewStore || (LogEntryCount > 0) || RemovedEntries)
 	{
-		MakeIndexSnapshot();
+		MakeIndexSnapshot(/*ResetLog*/ true);
 	}
 
 	// TODO: should validate integrity of container files here
@@ -1573,6 +1663,423 @@ TEST_CASE("compactcas.threadedinsert")
 	}
 }
 
+TEST_CASE("compactcas.restart")
+{
+	uint64_t ExpectedSize = 0;
+
+	auto GenerateChunks = [&](CasContainerStrategy& Cas, size_t ChunkCount, uint64_t ChunkSize, std::vector<IoHash>& Hashes) {
+		WorkerThreadPool ThreadPool(Max(std::thread::hardware_concurrency() - 1u, 2u), "put");
+
+		Latch								   WorkLatch(1);
+		tsl::robin_set<IoHash, IoHash::Hasher> ChunkHashesLookup;
+		ChunkHashesLookup.reserve(ChunkCount);
+		RwLock InsertLock;
+		for (size_t Offset = 0; Offset < ChunkCount;)
+		{
+			size_t BatchCount = Min<size_t>(ChunkCount - Offset, 512u);
+			WorkLatch.AddCount(1);
+			ThreadPool.ScheduleWork(
+				[&WorkLatch, &InsertLock, &ChunkHashesLookup, &ExpectedSize, &Hashes, &Cas, Offset, BatchCount, ChunkSize]() {
+					auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
+
+					std::vector<IoBuffer> BatchBlobs;
+					std::vector<IoHash>	  BatchHashes;
+					BatchBlobs.reserve(BatchCount);
+					BatchHashes.reserve(BatchCount);
+
+					while (BatchBlobs.size() < BatchCount)
+					{
+						IoBuffer Chunk =
+							CreateSemiRandomBlob(ChunkSize + ((BatchHashes.size() % 100) + (BatchHashes.size() % 7) * 315u + Offset % 377));
+						IoHash Hash = IoHash::HashBuffer(Chunk);
+						{
+							RwLock::ExclusiveLockScope __(InsertLock);
+							if (ChunkHashesLookup.contains(Hash))
+							{
+								continue;
+							}
+							ChunkHashesLookup.insert(Hash);
+							ExpectedSize += Chunk.Size();
+						}
+
+						BatchBlobs.emplace_back(CompressedBuffer::Compress(SharedBuffer(Chunk)).GetCompressed().Flatten().AsIoBuffer());
+						BatchHashes.push_back(Hash);
+					}
+
+					Cas.InsertChunks(BatchBlobs, BatchHashes);
+					{
+						RwLock::ExclusiveLockScope __(InsertLock);
+						Hashes.insert(Hashes.end(), BatchHashes.begin(), BatchHashes.end());
+					}
+				});
+			Offset += BatchCount;
+		}
+		WorkLatch.CountDown();
+		WorkLatch.Wait();
+	};
+
+	ScopedTemporaryDirectory TempDir;
+	std::filesystem::path	 CasPath = TempDir.Path();
+	CreateDirectories(CasPath);
+
+	bool Generate = false;
+	if (!Generate)
+	{
+		GcManager			 Gc;
+		CasContainerStrategy Cas(Gc);
+		Cas.Initialize(CasPath, "test", 65536 * 128, 8, false);
+	}
+
+	const uint64_t kChunkSize  = 1048 + 395;
+	const size_t   kChunkCount = 7167;
+
+	std::vector<IoHash> Hashes;
+	Hashes.reserve(kChunkCount);
+
+	auto ValidateChunks = [&](CasContainerStrategy& Cas, std::span<const IoHash> Hashes, bool ShouldExist) {
+		for (const IoHash& Hash : Hashes)
+		{
+			if (ShouldExist)
+			{
+				CHECK(Cas.HaveChunk(Hash));
+				IoBuffer Buffer = Cas.FindChunk(Hash);
+				CHECK(Buffer);
+				IoHash			 ValidateHash;
+				uint64_t		 ValidateRawSize;
+				CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), ValidateHash, ValidateRawSize);
+				CHECK(Compressed);
+				CHECK(ValidateHash == Hash);
+			}
+			else
+			{
+				CHECK(!Cas.HaveChunk(Hash));
+				IoBuffer Buffer = Cas.FindChunk(Hash);
+				CHECK(!Buffer);
+			}
+		}
+	};
+
+	{
+		GcManager			 Gc;
+		CasContainerStrategy Cas(Gc);
+		Cas.Initialize(CasPath, "test", 65536 * 128, 8, true);
+		GenerateChunks(Cas, kChunkCount, kChunkSize, Hashes);
+		ValidateChunks(Cas, Hashes, true);
+		Cas.Flush();
+		ValidateChunks(Cas, Hashes, true);
+	}
+
+	{
+		GcManager			 Gc;
+		CasContainerStrategy Cas(Gc);
+		Cas.Initialize(CasPath, "test", 65536 * 128, 8, false);
+		ValidateChunks(Cas, Hashes, true);
+		GenerateChunks(Cas, kChunkCount, kChunkSize / 4, Hashes);
+		ValidateChunks(Cas, Hashes, true);
+	}
+
+	class GcRefChecker : public GcReferenceChecker
+	{
+	public:
+		explicit GcRefChecker(std::vector<IoHash>&& HashesToKeep) : m_HashesToKeep(std::move(HashesToKeep)) {}
+		~GcRefChecker() {}
+		std::string GetGcName(GcCtx& Ctx) override
+		{
+			ZEN_UNUSED(Ctx);
+			return "test";
+		}
+		void			  PreCache(GcCtx& Ctx) override { FilterReferences(Ctx, "test", m_HashesToKeep); }
+		void			  UpdateLockedState(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); }
+		std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
+		{
+			ZEN_UNUSED(Ctx);
+			return KeepUnusedReferences(m_HashesToKeep, IoCids);
+		}
+
+	private:
+		std::vector<IoHash> m_HashesToKeep;
+	};
+
+	class GcRef : public GcReferencer
+	{
+	public:
+		GcRef(GcManager& Gc, std::span<const IoHash> HashesToKeep) : m_Gc(Gc)
+		{
+			m_HashesToKeep.insert(m_HashesToKeep.begin(), HashesToKeep.begin(), HashesToKeep.end());
+			m_Gc.AddGcReferencer(*this);
+		}
+		~GcRef() { m_Gc.RemoveGcReferencer(*this); }
+		std::string GetGcName(GcCtx& Ctx) override
+		{
+			ZEN_UNUSED(Ctx);
+			return "test";
+		}
+		GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override
+		{
+			ZEN_UNUSED(Ctx, Stats);
+			return nullptr;
+		}
+		std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override
+		{
+			ZEN_UNUSED(Ctx);
+			return {new GcRefChecker(std::move(m_HashesToKeep))};
+		}
+		std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override
+		{
+			ZEN_UNUSED(Ctx);
+			return {};
+		}
+
+	private:
+		GcManager&			m_Gc;
+		std::vector<IoHash> m_HashesToKeep;
+	};
+
+	{
+		GcManager			 Gc;
+		CasContainerStrategy Cas(Gc);
+		Cas.Initialize(CasPath, "test", 65536 * 128, 8, false);
+		GenerateChunks(Cas, kChunkCount, kChunkSize / 5, Hashes);
+	}
+
+	{
+		GcManager			 Gc;
+		CasContainerStrategy Cas(Gc);
+		Cas.Initialize(CasPath, "test", 65536 * 128, 8, false);
+		ValidateChunks(Cas, Hashes, true);
+		GenerateChunks(Cas, kChunkCount, kChunkSize / 2, Hashes);
+		ValidateChunks(Cas, Hashes, true);
+		if (true)
+		{
+			std::vector<IoHash> DropHashes;
+			std::vector<IoHash> KeepHashes;
+			for (size_t Index = 0; Index < Hashes.size(); Index++)
+			{
+				if (Index % 5 == 0)
+				{
+					KeepHashes.push_back(Hashes[Index]);
+				}
+				else
+				{
+					DropHashes.push_back(Hashes[Index]);
+				}
+			}
+			//			std::span<const IoHash> KeepHashes(Hashes);
+			//			ZEN_ASSERT(ExpectedGcCount < Hashes.size());
+			//			KeepHashes = KeepHashes.subspan(ExpectedGcCount);
+			GcRef Ref(Gc, KeepHashes);
+			Gc.CollectGarbage(GcSettings{.CollectSmallObjects = true, .IsDeleteMode = true});
+			ValidateChunks(Cas, KeepHashes, true);
+			ValidateChunks(Cas, DropHashes, false);
+			Hashes = KeepHashes;
+		}
+		GenerateChunks(Cas, kChunkCount, kChunkSize / 3, Hashes);
+	}
+
+	{
+		GcManager			 Gc;
+		CasContainerStrategy Cas(Gc);
+		Cas.Initialize(CasPath, "test", 65536 * 128, 8, false);
+		ValidateChunks(Cas, Hashes, true);
+		Cas.Flush();
+		ValidateChunks(Cas, Hashes, true);
+	}
+
+	{
+		GcManager			 Gc;
+		CasContainerStrategy Cas(Gc);
+		Cas.Initialize(CasPath, "test", 65536 * 128, 8, false);
+		ValidateChunks(Cas, Hashes, true);
+	}
+}
+
+TEST_CASE("compactcas.iteratechunks")
+{
+	std::atomic<size_t> WorkCompleted = 0;
+	WorkerThreadPool	ThreadPool(Max(std::thread::hardware_concurrency() - 1u, 2u), "put");
+
+	const uint64_t kChunkSize  = 1048 + 395;
+	const size_t   kChunkCount = 63840;
+
+	for (uint32_t N = 0; N < 4; N++)
+	{
+		GcManager				 Gc;
+		CasContainerStrategy	 Cas(Gc);
+		ScopedTemporaryDirectory TempDir;
+		Cas.Initialize(TempDir.Path(), "test", 65536 * 128, 8, true);
+
+		CHECK(Cas.IterateChunks(
+			{},
+			[](size_t Index, const IoBuffer& Payload) {
+				ZEN_UNUSED(Index, Payload);
+				return true;
+			},
+			&ThreadPool,
+			2048u));
+
+		uint64_t ExpectedSize = 0;
+
+		std::vector<IoHash> Hashes;
+		Hashes.reserve(kChunkCount);
+
+		{
+			Latch								   WorkLatch(1);
+			tsl::robin_set<IoHash, IoHash::Hasher> ChunkHashesLookup;
+			ChunkHashesLookup.reserve(kChunkCount);
+			RwLock InsertLock;
+			for (size_t Offset = 0; Offset < kChunkCount;)
+			{
+				size_t BatchCount = Min<size_t>(kChunkCount - Offset, 512u);
+				WorkLatch.AddCount(1);
+				ThreadPool.ScheduleWork(
+					[N, &WorkLatch, &InsertLock, &ChunkHashesLookup, &ExpectedSize, &Hashes, &Cas, Offset, BatchCount]() {
+						auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
+
+						std::vector<IoBuffer> BatchBlobs;
+						std::vector<IoHash>	  BatchHashes;
+						BatchBlobs.reserve(BatchCount);
+						BatchHashes.reserve(BatchCount);
+
+						while (BatchBlobs.size() < BatchCount)
+						{
+							IoBuffer Chunk = CreateRandomBlob(
+								N + kChunkSize + ((BatchHashes.size() % 100) + (BatchHashes.size() % 7) * 315u + Offset % 377));
+							IoHash Hash = IoHash::HashBuffer(Chunk);
+							{
+								RwLock::ExclusiveLockScope __(InsertLock);
+								if (ChunkHashesLookup.contains(Hash))
+								{
+									continue;
+								}
+								ChunkHashesLookup.insert(Hash);
+								ExpectedSize += Chunk.Size();
+							}
+
+							BatchBlobs.emplace_back(std::move(Chunk));
+							BatchHashes.push_back(Hash);
+						}
+
+						Cas.InsertChunks(BatchBlobs, BatchHashes);
+						{
+							RwLock::ExclusiveLockScope __(InsertLock);
+							Hashes.insert(Hashes.end(), BatchHashes.begin(), BatchHashes.end());
+						}
+					});
+				Offset += BatchCount;
+			}
+			WorkLatch.CountDown();
+			WorkLatch.Wait();
+		}
+
+		WorkerThreadPool BatchWorkerPool(Max(std::thread::hardware_concurrency() - 1u, 2u), "fetch");
+		{
+			std::vector<std::atomic<bool>> FetchedFlags(Hashes.size());
+			std::atomic<uint64_t>		   FetchedSize = 0;
+			CHECK(Cas.IterateChunks(
+				Hashes,
+				[&Hashes, &FetchedFlags, &FetchedSize](size_t Index, const IoBuffer& Payload) {
+					CHECK(FetchedFlags[Index].load() == false);
+					FetchedFlags[Index].store(true);
+					const IoHash& Hash = Hashes[Index];
+					CHECK(Hash == IoHash::HashBuffer(Payload));
+					FetchedSize += Payload.GetSize();
+					return true;
+				},
+				&BatchWorkerPool,
+				2048u));
+			for (const auto& Flag : FetchedFlags)
+			{
+				CHECK(Flag.load());
+			}
+			CHECK(FetchedSize == ExpectedSize);
+		}
+
+		Latch WorkLatch(1);
+		for (size_t I = 0; I < 2; I++)
+		{
+			WorkLatch.AddCount(1);
+			ThreadPool.ScheduleWork([&Cas, &Hashes, &BatchWorkerPool, &WorkLatch, I]() {
+				auto				_ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
+				std::vector<IoHash> PartialHashes;
+				PartialHashes.reserve(Hashes.size() / 4);
+				for (size_t Index = 0; Index < Hashes.size(); Index++)
+				{
+					size_t TestIndex = Index + I;
+					if ((TestIndex % 7 == 1) || (TestIndex % 13 == 1) || (TestIndex % 17 == 1))
+					{
+						PartialHashes.push_back(Hashes[Index]);
+					}
+				}
+				std::reverse(PartialHashes.begin(), PartialHashes.end());
+
+				std::vector<IoHash> NoFoundHashes;
+				std::vector<size_t> NoFindIndexes;
+
+				NoFoundHashes.reserve(9);
+				for (size_t J = 0; J < 9; J++)
+				{
+					std::string Data = fmt::format("oh no, we don't exist {}", J + 1);
+					NoFoundHashes.push_back(IoHash::HashBuffer(Data.data(), Data.length()));
+				}
+
+				NoFindIndexes.reserve(9);
+
+				// Sprinkle in chunks that are not found!
+				auto It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 0, NoFoundHashes[0]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 0 + 1, NoFoundHashes[1]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 1, NoFoundHashes[2]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 1 + 1, NoFoundHashes[3]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 2, NoFoundHashes[4]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 3, NoFoundHashes[5]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 3 + 1, NoFoundHashes[6]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 4, NoFoundHashes[7]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+				It = PartialHashes.insert(PartialHashes.end(), NoFoundHashes[8]);
+				NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It));
+
+				std::vector<std::atomic<bool>>	   FoundFlags(PartialHashes.size() + NoFoundHashes.size());
+				std::vector<std::atomic<uint32_t>> FetchedCounts(PartialHashes.size() + NoFoundHashes.size());
+
+				CHECK(Cas.IterateChunks(
+					PartialHashes,
+					[&PartialHashes, &FoundFlags, &FetchedCounts, &NoFindIndexes](size_t Index, const IoBuffer& Payload) {
+						CHECK_EQ(NoFindIndexes.end(), std::find(NoFindIndexes.begin(), NoFindIndexes.end(), Index));
+						uint32_t PreviousCount = FetchedCounts[Index].fetch_add(1);
+						CHECK(PreviousCount == 0);
+						FoundFlags[Index]  = !!Payload;
+						const IoHash& Hash = PartialHashes[Index];
+						CHECK(Hash == IoHash::HashBuffer(Payload));
+						return true;
+					},
+					&BatchWorkerPool,
+					2048u));
+
+				for (size_t FoundIndex = 0; FoundIndex < PartialHashes.size(); FoundIndex++)
+				{
+					CHECK(FetchedCounts[FoundIndex].load() <= 1);
+					if (std::find(NoFindIndexes.begin(), NoFindIndexes.end(), FoundIndex) == NoFindIndexes.end())
+					{
+						CHECK(FoundFlags[FoundIndex]);
+					}
+					else
+					{
+						CHECK(!FoundFlags[FoundIndex]);
+					}
+				}
+			});
+		}
+		WorkLatch.CountDown();
+		WorkLatch.Wait();
+	}
+}
+
 #endif
 
 void
diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h
index 07e620086..15e4cbf81 100644
--- a/src/zenstore/compactcas.h
+++ b/src/zenstore/compactcas.h
@@ -52,11 +52,11 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore
 	~CasContainerStrategy();
 
 	CasStore::InsertResult				InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
-	std::vector<CasStore::InsertResult> InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes);
+	std::vector<CasStore::InsertResult> InsertChunks(std::span<const IoBuffer> Chunks, std::span<const IoHash> ChunkHashes);
 	IoBuffer							FindChunk(const IoHash& ChunkHash);
 	bool								HaveChunk(const IoHash& ChunkHash);
 	void								FilterChunks(HashKeySet& InOutChunks);
-	bool								IterateChunks(std::span<IoHash>													ChunkHashes,
+	bool								IterateChunks(std::span<const IoHash>											ChunkHashes,
 													  const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
 													  WorkerThreadPool*													OptionalWorkerPool,
 													  uint64_t															LargeSizeLimit);
@@ -77,7 +77,7 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore
 
 private:
 	CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
-	void				   MakeIndexSnapshot();
+	void				   MakeIndexSnapshot(bool ResetLog);
 	uint64_t			   ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
 	uint64_t			   ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
 	void				   OpenContainer(bool IsNewStore);
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 34db51aa9..68644be2d 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -20,6 +20,7 @@
 #include <zencore/workthreadpool.h>
 #include <zenstore/gc.h>
 #include <zenstore/scrubcontext.h>
+#include <zenutil/parallelwork.h>
 
 #if ZEN_WITH_TESTS
 #	include <zencore/compactbinarybuilder.h>
@@ -176,10 +177,10 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 
 	if (IsNewStore)
 	{
-		std::filesystem::remove(LogPath);
-		std::filesystem::remove(IndexPath);
+		RemoveFile(LogPath);
+		RemoveFile(IndexPath);
 
-		if (std::filesystem::is_directory(m_RootDirectory))
+		if (IsDir(m_RootDirectory))
 		{
 			// We need to explicitly only delete sharded root folders as the cas manifest, tinyobject and smallobject cas folders may reside
 			// in this folder as well
@@ -211,24 +212,24 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 			Traversal.TraverseFileSystem(m_RootDirectory, CasVisitor);
 			for (const std::filesystem::path& SharededRoot : CasVisitor.ShardedRoots)
 			{
-				std::filesystem::remove_all(SharededRoot);
+				DeleteDirectories(SharededRoot);
 			}
 		}
 	}
 
-	if (std::filesystem::is_regular_file(IndexPath))
+	if (IsFile(IndexPath))
 	{
 		uint32_t IndexVersion = 0;
 		m_LogFlushPosition	  = ReadIndexFile(IndexPath, IndexVersion);
 		if (IndexVersion == 0)
 		{
 			ZEN_WARN("removing invalid index file at '{}'", IndexPath);
-			std::filesystem::remove(IndexPath);
+			RemoveFile(IndexPath);
 		}
 	}
 
 	uint64_t LogEntryCount = 0;
-	if (std::filesystem::is_regular_file(LogPath))
+	if (IsFile(LogPath))
 	{
 		if (TCasLogFile<FileCasIndexEntry>::IsValid(LogPath))
 		{
@@ -237,7 +238,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 		else
 		{
 			ZEN_WARN("removing invalid cas log at '{}'", LogPath);
-			std::filesystem::remove(LogPath);
+			RemoveFile(LogPath);
 		}
 	}
 
@@ -251,7 +252,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 
 	if (IsNewStore || LogEntryCount > 0)
 	{
-		MakeIndexSnapshot();
+		MakeIndexSnapshot(/*ResetLog*/ true);
 	}
 }
 
@@ -327,7 +328,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore::
 	{
 		std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString());
 		std::error_code		  Ec;
-		std::filesystem::rename(ChunkPath, TempPath, Ec);
+		RenameFile(ChunkPath, TempPath, Ec);
 		if (Ec)
 		{
 			throw std::system_error(Ec, fmt::format("unable to move existing CAS file {} to {}", ChunkPath, TempPath));
@@ -452,7 +453,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore::
 	{
 		PayloadFile.Close();
 		std::error_code DummyEc;
-		std::filesystem::remove(ChunkPath, DummyEc);
+		RemoveFile(ChunkPath, DummyEc);
 		throw;
 	}
 	bool IsNew = UpdateIndex(ChunkHash, Chunk.Size());
@@ -503,7 +504,7 @@ FileCasStrategy::SafeOpenChunk(const IoHash& ChunkHash, uint64 ExpectedSize)
 		{
 			std::error_code		  Ec;
 			std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString());
-			std::filesystem::rename(ChunkPath, TempPath, Ec);
+			RenameFile(ChunkPath, TempPath, Ec);
 			if (!Ec)
 			{
 				Chunk.SetDeleteOnClose(true);
@@ -563,8 +564,24 @@ FileCasStrategy::HaveChunk(const IoHash& ChunkHash)
 {
 	ZEN_ASSERT(m_IsInitialized);
 
-	RwLock::SharedLockScope _(m_Lock);
-	return m_Index.contains(ChunkHash);
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		if (auto It = m_Index.find(ChunkHash); It == m_Index.end())
+		{
+			return false;
+		}
+	}
+
+	ShardingHelper				Name(m_RootDirectory, ChunkHash);
+	const std::filesystem::path ChunkPath = Name.ShardedPath.ToPath();
+	RwLock::SharedLockScope		ShardLock(LockForHash(ChunkHash));
+
+	if (IsFile(ChunkPath))
+	{
+		return true;
+	}
+
+	return false;
 }
 
 void
@@ -574,7 +591,7 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec)
 
 	ShardingHelper				Name(m_RootDirectory, ChunkHash);
 	const std::filesystem::path ChunkPath = Name.ShardedPath.ToPath();
-	uint64_t					FileSize  = static_cast<uint64_t>(std::filesystem::file_size(ChunkPath, Ec));
+	uint64_t					FileSize  = static_cast<uint64_t>(FileSizeFromPath(ChunkPath, Ec));
 	if (Ec)
 	{
 		ZEN_WARN("get file size FAILED, file cas '{}'", ChunkPath);
@@ -582,9 +599,9 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec)
 	}
 
 	ZEN_DEBUG("deleting CAS payload file '{}' {}", ChunkPath, NiceBytes(FileSize));
-	std::filesystem::remove(ChunkPath, Ec);
+	RemoveFile(ChunkPath, Ec);
 
-	if (!Ec || !std::filesystem::exists(ChunkPath))
+	if (!Ec || !IsFile(ChunkPath))
 	{
 		{
 			RwLock::ExclusiveLockScope _(m_Lock);
@@ -632,10 +649,11 @@ FileCasStrategy::IterateChunks(std::span<IoHash>												 ChunkHashes,
 			}
 		}
 	}
-	std::atomic_bool Continue = true;
+	std::atomic<bool> AsyncContinue = true;
 	if (!FoundChunkIndexes.empty())
 	{
-		auto ProcessOne = [this, &ChunkHashes, &Continue, &AsyncCallback](size_t ChunkIndex, uint64_t ExpectedSize) {
+		auto ProcessOne = [this, &ChunkHashes, &AsyncCallback](size_t ChunkIndex, uint64_t ExpectedSize) {
+			ZEN_ASSERT(ChunkIndex < ChunkHashes.size());
 			const IoHash& ChunkHash = ChunkHashes[ChunkIndex];
 			IoBuffer	  Payload	= SafeOpenChunk(ChunkHash, ExpectedSize);
 			if (!AsyncCallback(ChunkIndex, std::move(Payload)))
@@ -645,49 +663,70 @@ FileCasStrategy::IterateChunks(std::span<IoHash>												 ChunkHashes,
 			return true;
 		};
 
-		Latch WorkLatch(1);
-		for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++)
+		std::atomic<bool> AbortFlag;
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
+		try
 		{
-			size_t	 ChunkIndex	  = FoundChunkIndexes[Index];
-			uint64_t ExpectedSize = FoundChunkExpectedSizes[Index];
-			if (!Continue)
+			for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++)
 			{
-				break;
-			}
-			if (OptionalWorkerPool)
-			{
-				WorkLatch.AddCount(1);
-				OptionalWorkerPool->ScheduleWork([this, &WorkLatch, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &Continue]() {
-					auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
-					if (!Continue)
-					{
-						return;
-					}
-					try
-					{
-						if (!ProcessOne(ChunkIndex, ExpectedSize))
-						{
-							Continue = false;
-						}
-					}
-					catch (const std::exception& Ex)
+				if (AbortFlag)
+				{
+					AsyncContinue.store(false);
+				}
+				if (!AsyncContinue)
+				{
+					break;
+				}
+				size_t	 ChunkIndex	  = FoundChunkIndexes[Index];
+				uint64_t ExpectedSize = FoundChunkExpectedSizes[Index];
+				if (OptionalWorkerPool)
+				{
+					Work.ScheduleWork(
+						*OptionalWorkerPool,
+						[this, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &AsyncContinue](std::atomic<bool>& AbortFlag) {
+							if (AbortFlag)
+							{
+								AsyncContinue.store(false);
+							}
+							if (!AsyncContinue)
+							{
+								return;
+							}
+							try
+							{
+								if (!ProcessOne(ChunkIndex, ExpectedSize))
+								{
+									AsyncContinue.store(false);
+								}
+							}
+							catch (const std::exception& Ex)
+							{
+								ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'",
+										 m_RootDirectory,
+										 ChunkHashes[ChunkIndex],
+										 Ex.what());
+								AsyncContinue.store(false);
+							}
+						});
+				}
+				else
+				{
+					if (!ProcessOne(ChunkIndex, ExpectedSize))
 					{
-						ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'",
-								 m_RootDirectory,
-								 ChunkHashes[ChunkIndex],
-								 Ex.what());
+						AsyncContinue.store(false);
 					}
-				});
-			}
-			else
-			{
-				Continue = Continue && ProcessOne(ChunkIndex, ExpectedSize);
+				}
 			}
 		}
-		WorkLatch.CountDown();
-		WorkLatch.Wait();
+		catch (const std::exception& Ex)
+		{
+			AbortFlag.store(true);
+			ZEN_WARN("Failed iterating chunks in {}. Reason: '{}'", this->m_RootDirectory, Ex.what());
+		}
+		Work.Wait();
 	}
-	return Continue;
+	return AsyncContinue.load();
 }
 
 void
@@ -727,7 +766,7 @@ FileCasStrategy::Flush()
 	ZEN_TRACE_CPU("FileCas::Flush");
 
 	m_CasLog.Flush();
-	MakeIndexSnapshot();
+	MakeIndexSnapshot(/*ResetLog*/ false);
 }
 
 void
@@ -912,15 +951,14 @@ FileCasStrategy::ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutR
 }
 
 void
-FileCasStrategy::MakeIndexSnapshot()
+FileCasStrategy::MakeIndexSnapshot(bool ResetLog)
 {
 	ZEN_MEMSCOPE(GetFileCasTag());
 	ZEN_TRACE_CPU("FileCas::MakeIndexSnapshot");
 
 	using namespace filecas::impl;
 
-	uint64_t LogCount = m_CasLog.GetLogCount();
-	if (m_LogFlushPosition == LogCount)
+	if (m_LogFlushPosition == m_CasLog.GetLogCount())
 	{
 		return;
 	}
@@ -937,34 +975,17 @@ FileCasStrategy::MakeIndexSnapshot()
 
 	namespace fs = std::filesystem;
 
-	fs::path IndexPath	   = GetIndexPath(m_RootDirectory);
-	fs::path STmpIndexPath = GetTempIndexPath(m_RootDirectory);
-
-	// Move index away, we keep it if something goes wrong
-	if (fs::is_regular_file(STmpIndexPath))
-	{
-		std::error_code Ec;
-		if (!fs::remove(STmpIndexPath, Ec) || Ec)
-		{
-			ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message());
-			return;
-		}
-	}
+	const fs::path IndexPath = GetIndexPath(m_RootDirectory);
 
 	try
 	{
-		if (fs::is_regular_file(IndexPath))
-		{
-			fs::rename(IndexPath, STmpIndexPath);
-		}
-
 		// Write the current state of the location map to a new index state
 		std::vector<FileCasIndexEntry> Entries;
-		uint64_t					   IndexLogPosition = 0;
+		// Be defensive regarding log position as it is written to without acquiring m_LocationMapLock
+		const uint64_t IndexLogPosition = ResetLog ? 0 : m_CasLog.GetLogCount();
 
 		{
 			RwLock::SharedLockScope __(m_Lock);
-			IndexLogPosition = m_CasLog.GetLogCount();
 			Entries.resize(m_Index.size());
 
 			uint64_t EntryIndex = 0;
@@ -974,6 +995,7 @@ FileCasStrategy::MakeIndexSnapshot()
 				IndexEntry.Key				  = Entry.first;
 				IndexEntry.Size				  = Entry.second.Size;
 			}
+			EntryCount = m_Index.size();
 		}
 
 		TemporaryFile	ObjectIndexFile;
@@ -983,47 +1005,47 @@ FileCasStrategy::MakeIndexSnapshot()
 		{
 			throw std::system_error(Ec, fmt::format("Failed to create temp file for index snapshot at '{}'", IndexPath));
 		}
-		filecas::impl::FileCasIndexHeader Header = {.EntryCount = Entries.size(), .LogPosition = IndexLogPosition};
+		filecas::impl::FileCasIndexHeader Header = {.EntryCount = EntryCount, .LogPosition = IndexLogPosition};
 
 		Header.Checksum = filecas::impl::FileCasIndexHeader::ComputeChecksum(Header);
 
 		ObjectIndexFile.Write(&Header, sizeof(filecas::impl::FileCasIndexHeader), 0);
-		ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(FileCasIndexEntry), sizeof(filecas::impl::FileCasIndexHeader));
+		ObjectIndexFile.Write(Entries.data(), EntryCount * sizeof(FileCasIndexEntry), sizeof(filecas::impl::FileCasIndexHeader));
 		ObjectIndexFile.Flush();
 		ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec);
 		if (Ec)
 		{
-			throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", ObjectIndexFile.GetPath(), IndexPath));
+			throw std::system_error(Ec,
+									fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'",
+												ObjectIndexFile.GetPath(),
+												IndexPath,
+												Ec.message()));
 		}
-		EntryCount		   = Entries.size();
-		m_LogFlushPosition = IndexLogPosition;
-	}
-	catch (const std::exception& Err)
-	{
-		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
-
-		// Restore any previous snapshot
 
-		if (fs::is_regular_file(STmpIndexPath))
+		if (ResetLog)
 		{
-			std::error_code Ec;
-			fs::remove(IndexPath, Ec);	// We don't care if this fails, we try to move the old temp file regardless
-			fs::rename(STmpIndexPath, IndexPath, Ec);
-			if (Ec)
+			const std::filesystem::path LogPath = GetLogPath(m_RootDirectory);
+
+			if (IsFile(LogPath))
 			{
-				ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message());
+				m_CasLog.Close();
+				if (!RemoveFile(LogPath, Ec) || Ec)
+				{
+					// This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in
+					// the end it will be the same result
+					ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message());
+				}
+				m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
 			}
 		}
+		m_LogFlushPosition = IndexLogPosition;
 	}
-	if (fs::is_regular_file(STmpIndexPath))
+	catch (const std::exception& Err)
 	{
-		std::error_code Ec;
-		if (!fs::remove(STmpIndexPath, Ec) || Ec)
-		{
-			ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message());
-		}
+		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
 	}
 }
+
 uint64_t
 FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion)
 {
@@ -1032,7 +1054,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t&
 	using namespace filecas::impl;
 
 	std::vector<FileCasIndexEntry> Entries;
-	if (std::filesystem::is_regular_file(IndexPath))
+	if (IsFile(IndexPath))
 	{
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
@@ -1077,7 +1099,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t&
 		return 0;
 	}
 
-	if (std::filesystem::is_directory(m_RootDirectory))
+	if (IsDir(m_RootDirectory))
 	{
 		ZEN_INFO("missing index for file cas, scanning for cas files in {}", m_RootDirectory);
 		TCasLogFile<FileCasIndexEntry> CasLog;
@@ -1116,7 +1138,7 @@ FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntr
 
 	using namespace filecas::impl;
 
-	if (std::filesystem::is_regular_file(LogPath))
+	if (IsFile(LogPath))
 	{
 		uint64_t					   LogEntryCount = 0;
 		Stopwatch					   Timer;
@@ -1274,12 +1296,12 @@ public:
 								 ChunkPath);
 					}
 					std::error_code Ec;
-					uint64_t		SizeOnDisk = std::filesystem::file_size(ChunkPath, Ec);
+					uint64_t		SizeOnDisk = FileSizeFromPath(ChunkPath, Ec);
 					if (Ec)
 					{
 						SizeOnDisk = 0;
 					}
-					bool Existed = std::filesystem::remove(ChunkPath, Ec);
+					bool Existed = RemoveFile(ChunkPath, Ec);
 					if (Ec)
 					{
 						// Target file may be open for read, attempt to move it to a temp file and mark it delete on close
@@ -1290,7 +1312,7 @@ public:
 						if (OldChunk)
 						{
 							std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString());
-							std::filesystem::rename(ChunkPath, TempPath, Ec);
+							RenameFile(ChunkPath, TempPath, Ec);
 							if (!Ec)
 							{
 								OldChunk.SetDeleteOnClose(true);
@@ -1317,7 +1339,7 @@ public:
 				else
 				{
 					std::error_code Ec;
-					bool			Existed = std::filesystem::is_regular_file(ChunkPath, Ec);
+					bool			Existed = IsFile(ChunkPath, Ec);
 					if (Ec)
 					{
 						if (Ctx.Settings.Verbose)
@@ -1516,7 +1538,7 @@ TEST_CASE("cas.chunk.moveoverwrite")
 			Payload1.SetDeleteOnClose(true);
 			CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash());
 			CHECK_EQ(Result.New, true);
-			CHECK(!std::filesystem::exists(Payload1Path));
+			CHECK(!IsFile(Payload1Path));
 		}
 		{
 			std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"};
@@ -1526,9 +1548,9 @@ TEST_CASE("cas.chunk.moveoverwrite")
 
 			CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash());
 			CHECK_EQ(Result.New, false);
-			CHECK(std::filesystem::exists(Payload1BPath));
+			CHECK(IsFile(Payload1BPath));
 			Payload1B = {};
-			CHECK(!std::filesystem::exists(Payload1BPath));
+			CHECK(!IsFile(Payload1BPath));
 		}
 
 		IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash());
@@ -1554,7 +1576,7 @@ TEST_CASE("cas.chunk.moveoverwrite")
 		}
 
 		Payload2 = {};
-		CHECK(!std::filesystem::exists(Payload2Path));
+		CHECK(!IsFile(Payload2Path));
 
 		{
 			IoHash			 RawHash;
@@ -1598,9 +1620,9 @@ TEST_CASE("cas.chunk.copyoverwrite")
 			CasStore::InsertResult Result =
 				FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly);
 			CHECK_EQ(Result.New, true);
-			CHECK(std::filesystem::exists(Payload1Path));
+			CHECK(IsFile(Payload1Path));
 			Payload1 = {};
-			CHECK(!std::filesystem::exists(Payload1Path));
+			CHECK(!IsFile(Payload1Path));
 		}
 		{
 			std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"};
@@ -1611,9 +1633,9 @@ TEST_CASE("cas.chunk.copyoverwrite")
 			CasStore::InsertResult Result =
 				FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly);
 			CHECK_EQ(Result.New, false);
-			CHECK(std::filesystem::exists(Payload1BPath));
+			CHECK(IsFile(Payload1BPath));
 			Payload1B = {};
-			CHECK(!std::filesystem::exists(Payload1BPath));
+			CHECK(!IsFile(Payload1BPath));
 		}
 
 		IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash());
@@ -1640,7 +1662,7 @@ TEST_CASE("cas.chunk.copyoverwrite")
 		}
 
 		Payload2 = {};
-		CHECK(!std::filesystem::exists(Payload2Path));
+		CHECK(!IsFile(Payload2Path));
 
 		{
 			IoHash			 RawHash;
diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h
index 21d8c3b9e..e93356927 100644
--- a/src/zenstore/filecas.h
+++ b/src/zenstore/filecas.h
@@ -50,7 +50,7 @@ struct FileCasStrategy final : public GcStorage, public GcReferenceStore
 	virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override;
 
 private:
-	void	  MakeIndexSnapshot();
+	void	  MakeIndexSnapshot(bool ResetLog);
 	uint64_t  ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
 	uint64_t  ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
 	LoggerRef Log() { return m_Log; }
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 7ac10d613..a15a2e084 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -62,11 +62,11 @@ namespace {
 	{
 		if (Size == 0)
 		{
-			std::filesystem::remove(Path);
+			RemoveFile(Path);
 			return std::error_code{};
 		}
 		CreateDirectories(Path.parent_path());
-		if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size)
+		if (IsFile(Path) && FileSizeFromPath(Path) == Size)
 		{
 			return std::error_code();
 		}
@@ -709,7 +709,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 		RwLock																		StoreCompactorsLock;
 		std::unordered_map<std::unique_ptr<GcReferenceValidator>, size_t>			ReferenceValidators;
 		RwLock																		ReferenceValidatorsLock;
-		WorkerThreadPool&															PreCachePhaseThreadPool =
+		WorkerThreadPool&															ParallelWorkThreadPool =
 			  Settings.SingleThread ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Background);
 
 		if (!m_GcReferencers.empty())
@@ -721,7 +721,6 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 			ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size());
 			ZEN_TRACE_CPU("GcV2::RemoveExpiredData");
 
-			Latch WorkLeft(1);
 			{
 				// First remove any cache keys that may own references
 				SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) {
@@ -733,39 +732,45 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 				{
 					if (CheckGCCancel())
 					{
-						WorkLeft.CountDown();
-						WorkLeft.Wait();
 						return Sum(Result, true);
 					}
 					GcReferencer*							   Owner = m_GcReferencers[Index];
 					std::pair<std::string, GcReferencerStats>* Stats = &Result.ReferencerStats[Index];
-					WorkLeft.AddCount(1);
-					PreCachePhaseThreadPool.ScheduleWork([this, &Ctx, &WorkLeft, Owner, Stats, &StoreCompactorsLock, &StoreCompactors]() {
-						ZEN_MEMSCOPE(GetGcTag());
-
-						auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-						try
+					try
+					{
+						Stats->first = Owner->GetGcName(Ctx);
+						SCOPED_TIMER(Stats->second.RemoveExpiredDataStats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+						std::unique_ptr<GcStoreCompactor> StoreCompactor(
+							Owner->RemoveExpiredData(Ctx, Stats->second.RemoveExpiredDataStats));
+						if (StoreCompactor)
 						{
-							Stats->first = Owner->GetGcName(Ctx);
-							SCOPED_TIMER(Stats->second.RemoveExpiredDataStats.ElapsedMS =
-											 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-							std::unique_ptr<GcStoreCompactor> StoreCompactor(
-								Owner->RemoveExpiredData(Ctx, Stats->second.RemoveExpiredDataStats));
-							if (StoreCompactor)
-							{
-								RwLock::ExclusiveLockScope __(StoreCompactorsLock);
-								StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats);
-							}
+							RwLock::ExclusiveLockScope __(StoreCompactorsLock);
+							StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats);
 						}
-						catch (const std::exception& Ex)
+					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+						}
+						else
 						{
 							ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
-							SetCancelGC(true);
 						}
-					});
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 				}
-				WorkLeft.CountDown();
-				WorkLeft.Wait();
 			}
 		}
 
@@ -810,7 +815,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						GcReferenceStore*							   ReferenceStore = m_GcReferenceStores[Index];
 						std::pair<std::string, GcReferenceStoreStats>* Stats		  = &Result.ReferenceStoreStats[Index];
 						WorkLeft.AddCount(1);
-						PreCachePhaseThreadPool.ScheduleWork(
+						ParallelWorkThreadPool.ScheduleWork(
 							[this, &Ctx, ReferenceStore, Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
 								ZEN_MEMSCOPE(GetGcTag());
 
@@ -832,6 +837,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
 									}
 								}
+								catch (const std::system_error& Ex)
+								{
+									if (IsOOD(Ex) || IsOOM(Ex))
+									{
+										ZEN_WARN("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+												 ReferenceStore->GetGcName(Ctx),
+												 Ex.what());
+									}
+									else
+									{
+										ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+												  ReferenceStore->GetGcName(Ctx),
+												  Ex.what());
+									}
+									SetCancelGC(true);
+								}
+								catch (const std::bad_alloc& Ex)
+								{
+									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+											  ReferenceStore->GetGcName(Ctx),
+											  Ex.what());
+									SetCancelGC(true);
+								}
 								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
@@ -885,41 +913,70 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 								GcReferencer*							   Referencer = m_GcReferencers[Index];
 								std::pair<std::string, GcReferencerStats>* Stats	  = &Result.ReferencerStats[Index];
 								WorkLeft.AddCount(1);
-								PreCachePhaseThreadPool.ScheduleWork(
+								ParallelWorkThreadPool.ScheduleWork(
 									[this, &Ctx, &WorkLeft, Referencer, Index, Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
 										ZEN_MEMSCOPE(GetGcTag());
 
 										auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-										// The Referencer will create a reference checker that guarantees that the references do not change
-										// as long as it lives
-										std::vector<GcReferenceChecker*> Checkers;
-										try
+										if (!CheckGCCancel())
 										{
+											// The Referencer will create a reference checker that guarantees that the references do not
+											// change as long as it lives
+											std::vector<GcReferenceChecker*> Checkers;
+											auto							 __ = MakeGuard([&Checkers]() {
+												while (!Checkers.empty())
+												{
+													delete Checkers.back();
+													Checkers.pop_back();
+												}
+											});
+											try
 											{
-												SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS =
-																 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-												Checkers = Referencer->CreateReferenceCheckers(Ctx);
+												{
+													SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS =
+																	 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+													Checkers = Referencer->CreateReferenceCheckers(Ctx);
+												}
+												if (!Checkers.empty())
+												{
+													RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
+													for (auto& Checker : Checkers)
+													{
+														ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker),
+																						   Index);
+														Checker = nullptr;
+													}
+												}
 											}
-											if (!Checkers.empty())
+											catch (const std::system_error& Ex)
 											{
-												RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
-												for (auto& Checker : Checkers)
+												if (IsOOD(Ex) || IsOOM(Ex))
+												{
+													ZEN_WARN("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+															 Referencer->GetGcName(Ctx),
+															 Ex.what());
+												}
+												else
 												{
-													ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
-													Checker = nullptr;
+													ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+															  Referencer->GetGcName(Ctx),
+															  Ex.what());
 												}
+												SetCancelGC(true);
 											}
-										}
-										catch (const std::exception& Ex)
-										{
-											ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
-													  Referencer->GetGcName(Ctx),
-													  Ex.what());
-											SetCancelGC(true);
-											while (!Checkers.empty())
+											catch (const std::bad_alloc& Ex)
+											{
+												ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+														  Referencer->GetGcName(Ctx),
+														  Ex.what());
+												SetCancelGC(true);
+											}
+											catch (const std::exception& Ex)
 											{
-												delete Checkers.back();
-												Checkers.pop_back();
+												ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+														  Referencer->GetGcName(Ctx),
+														  Ex.what());
+												SetCancelGC(true);
 											}
 										}
 									});
@@ -962,19 +1019,26 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 							GcReferencer*							   Referencer	   = m_GcReferencers[Index];
 							std::pair<std::string, GcReferencerStats>* ReferemcerStats = &Result.ReferencerStats[Index];
 							WorkLeft.AddCount(1);
-							PreCachePhaseThreadPool.ScheduleWork([this,
-																  &Ctx,
-																  &WorkLeft,
-																  Referencer,
-																  Index,
-																  Result = &Result,
-																  ReferemcerStats,
-																  &ReferenceValidatorsLock,
-																  &ReferenceValidators]() {
+							ParallelWorkThreadPool.ScheduleWork([this,
+																 &Ctx,
+																 &WorkLeft,
+																 Referencer,
+																 Index,
+																 Result = &Result,
+																 ReferemcerStats,
+																 &ReferenceValidatorsLock,
+																 &ReferenceValidators]() {
 								ZEN_MEMSCOPE(GetGcTag());
 
 								auto							   _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
 								std::vector<GcReferenceValidator*> Validators;
+								auto							   __ = MakeGuard([&Validators]() {
+									  while (!Validators.empty())
+									  {
+										  delete Validators.back();
+										  Validators.pop_back();
+									  }
+								  });
 								try
 								{
 									{
@@ -995,17 +1059,35 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										}
 									}
 								}
+								catch (const std::system_error& Ex)
+								{
+									if (IsOOD(Ex) || IsOOM(Ex))
+									{
+										ZEN_WARN("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+												 Referencer->GetGcName(Ctx),
+												 Ex.what());
+									}
+									else
+									{
+										ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+												  Referencer->GetGcName(Ctx),
+												  Ex.what());
+									}
+									SetCancelGC(true);
+								}
+								catch (const std::bad_alloc& Ex)
+								{
+									ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+											  Referencer->GetGcName(Ctx),
+											  Ex.what());
+									SetCancelGC(true);
+								}
 								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
 											  Referencer->GetGcName(Ctx),
 											  Ex.what());
 									SetCancelGC(true);
-									while (!Validators.empty())
-									{
-										delete Validators.back();
-										Validators.pop_back();
-									}
 								}
 							});
 						}
@@ -1023,8 +1105,6 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 					ZEN_INFO("GCV2: Precaching state for {} reference checkers", ReferenceCheckers.size());
 					ZEN_TRACE_CPU("GcV2::PreCache");
 
-					Latch WorkLeft(1);
-
 					{
 						SCOPED_TIMER(Result.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
 									 if (Ctx.Settings.Verbose) {
@@ -1036,33 +1116,40 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						{
 							if (CheckGCCancel())
 							{
-								WorkLeft.CountDown();
-								WorkLeft.Wait();
 								return Sum(Result, true);
 							}
 
 							GcReferenceChecker*						   Checker = It.first.get();
 							size_t									   Index   = It.second;
 							std::pair<std::string, GcReferencerStats>* Stats   = &Result.ReferencerStats[Index];
-							WorkLeft.AddCount(1);
-							PreCachePhaseThreadPool.ScheduleWork([this, &Ctx, Checker, Index, Stats, &WorkLeft]() {
-								ZEN_MEMSCOPE(GetGcTag());
-
-								auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-								try
+							try
+							{
+								SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+								Checker->PreCache(Ctx);
+							}
+							catch (const std::system_error& Ex)
+							{
+								if (IsOOD(Ex) || IsOOM(Ex))
 								{
-									SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-									Checker->PreCache(Ctx);
+									ZEN_WARN("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
 								}
-								catch (const std::exception& Ex)
+								else
 								{
 									ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
-									SetCancelGC(true);
 								}
-							});
+								SetCancelGC(true);
+							}
+							catch (const std::bad_alloc& Ex)
+							{
+								ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+								SetCancelGC(true);
+							}
+							catch (const std::exception& Ex)
+							{
+								ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+								SetCancelGC(true);
+							}
 						}
-						WorkLeft.CountDown();
-						WorkLeft.Wait();
 					}
 				}
 
@@ -1081,7 +1168,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
 						{
 							ZEN_TRACE_CPU("GcV2::LockReferencers");
-							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until
 							// we delete the ReferenceLockers
 							Latch WorkLeft(1);
 							{
@@ -1108,7 +1195,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 							ZEN_TRACE_CPU("GcV2::UpdateLockedState");
 
 							// Locking all references checkers so we have a steady state of which references are used
-							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until
 							// we delete the ReferenceCheckers
 							Latch WorkLeft(1);
 
@@ -1142,6 +1229,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 															 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 											Checker->UpdateLockedState(Ctx);
 										}
+										catch (const std::system_error& Ex)
+										{
+											if (IsOOD(Ex) || IsOOM(Ex))
+											{
+												ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+														 Checker->GetGcName(Ctx),
+														 Ex.what());
+											}
+											else
+											{
+												ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+														  Checker->GetGcName(Ctx),
+														  Ex.what());
+											}
+											SetCancelGC(true);
+										}
+										catch (const std::bad_alloc& Ex)
+										{
+											ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+													 Checker->GetGcName(Ctx),
+													 Ex.what());
+											SetCancelGC(true);
+										}
 										catch (const std::exception& Ex)
 										{
 											ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'",
@@ -1231,6 +1341,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 											StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->CompactStoreStats);
 										}
 									}
+									catch (const std::system_error& Ex)
+									{
+										if (IsOOD(Ex) || IsOOM(Ex))
+										{
+											ZEN_WARN("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													 Pruner->GetGcName(Ctx),
+													 Ex.what());
+										}
+										else
+										{
+											ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													  Pruner->GetGcName(Ctx),
+													  Ex.what());
+										}
+										SetCancelGC(true);
+									}
+									catch (const std::bad_alloc& Ex)
+									{
+										ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+												  Pruner->GetGcName(Ctx),
+												  Ex.what());
+										SetCancelGC(true);
+									}
 									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
@@ -1262,12 +1395,12 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 			ZEN_TRACE_CPU("GcV2::CompactStores");
 
 			auto ClaimDiskReserve = [&]() -> uint64_t {
-				if (!std::filesystem::is_regular_file(Settings.DiskReservePath))
+				if (!IsFile(Settings.DiskReservePath))
 				{
 					return 0;
 				}
-				uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath);
-				if (std::filesystem::remove(Settings.DiskReservePath))
+				uint64_t ReclaimedSize = FileSizeFromPath(Settings.DiskReservePath);
+				if (RemoveFile(Settings.DiskReservePath))
 				{
 					return ReclaimedSize;
 				}
@@ -1294,6 +1427,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve);
 					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						}
+						else
+						{
+							ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						}
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 					catch (const std::exception& Ex)
 					{
 						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
@@ -1335,6 +1485,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						ReferenceValidator->Validate(Ctx, Stats);
 					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						}
+						else
+						{
+							ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						}
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 					catch (const std::exception& Ex)
 					{
 						ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
@@ -1557,7 +1724,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 		m_Config.LightweightInterval = m_Config.MonitorInterval;
 	}
 
-	std::filesystem::create_directories(Config.RootDirectory);
+	CreateDirectories(Config.RootDirectory);
 
 	std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
 	if (Ec)
@@ -1739,6 +1906,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons
 				{
 					Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime);
 					Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime);
+					Writer << "BuildStoreExpireTime"sv << ToDateTime(Settings.BuildStoreExpireTime);
 					Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects;
 					Writer << "IsDeleteMode"sv << Settings.IsDeleteMode;
 					Writer << "SkipCidDelete"sv << Settings.SkipCidDelete;
@@ -1849,7 +2017,7 @@ GcScheduler::GetState() const
 	if (Result.Config.DiskReserveSize != 0)
 	{
 		Ec.clear();
-		Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
+		Result.HasDiskReserve = IsFile(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
 	}
 
 	if (Result.Status != GcSchedulerStatus::kRunning)
@@ -1900,17 +2068,46 @@ GcScheduler::SchedulerThread()
 	ZEN_MEMSCOPE(GetGcTag());
 	SetCurrentThreadName("GcScheduler");
 
-	std::chrono::seconds WaitTime{0};
-
-	bool SilenceErrors = false;
+	std::chrono::seconds	   WaitTime{0};
+	const std::chrono::seconds ShortWaitTime{5};
+	bool					   SilenceErrors = false;
 
 	for (;;)
 	{
-		bool Timeout = false;
+		(void)CheckDiskSpace();
+
+		std::chrono::seconds WaitedTime{0};
+		bool				 Timeout = false;
 		{
 			ZEN_ASSERT(WaitTime.count() >= 0);
 			std::unique_lock Lock(m_GcMutex);
-			Timeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, WaitTime);
+			while (!Timeout)
+			{
+				std::chrono::seconds ShortWait	  = Min(WaitTime, ShortWaitTime);
+				bool				 ShortTimeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, ShortWait);
+				if (ShortTimeout)
+				{
+					if (WaitTime > ShortWaitTime)
+					{
+						DiskSpace Space = CheckDiskSpace();
+						if (!AreDiskWritesAllowed())
+						{
+							ZEN_INFO("Triggering GC due to low disk space ({}) on {}", NiceBytes(Space.Free), m_Config.RootDirectory);
+							Timeout = true;
+						}
+						WaitTime -= ShortWaitTime;
+					}
+					else
+					{
+						Timeout = true;
+					}
+				}
+				else
+				{
+					// We got a signal
+					break;
+				}
+			}
 		}
 
 		if (Status() == GcSchedulerStatus::kStopped)
@@ -1940,7 +2137,9 @@ GcScheduler::SchedulerThread()
 			std::chrono::seconds LightweightGcInterval			   = m_Config.LightweightInterval;
 			std::chrono::seconds MaxCacheDuration				   = m_Config.MaxCacheDuration;
 			std::chrono::seconds MaxProjectStoreDuration		   = m_Config.MaxProjectStoreDuration;
+			std::chrono::seconds MaxBuildStoreDuration			   = m_Config.MaxBuildStoreDuration;
 			uint64_t			 DiskSizeSoftLimit				   = m_Config.DiskSizeSoftLimit;
+			uint64_t			 MinimumFreeDiskSpaceToAllowWrites = m_Config.MinimumFreeDiskSpaceToAllowWrites;
 			bool				 SkipCid						   = false;
 			GcVersion			 UseGCVersion					   = m_Config.UseGCVersion;
 			uint32_t			 CompactBlockUsageThresholdPercent = m_Config.CompactBlockUsageThresholdPercent;
@@ -1955,8 +2154,9 @@ GcScheduler::SchedulerThread()
 			uint8_t NextAttachmentPassIndex =
 				ComputeAttachmentRange(m_AttachmentPassIndex, m_Config.AttachmentPassCount, AttachmentRangeMin, AttachmentRangeMax);
 
-			bool DiskSpaceGCTriggered = false;
-			bool TimeBasedGCTriggered = false;
+			bool LowDiskSpaceGCTriggered	   = false;
+			bool HighDiskSpaceUsageGCTriggered = false;
+			bool TimeBasedGCTriggered		   = false;
 
 			GcClock::TimePoint Now = GcClock::Now();
 
@@ -1975,6 +2175,10 @@ GcScheduler::SchedulerThread()
 				{
 					MaxProjectStoreDuration = TriggerParams.MaxProjectStoreDuration;
 				}
+				if (TriggerParams.MaxBuildStoreDuration != std::chrono::seconds::max())
+				{
+					MaxBuildStoreDuration = TriggerParams.MaxBuildStoreDuration;
+				}
 				if (TriggerParams.DiskSizeSoftLimit != 0)
 				{
 					DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit;
@@ -2046,6 +2250,8 @@ GcScheduler::SchedulerThread()
 				MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
 			GcClock::TimePoint ProjectStoreExpireTime =
 				MaxProjectStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxProjectStoreDuration;
+			GcClock::TimePoint BuildStoreExpireTime =
+				MaxBuildStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxBuildStoreDuration;
 
 			const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
 
@@ -2087,12 +2293,32 @@ GcScheduler::SchedulerThread()
 					}
 				}
 
-				uint64_t GcDiskSpaceGoal = 0;
+				uint64_t MaximumDiskUseGcSpaceGoal	= 0;
+				uint64_t MinimumFreeDiskGcSpaceGoal = 0;
+
 				if (DiskSizeSoftLimit != 0 && TotalSize.DiskSize > DiskSizeSoftLimit)
 				{
-					GcDiskSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit;
+					MaximumDiskUseGcSpaceGoal	  = TotalSize.DiskSize - DiskSizeSoftLimit;
+					HighDiskSpaceUsageGCTriggered = true;
+				}
+
+				if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free < MinimumFreeDiskSpaceToAllowWrites)
+				{
+					MinimumFreeDiskGcSpaceGoal = MinimumFreeDiskSpaceToAllowWrites - Space.Free;
+					if (MinimumFreeDiskGcSpaceGoal > MaximumDiskUseGcSpaceGoal)
+					{
+						LowDiskSpaceGCTriggered = true;
+						EnableValidation		= false;
+					}
+				}
+
+				if (MaximumDiskUseGcSpaceGoal > 0 || MinimumFreeDiskGcSpaceGoal > 0)
+				{
+					const uint64_t GcDiskSpaceRemoveGoal = Max(MaximumDiskUseGcSpaceGoal, MinimumFreeDiskGcSpaceGoal);
+
 					std::unique_lock Lock(m_GcMutex);
-					GcClock::Tick	 AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count());
+					GcClock::Tick	 AgeTick =
+						m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceRemoveGoal, Now.time_since_epoch().count());
 					GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick);
 					if (SizeBasedExpireTime > CacheExpireTime)
 					{
@@ -2102,6 +2328,10 @@ GcScheduler::SchedulerThread()
 					{
 						ProjectStoreExpireTime = SizeBasedExpireTime;
 					}
+					if (SizeBasedExpireTime > BuildStoreExpireTime)
+					{
+						BuildStoreExpireTime = SizeBasedExpireTime;
+					}
 				}
 
 				std::chrono::seconds RemainingTimeUntilGc =
@@ -2130,29 +2360,33 @@ GcScheduler::SchedulerThread()
 					RemainingTimeUntilLightweightGc = RemainingTimeUntilGc;
 				}
 
-				if (GcDiskSpaceGoal > 0)
-				{
-					DiskSpaceGCTriggered = true;
-				}
-				else if (RemainingTimeUntilGc.count() == 0)
-				{
-					TimeBasedGCTriggered = true;
-				}
-				else if (RemainingTimeUntilLightweightGc.count() == 0)
+				if (MaximumDiskUseGcSpaceGoal == 0 && MinimumFreeDiskGcSpaceGoal == 0)
 				{
-					TimeBasedGCTriggered = true;
-					SkipCid				 = true;
+					if (RemainingTimeUntilGc.count() == 0)
+					{
+						TimeBasedGCTriggered = true;
+					}
+					else if (RemainingTimeUntilLightweightGc.count() == 0)
+					{
+						TimeBasedGCTriggered = true;
+						SkipCid				 = true;
+					}
 				}
 
 				std::string NextTriggerStatus;
-				if (GcInterval.count() != 0 || LightweightGcInterval.count() != 0 || DiskSizeSoftLimit != 0)
 				{
 					ExtendableStringBuilder<256> Sb;
-					if (DiskSpaceGCTriggered)
+					if (LowDiskSpaceGCTriggered)
+					{
+						Sb.Append(fmt::format(" Free disk space is below {}, trying to reclaim {}.",
+											  NiceBytes(MinimumFreeDiskSpaceToAllowWrites),
+											  NiceBytes(MinimumFreeDiskGcSpaceGoal)));
+					}
+					else if (HighDiskSpaceUsageGCTriggered)
 					{
 						Sb.Append(fmt::format(" Disk space exceeds {}, trying to reclaim {}.",
 											  NiceBytes(DiskSizeSoftLimit),
-											  NiceBytes(GcDiskSpaceGoal)));
+											  NiceBytes(MaximumDiskUseGcSpaceGoal)));
 					}
 					else if (TimeBasedGCTriggered)
 					{
@@ -2182,6 +2416,10 @@ GcScheduler::SchedulerThread()
 						{
 							Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(DiskSizeSoftLimit - TotalSize.DiskSize)));
 						}
+						else if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free > MinimumFreeDiskSpaceToAllowWrites)
+						{
+							Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(Space.Free - MinimumFreeDiskSpaceToAllowWrites)));
+						}
 					}
 					NextTriggerStatus = Sb;
 				}
@@ -2198,7 +2436,7 @@ GcScheduler::SchedulerThread()
 						 NiceBytes(MaxLoad / uint64_t(std::chrono::seconds(m_Config.MonitorInterval).count())),
 						 NextTriggerStatus);
 
-				if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered)
+				if (!HighDiskSpaceUsageGCTriggered && !LowDiskSpaceGCTriggered && !TimeBasedGCTriggered)
 				{
 					WaitTime = m_Config.MonitorInterval;
 					if (RemainingTimeUntilGc < WaitTime)
@@ -2227,6 +2465,7 @@ GcScheduler::SchedulerThread()
 
 			bool GcSuccess = CollectGarbage(CacheExpireTime,
 											ProjectStoreExpireTime,
+											BuildStoreExpireTime,
 											DoDelete,
 											CollectSmallObjects,
 											SkipCid,
@@ -2333,6 +2572,7 @@ GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds Time
 bool
 GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							const GcClock::TimePoint& ProjectStoreExpireTime,
+							const GcClock::TimePoint& BuildStoreExpireTime,
 							bool					  Delete,
 							bool					  CollectSmallObjects,
 							bool					  SkipCid,
@@ -2375,12 +2615,12 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 		{
 			// We are low on disk, check if we can release our extra storage reserve, if we can't bail from doing GC
 			auto ClaimDiskReserve = [&]() -> uint64_t {
-				if (!std::filesystem::is_regular_file(DiskReservePath))
+				if (!IsFile(DiskReservePath))
 				{
 					return 0;
 				}
-				uint64_t ReclaimedSize = std::filesystem::file_size(DiskReservePath);
-				if (std::filesystem::remove(DiskReservePath))
+				uint64_t ReclaimedSize = FileSizeFromPath(DiskReservePath);
+				if (RemoveFile(DiskReservePath))
 				{
 					return ReclaimedSize;
 				}
@@ -2416,6 +2656,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 
 						const GcSettings Settings = {.CacheExpireTime					= CacheExpireTime,
 													 .ProjectStoreExpireTime			= ProjectStoreExpireTime,
+													 .BuildStoreExpireTime				= BuildStoreExpireTime,
 													 .CollectSmallObjects				= CollectSmallObjects,
 													 .IsDeleteMode						= Delete,
 													 .SkipCidDelete						= SkipCid,
@@ -2447,6 +2688,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							}
 							SB.Append(fmt::format("      Cache cutoff time:           {}\n", Settings.CacheExpireTime));
 							SB.Append(fmt::format("      Project store cutoff time:   {}\n", Settings.ProjectStoreExpireTime));
+							SB.Append(fmt::format("      Build store cutoff time:     {}\n", Settings.BuildStoreExpireTime));
 						};
 
 						{
@@ -2522,7 +2764,11 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 
 						ZEN_INFO("GCV2: {}", SB.ToView());
 
-						AppendGCLog(GcId, GcStartTime, Settings, Result);
+						CheckDiskSpace();
+						if (!m_AreDiskWritesBlocked.load())
+						{
+							AppendGCLog(GcId, GcStartTime, Settings, Result);
+						}
 
 						if (SkipCid)
 						{
@@ -2552,6 +2798,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				if (Delete)
 				{
 					GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime);
+					KeepRangeStart					  = Min(KeepRangeStart, BuildStoreExpireTime);
 					m_LastGcExpireTime				  = KeepRangeStart;
 					std::unique_lock Lock(m_GcMutex);
 					m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count());
@@ -2563,65 +2810,69 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				m_LastFullGCDiff		= Diff;
 			}
 
-			for (uint32_t RetryCount = 0; RetryCount < 3; RetryCount++)
+			CheckDiskSpace();
+			if (!m_AreDiskWritesBlocked.load())
 			{
-				if (RetryCount > 0)
+				for (uint32_t RetryCount = 0; RetryCount < 3; RetryCount++)
 				{
-					ZEN_INFO("Writing GC state failed {} time(s), pausing and trying again", RetryCount);
-					Sleep(250);
-				}
-				try
-				{
-					const fs::path Path = m_Config.RootDirectory / "gc_state";
-					ZEN_DEBUG("saving scheduler state to '{}'", Path);
-					CbObjectWriter SchedulerState;
-					SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
-					SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
-					SchedulerState << "AttachmentPassIndex"sv << m_AttachmentPassIndex;
-
-					SaveCompactBinaryObject(Path, SchedulerState.Save());
 					if (RetryCount > 0)
 					{
-						ZEN_INFO("Writing GC state succeeded after {} attempts", RetryCount + 1);
-					}
-					break;
-				}
-				catch (const std::system_error& SystemError)
-				{
-					if (IsOOM(SystemError.code()))
-					{
-						ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", SystemError.what());
+						ZEN_INFO("Writing GC state failed {} time(s), pausing and trying again", RetryCount);
+						Sleep(250);
 					}
-					else if (IsOOD(SystemError.code()))
-					{
-						ZEN_WARN("writing gc scheduler state ran out of disk space: '{}'", SystemError.what());
-					}
-					if (RetryCount == 0)
+					try
 					{
-						ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}' ({})",
-								  SystemError.what(),
-								  SystemError.code().value());
+						const fs::path Path = m_Config.RootDirectory / "gc_state";
+						ZEN_DEBUG("saving scheduler state to '{}'", Path);
+						CbObjectWriter SchedulerState;
+						SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
+						SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
+						SchedulerState << "AttachmentPassIndex"sv << m_AttachmentPassIndex;
+
+						SaveCompactBinaryObject(Path, SchedulerState.Save());
+						if (RetryCount > 0)
+						{
+							ZEN_INFO("Writing GC state succeeded after {} attempts", RetryCount + 1);
+						}
+						break;
 					}
-					else
+					catch (const std::system_error& SystemError)
 					{
-						ZEN_WARN("writing gc scheduler state failed with system error exception: '{}' ({})",
-								 SystemError.what(),
-								 SystemError.code().value());
+						if (IsOOM(SystemError.code()))
+						{
+							ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", SystemError.what());
+						}
+						else if (IsOOD(SystemError.code()))
+						{
+							ZEN_WARN("writing gc scheduler state ran out of disk space: '{}'", SystemError.what());
+						}
+						else if (RetryCount == 0)
+						{
+							ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}' ({})",
+									  SystemError.what(),
+									  SystemError.code().value());
+						}
+						else
+						{
+							ZEN_WARN("writing gc scheduler state failed with system error exception: '{}' ({})",
+									 SystemError.what(),
+									 SystemError.code().value());
+						}
 					}
-				}
-				catch (const std::bad_alloc& BadAlloc)
-				{
-					ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what());
-				}
-				catch (const std::exception& Ex)
-				{
-					if (RetryCount == 0)
+					catch (const std::bad_alloc& BadAlloc)
 					{
-						ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what());
+						ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what());
 					}
-					else
+					catch (const std::exception& Ex)
 					{
-						ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what());
+						if (RetryCount == 0)
+						{
+							ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what());
+						}
+						else
+						{
+							ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what());
+						}
 					}
 				}
 			}
diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h
new file mode 100644
index 000000000..e53937b52
--- /dev/null
+++ b/src/zenstore/include/zenstore/accesstime.h
@@ -0,0 +1,53 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenstore/gc.h>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch
+struct AccessTime
+{
+	explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSecondsSinceEpoch(Tick)) {}
+	AccessTime& operator=(GcClock::Tick Tick) noexcept
+	{
+		SecondsSinceEpoch.store(ToSecondsSinceEpoch(Tick), std::memory_order_relaxed);
+		return *this;
+	}
+	operator GcClock::Tick() const noexcept
+	{
+		return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed)))
+			.count();
+	}
+
+	AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
+	AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
+	AccessTime& operator=(AccessTime&& Rhs) noexcept
+	{
+		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
+		return *this;
+	}
+	AccessTime& operator=(const AccessTime& Rhs) noexcept
+	{
+		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
+		return *this;
+	}
+
+	void SetSecondsSinceEpoch(uint32_t InSecondsSinceEpoch) { SecondsSinceEpoch.store(InSecondsSinceEpoch); }
+
+	uint32_t GetSecondsSinceEpoch() const { return SecondsSinceEpoch.load(); }
+
+private:
+	AccessTime(uint32_t InSecondsSinceEpoch) noexcept : SecondsSinceEpoch(InSecondsSinceEpoch) {}
+
+	static uint32_t ToSecondsSinceEpoch(GcClock::Tick Tick)
+	{
+		return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count());
+	}
+	std::atomic_uint32_t SecondsSinceEpoch;
+};
+
+}  // namespace zen
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 97357e5cb..fce05766f 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -94,7 +94,7 @@ struct BlockStoreFile : public RefCounted
 	IoBuffer					 GetChunk(uint64_t Offset, uint64_t Size);
 	void						 Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
-	void						 Flush();
+	void						 Flush(uint64_t FinalSize = (uint64_t)-1);
 	BasicFile&					 GetBasicFile();
 	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
 	bool	 IsOpen() const;
@@ -107,7 +107,7 @@ private:
 	const std::filesystem::path m_Path;
 	IoBuffer					m_IoBuffer;
 	BasicFile					m_File;
-	uint64_t					m_CachedFileSize = 0;
+	std::atomic<uint64_t>		m_CachedFileSize = 0;
 };
 
 class BlockStoreCompactState;
@@ -127,7 +127,8 @@ public:
 	typedef std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray;
 	typedef std::vector<size_t>								   ChunkIndexArray;
 
-	typedef std::function<bool(const MovedChunksArray& MovedChunks, uint64_t FreedDiskSpace)>			 CompactCallback;
+	typedef std::function<bool(const MovedChunksArray& MovedChunks, const ChunkIndexArray& ScrubbedChunks, uint64_t FreedDiskSpace)>
+																										 CompactCallback;
 	typedef std::function<uint64_t()>																	 ClaimDiskReserveCallback;
 	typedef std::function<bool(size_t ChunkIndex, const void* Data, uint64_t Size)>						 IterateChunksSmallSizeCallback;
 	typedef std::function<bool(size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size)> IterateChunksLargeSizeCallback;
@@ -146,18 +147,19 @@ public:
 
 	typedef tsl::robin_set<uint32_t> BlockIndexSet;
 
-	// Ask the store to create empty blocks for all locations that does not have a block
 	// Remove any block that is not referenced
-	void			   SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks);
-	BlockEntryCountMap GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent);
+	// Return a list of blocks that are not present
+	[[nodiscard]] BlockIndexSet SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks);
+	BlockEntryCountMap			GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent);
 
 	void Close();
 
 	void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback);
 
 	typedef std::function<void(std::span<BlockStoreLocation> Locations)> WriteChunksCallback;
-	void WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback);
+	void WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback);
 
+	bool	 HasChunk(const BlockStoreLocation& Location) const;
 	IoBuffer TryGetChunk(const BlockStoreLocation& Location) const;
 	void	 Flush(bool ForceNewBlock);
 
@@ -172,7 +174,7 @@ public:
 	void CompactBlocks(
 		const BlockStoreCompactState&	CompactState,
 		uint32_t						PayloadAlignment,
-		const CompactCallback&			ChangeCallback		= [](const MovedChunksArray&, uint64_t) { return true; },
+		const CompactCallback&			ChangeCallback = [](const MovedChunksArray&, const ChunkIndexArray&, uint64_t) { return true; },
 		const ClaimDiskReserveCallback& DiskReserveCallback = []() { return 0; },
 		std::string_view				LogPrefix			= {});
 
diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h
new file mode 100644
index 000000000..adf48dc26
--- /dev/null
+++ b/src/zenstore/include/zenstore/buildstore/buildstore.h
@@ -0,0 +1,228 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenstore/blockstore.h>
+
+#include <zencore/iohash.h>
+#include <zenstore/accesstime.h>
+#include <zenstore/caslog.h>
+#include <zenstore/gc.h>
+#include "../compactcas.h"
+#include "../filecas.h"
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+struct BuildStoreConfig
+{
+	std::filesystem::path RootDirectory;
+	uint32_t			  SmallBlobBlockStoreMaxBlockSize	   = 256 * 1024 * 1024;
+	uint64_t			  SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024;
+	uint32_t			  SmallBlobBlockStoreAlignement		   = 16;
+	uint32_t			  MetadataBlockStoreMaxBlockSize	   = 64 * 1024 * 1024;
+	uint32_t			  MetadataBlockStoreAlignement		   = 8;
+	uint64_t			  MaxDiskSpaceLimit					   = 1u * 1024u * 1024u * 1024u * 1024u;  // 1TB
+};
+
+class BuildStore : public GcReferencer, public GcReferenceLocker, public GcStorage
+{
+public:
+	explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc);
+	virtual ~BuildStore();
+
+	void	 PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload);
+	IoBuffer GetBlob(const IoHash& BlobHashes);
+
+	struct BlobExistsResult
+	{
+		bool HasBody	 = 0;
+		bool HasMetadata = 0;
+	};
+
+	std::vector<BlobExistsResult> BlobsExists(std::span<const IoHash> BlobHashes);
+
+	void				  PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas);
+	std::vector<IoBuffer> GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool);
+
+	void Flush();
+
+	struct StorageStats
+	{
+		uint64_t EntryCount		   = 0;
+		uint64_t LargeBlobCount	   = 0;
+		uint64_t LargeBlobBytes	   = 0;
+		uint64_t SmallBlobCount	   = 0;
+		uint64_t SmallBlobBytes	   = 0;
+		uint64_t MetadataCount	   = 0;
+		uint64_t MetadataByteCount = 0;
+	};
+
+	StorageStats GetStorageStats() const;
+
+#if ZEN_WITH_TESTS
+	std::optional<AccessTime> GetLastAccessTime(const IoHash& Key) const;
+	bool					  SetLastAccessTime(const IoHash& Key, const AccessTime& Time);
+#endif	// ZEN_WITH_TESTS
+
+private:
+	LoggerRef Log() { return m_Log; }
+
+	void CompactState();
+
+	uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
+	uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
+	void	 WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath);
+	void	 ReadAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath);
+
+	//////// GcReferencer
+	virtual std::string						   GetGcName(GcCtx& Ctx) override;
+	virtual GcStoreCompactor*				   RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
+	virtual std::vector<GcReferenceChecker*>   CreateReferenceCheckers(GcCtx& Ctx) override;
+	virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override;
+
+	//////// GcReferenceLocker
+	virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override;
+
+	//////// GcStorage
+	virtual void		  ScrubStorage(ScrubContext& ScrubCtx) override;
+	virtual GcStorageSize StorageSize() const override;
+
+#pragma pack(push)
+#pragma pack(1)
+	struct PayloadEntry
+	{
+		PayloadEntry() {}
+		PayloadEntry(uint64_t Flags, uint64_t Size)
+		{
+			ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size);
+			ZEN_ASSERT((Flags & (kTombStone | kStandalone)) == Flags);
+			FlagsAndSize = (Size << 8) | Flags;
+		}
+		static const uint8_t kTombStone	 = 0x10u;  // Represents a deleted key/value
+		static const uint8_t kStandalone = 0x20u;  // This payload is stored as a standalone value
+
+		uint64_t FlagsAndSize = 0;
+		uint64_t GetSize() const { return FlagsAndSize >> 8; }
+		uint8_t	 GetFlags() const { return uint8_t(FlagsAndSize & 0xff); }
+		void	 AddFlag(uint8_t Flag) { FlagsAndSize |= Flag; }
+		void	 SetSize(uint64_t Size)
+		{
+			ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size);
+			FlagsAndSize = (Size << 8) | (FlagsAndSize & 0xff);
+		}
+		void SetFlags(uint8_t Flags) { FlagsAndSize = (FlagsAndSize & 0xffffffffffffff00u) | Flags; }
+	};
+	static_assert(sizeof(PayloadEntry) == 8);
+
+	struct PayloadDiskEntry
+	{
+		PayloadEntry Entry;		// 8 bytes
+		IoHash		 BlobHash;	// 20 bytes
+	};
+	static_assert(sizeof(PayloadDiskEntry) == 28);
+
+	struct MetadataEntry
+	{
+		BlockStoreLocation Location;  // 12 bytes
+
+		ZenContentType		 ContentType = ZenContentType::kCOUNT;	// 1 byte
+		static const uint8_t kTombStone	 = 0x10u;					// Represents a deleted key/value
+		uint8_t				 Flags		 = 0;						// 1 byte
+
+		uint8_t Reserved1 = 0;
+		uint8_t Reserved2 = 0;
+	};
+	static_assert(sizeof(MetadataEntry) == 16);
+
+	struct MetadataDiskEntry
+	{
+		MetadataEntry Entry;	 // 16 bytes
+		IoHash		  BlobHash;	 // 20 bytes
+		uint8_t		  Reserved1 = 0;
+		uint8_t		  Reserved2 = 0;
+		uint8_t		  Reserved3 = 0;
+		uint8_t		  Reserved4 = 0;
+	};
+	static_assert(sizeof(MetadataDiskEntry) == 40);
+
+#pragma pack(pop)
+
+	static bool ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason);
+	static bool ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason);
+
+	struct PayloadIndex
+	{
+		uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+		operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+		PayloadIndex() = default;
+		explicit PayloadIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+					operator size_t() const { return Index; };
+		inline auto operator<=>(const PayloadIndex& Other) const = default;
+	};
+
+	struct MetadataIndex
+	{
+		uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+		operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+		MetadataIndex() = default;
+		explicit MetadataIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+					operator size_t() const { return Index; };
+		inline auto operator<=>(const MetadataIndex& Other) const = default;
+	};
+
+	struct BlobIndex
+	{
+		uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+		operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+		BlobIndex() = default;
+		explicit BlobIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+					operator size_t() const { return Index; };
+		inline auto operator<=>(const BlobIndex& Other) const = default;
+	};
+
+	struct BlobEntry
+	{
+		PayloadIndex  Payload;
+		MetadataIndex Metadata;
+		AccessTime	  LastAccessTime;
+	};
+	static_assert(sizeof(BlobEntry) == 12);
+
+	LoggerRef			   m_Log;
+	const BuildStoreConfig m_Config;
+	GcManager&			   m_Gc;
+
+	mutable RwLock m_Lock;
+
+	std::vector<PayloadEntry>  m_PayloadEntries;
+	std::vector<MetadataEntry> m_MetadataEntries;
+
+	std::vector<BlobEntry>							  m_BlobEntries;
+	tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> m_BlobLookup;
+
+	FileCasStrategy		 m_LargeBlobStore;
+	CasContainerStrategy m_SmallBlobStore;
+	BlockStore			 m_MetadataBlockStore;
+
+	TCasLogFile<PayloadDiskEntry>  m_PayloadlogFile;
+	TCasLogFile<MetadataDiskEntry> m_MetadatalogFile;
+	uint64_t					   m_BlobLogFlushPosition = 0;
+	uint64_t					   m_MetaLogFlushPosition = 0;
+
+	std::unique_ptr<HashSet> m_TrackedCacheKeys;
+	std::atomic<uint64_t>	 m_LastAccessTimeUpdateCount;
+
+	friend class BuildStoreGcReferenceChecker;
+	friend class BuildStoreGcReferencePruner;
+	friend class BuildStoreGcCompator;
+};
+
+void buildstore_forcelink();
+
+}  // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 05400c784..3cd2d6423 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -5,6 +5,7 @@
 #include "cacheshared.h"
 
 #include <zencore/stats.h>
+#include <zenstore/accesstime.h>
 #include <zenstore/blockstore.h>
 #include <zenstore/caslog.h>
 
@@ -118,6 +119,9 @@ public:
 
 	struct Configuration
 	{
+		typedef eastl::unordered_map<std::string, BucketConfiguration, std::hash<std::string>, std::equal_to<std::string>>
+							BucketConfigMap_t;
+		BucketConfigMap_t	BucketConfigMap;
 		BucketConfiguration BucketConfig;
 		uint64_t			MemCacheTargetFootprintBytes = 512 * 1024 * 1024;
 		uint64_t			MemCacheTrimIntervalSeconds	 = 60;
@@ -179,15 +183,15 @@ public:
 	PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult);
 	void			EndPutBatch(PutBatchHandle* Batch) noexcept;
 
-	void Put(std::string_view	  Bucket,
-			 const IoHash&		  HashKey,
-			 const ZenCacheValue& Value,
-			 std::span<IoHash>	  References,
-			 PutBatchHandle*	  OptionalBatchHandle);
-	bool Drop();
-	bool DropBucket(std::string_view Bucket);
-	void Flush();
-	void ScrubStorage(ScrubContext& Ctx);
+	void				  Put(std::string_view	   Bucket,
+							  const IoHash&		   HashKey,
+							  const ZenCacheValue& Value,
+							  std::span<IoHash>	   References,
+							  PutBatchHandle*	   OptionalBatchHandle);
+	std::function<void()> Drop();
+	std::function<void()> DropBucket(std::string_view Bucket);
+	void				  Flush();
+	void				  ScrubStorage(ScrubContext& Ctx);
 
 	void		  DiscoverBuckets();
 	GcStorageSize StorageSize() const;
@@ -236,9 +240,9 @@ public:
 		void			EndPutBatch(PutBatchHandle* Batch) noexcept;
 		void	 Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, PutBatchHandle* OptionalBatchHandle);
 		uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime);
-		bool	 Drop();
-		void	 Flush();
-		void	 ScrubStorage(ScrubContext& Ctx);
+		std::function<void()>	Drop();
+		void					Flush();
+		void					ScrubStorage(ScrubContext& Ctx);
 		RwLock::SharedLockScope GetGcReferencerLock();
 
 		struct ReferencesStats
@@ -405,20 +409,23 @@ public:
 		void SaveSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
 		void WriteIndexSnapshot(
 			RwLock::ExclusiveLockScope&,
-			bool							 FlushLockPosition,
+			uint64_t						 LogPosition,
+			bool							 ResetLog,
 			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
 		{
-			WriteIndexSnapshotLocked(FlushLockPosition, ClaimDiskReserveFunc);
+			WriteIndexSnapshotLocked(LogPosition, ResetLog, ClaimDiskReserveFunc);
 		}
 		void WriteIndexSnapshot(
 			RwLock::SharedLockScope&,
-			bool							 FlushLockPosition,
+			uint64_t						 LogPosition,
+			bool							 ResetLog,
 			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
 		{
-			WriteIndexSnapshotLocked(FlushLockPosition, ClaimDiskReserveFunc);
+			WriteIndexSnapshotLocked(LogPosition, ResetLog, ClaimDiskReserveFunc);
 		}
 		void WriteIndexSnapshotLocked(
-			bool							 FlushLockPosition,
+			uint64_t						 LogPosition,
+			bool							 ResetLog,
 			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
 
 		void CompactState(RwLock::ExclusiveLockScope&  IndexLock,
diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h
index 521c78bb1..ef1b803de 100644
--- a/src/zenstore/include/zenstore/cache/cacheshared.h
+++ b/src/zenstore/include/zenstore/cache/cacheshared.h
@@ -72,42 +72,4 @@ struct CacheContentStats
 bool IsKnownBadBucketName(std::string_view BucketName);
 bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer);
 
-//////////////////////////////////////////////////////////////////////////
-
-// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch
-struct AccessTime
-{
-	explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {}
-	AccessTime& operator=(GcClock::Tick Tick) noexcept
-	{
-		SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed);
-		return *this;
-	}
-	operator GcClock::Tick() const noexcept
-	{
-		return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed)))
-			.count();
-	}
-
-	AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
-	AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
-	AccessTime& operator=(AccessTime&& Rhs) noexcept
-	{
-		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
-		return *this;
-	}
-	AccessTime& operator=(const AccessTime& Rhs) noexcept
-	{
-		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
-		return *this;
-	}
-
-private:
-	static uint32_t ToSeconds(GcClock::Tick Tick)
-	{
-		return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count());
-	}
-	std::atomic_uint32_t SecondsSinceEpoch;
-};
-
 }  // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h
index 5e056cf2d..48fc17960 100644
--- a/src/zenstore/include/zenstore/cache/structuredcachestore.h
+++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h
@@ -101,8 +101,8 @@ public:
 	void EnumerateBucketContents(std::string_view																		 Bucket,
 								 std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
 
-	bool Drop();
-	void Flush();
+	std::function<void()> Drop();
+	void				  Flush();
 
 	// GcStorage
 	virtual void		  ScrubStorage(ScrubContext& ScrubCtx) override;
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 3daae0a93..3223fba39 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -55,6 +55,7 @@ struct GcSettings
 {
 	GcClock::TimePoint CacheExpireTime		  = GcClock::Now();
 	GcClock::TimePoint ProjectStoreExpireTime = GcClock::Now();
+	GcClock::TimePoint BuildStoreExpireTime	  = GcClock::Now();
 	bool			   CollectSmallObjects	  = false;
 	bool			   IsDeleteMode			  = false;
 	bool			   SkipCidDelete		  = false;
@@ -412,6 +413,7 @@ struct GcSchedulerConfig
 	std::chrono::seconds  Interval{};
 	std::chrono::seconds  MaxCacheDuration{86400};
 	std::chrono::seconds  MaxProjectStoreDuration{604800};
+	std::chrono::seconds  MaxBuildStoreDuration{604800};
 	bool				  CollectSmallObjects				= true;
 	bool				  Enabled							= true;
 	uint64_t			  DiskReserveSize					= 1ul << 28;
@@ -496,6 +498,7 @@ public:
 		bool					 CollectSmallObjects	 = false;
 		std::chrono::seconds	 MaxCacheDuration		 = std::chrono::seconds::max();
 		std::chrono::seconds	 MaxProjectStoreDuration = std::chrono::seconds::max();
+		std::chrono::seconds	 MaxBuildStoreDuration	 = std::chrono::seconds::max();
 		uint64_t				 DiskSizeSoftLimit		 = 0;
 		bool					 SkipCid				 = false;
 		bool					 SkipDelete				 = false;
@@ -528,6 +531,7 @@ private:
 	void		 SchedulerThread();
 	bool		 CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 								const GcClock::TimePoint& ProjectStoreExpireTime,
+								const GcClock::TimePoint& BuildStoreExpireTime,
 								bool					  Delete,
 								bool					  CollectSmallObjects,
 								bool					  SkipCid,
@@ -582,16 +586,3 @@ private:
 void gc_forcelink();
 
 }  // namespace zen
-
-template<>
-struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
-{
-	template<typename FormatContext>
-	auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx) const
-	{
-		std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
-		char		TimeString[std::size("yyyy-mm-ddThh:mm:ss")];
-		std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time));
-		return fmt::format_to(ctx.out(), "{}", TimeString);
-	}
-};
diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp
index 02a83d2a6..0ca2adab2 100644
--- a/src/zenstore/workspaces.cpp
+++ b/src/zenstore/workspaces.cpp
@@ -444,7 +444,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId)
 	{
 		const std::filesystem::path& RootPath	= Workspace->GetConfig().RootPath;
 		std::filesystem::path		 ConfigPath = RootPath / WorkspaceConfigName;
-		if (std::filesystem::exists(ConfigPath))
+		if (IsFile(ConfigPath))
 		{
 			std::string											 Error;
 			std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error);
@@ -458,7 +458,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId)
 				{
 					const std::filesystem::path& SharePath = Configuration.SharePath;
 
-					if (std::filesystem::is_directory(RootPath / SharePath))
+					if (IsDir(RootPath / SharePath))
 					{
 						DeletedShares.erase(Configuration.Id);
 
@@ -808,7 +808,7 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work
 	ZEN_DEBUG("Reading workspaces state from {}", WorkspaceStatePath);
 
 	const std::filesystem::path ConfigPath = WorkspaceStatePath / WorkspacesConfigName;
-	if (std::filesystem::exists(ConfigPath))
+	if (IsFile(ConfigPath))
 	{
 		std::vector<Workspaces::WorkspaceConfiguration> Workspaces =
 			WorkspacesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError);
@@ -847,7 +847,7 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p
 	ZEN_DEBUG("Reading workspace state from {}", WorkspaceRoot);
 
 	std::filesystem::path ConfigPath = WorkspaceRoot / WorkspaceConfigName;
-	if (std::filesystem::exists(ConfigPath))
+	if (IsFile(ConfigPath))
 	{
 		std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares =
 			WorkspaceSharesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError);
@@ -886,7 +886,7 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work
 	{
 		throw std::invalid_argument(fmt::format("invalid root path '{}' for workspace {}", Configuration.RootPath, Configuration.Id));
 	}
-	if (!std::filesystem::is_directory(Configuration.RootPath))
+	if (!IsDir(Configuration.RootPath))
 	{
 		throw std::invalid_argument(
 			fmt::format("workspace root path '{}' does not exist for workspace '{}'", Configuration.RootPath, Configuration.Id));
@@ -965,7 +965,7 @@ Workspaces::AddWorkspaceShare(const LoggerRef&					 Log,
 		throw std::invalid_argument(
 			fmt::format("workspace share path '{}' is not a sub-path of workspace path '{}'", Configuration.SharePath, WorkspaceRoot));
 	}
-	if (!std::filesystem::is_directory(WorkspaceRoot / Configuration.SharePath))
+	if (!IsDir(WorkspaceRoot / Configuration.SharePath))
 	{
 		throw std::invalid_argument(
 			fmt::format("workspace share path '{}' does not exist in workspace path '{}'", Configuration.SharePath, WorkspaceRoot));
@@ -1244,7 +1244,7 @@ Workspaces::FindWorkspaceShare(const Oid& WorkspaceId, const Oid& ShareId, bool
 	const Workspaces::WorkspaceConfiguration&	   WorkspaceConfig = Workspace->GetConfig();
 	const Workspaces::WorkspaceShareConfiguration& ShareConfig	   = Share->GetConfig();
 	std::filesystem::path						   FullSharePath   = WorkspaceConfig.RootPath / ShareConfig.SharePath;
-	if (std::filesystem::is_directory(FullSharePath))
+	if (IsDir(FullSharePath))
 	{
 		if (ForceRefresh || !Share->IsInitialized())
 		{
@@ -1306,18 +1306,18 @@ namespace {
 		std::filesystem::path EmptyFolder(RootPath / "empty_folder");
 
 		std::filesystem::path FirstFolder(RootPath / "first_folder");
-		std::filesystem::create_directory(FirstFolder);
+		CreateDirectories(FirstFolder);
 		Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22)));
 		Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122)));
 
 		std::filesystem::path SecondFolder(RootPath / "second_folder");
-		std::filesystem::create_directory(SecondFolder);
+		CreateDirectories(SecondFolder);
 		Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522)));
 		Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122)));
 		Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225)));
 
 		std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second");
-		std::filesystem::create_directory(SecondFolderChild);
+		CreateDirectories(SecondFolderChild);
 		Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622)));
 
 		for (const auto& It : Result)
@@ -1365,13 +1365,13 @@ TEST_CASE("workspaces.scanfolder")
 
 	Structure->IterateEntries([&](const Oid& Id, const FolderStructure::FileEntry& Entry) {
 		std::filesystem::path AbsPath = RootPath / Entry.RelativePath;
-		CHECK(std::filesystem::is_regular_file(AbsPath));
-		CHECK(std::filesystem::file_size(AbsPath) == Entry.Size);
+		CHECK(IsFile(AbsPath));
+		CHECK(FileSizeFromPath(AbsPath) == Entry.Size);
 		const FolderStructure::FileEntry* FindEntry = Structure->FindEntry(Id);
 		CHECK(FindEntry);
 		std::filesystem::path Path = RootPath / FindEntry->RelativePath;
 		CHECK(AbsPath == Path);
-		CHECK(std::filesystem::file_size(AbsPath) == FindEntry->Size);
+		CHECK(FileSizeFromPath(AbsPath) == FindEntry->Size);
 	});
 }
 
diff --git a/src/zenstore/zenstore.cpp b/src/zenstore/zenstore.cpp
index c697647d2..654fb3510 100644
--- a/src/zenstore/zenstore.cpp
+++ b/src/zenstore/zenstore.cpp
@@ -5,6 +5,7 @@
 #if ZEN_WITH_TESTS
 
 #	include <zenstore/blockstore.h>
+#	include <zenstore/buildstore/buildstore.h>
 #	include <zenstore/cache/structuredcachestore.h>
 #	include <zenstore/workspaces.h>
 #	include <zenstore/gc.h>
@@ -19,6 +20,7 @@ namespace zen {
 void
 zenstore_forcelinktests()
 {
+	buildstore_forcelink();
 	CAS_forcelink();
 	filecas_forcelink();
 	blockstore_forcelink();
diff --git a/src/zenutil-test/zenutil-test.cpp b/src/zenutil-test/zenutil-test.cpp
index cca88b984..eb4a17918 100644
--- a/src/zenutil-test/zenutil-test.cpp
+++ b/src/zenutil-test/zenutil-test.cpp
@@ -2,6 +2,7 @@
 
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 #include <zenutil/zenutil.h>
 
 #include <zencore/memory/newdelete.h>
@@ -22,6 +23,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 	zen::IgnoreChildSignals();
 #	endif
 
+	zen::TraceInit("zencore-test");
 	zen::logging::InitializeLogging();
 	zen::MaximizeOpenFileCount();
 
diff --git a/src/zenutil/bufferedwritefilecache.cpp b/src/zenutil/bufferedwritefilecache.cpp
new file mode 100644
index 000000000..a52850314
--- /dev/null
+++ b/src/zenutil/bufferedwritefilecache.cpp
@@ -0,0 +1,177 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/bufferedwritefilecache.h>
+
+#include <zencore/logging.h>
+#include <zencore/trace.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+BufferedWriteFileCache::BufferedWriteFileCache() : m_CacheHitCount(0), m_CacheMissCount(0), m_OpenHandleCount(0), m_DroppedHandleCount(0)
+{
+}
+
+BufferedWriteFileCache::~BufferedWriteFileCache()
+{
+	ZEN_TRACE_CPU("~BufferedWriteFileCache()");
+
+	try
+	{
+		for (TOpenHandles& OpenHandles : m_OpenFiles)
+		{
+			while (BasicFile* File = OpenHandles.Pop())
+			{
+				std::unique_ptr<BasicFile> FileToClose(File);
+				m_OpenHandleCount--;
+			}
+		}
+		m_OpenFiles.clear();
+		m_ChunkWriters.clear();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("~BufferedWriteFileCache() threw exeption: {}", Ex.what());
+	}
+}
+
+std::unique_ptr<BasicFile>
+BufferedWriteFileCache::Get(uint32_t FileIndex)
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::Get");
+
+	RwLock::ExclusiveLockScope _(m_WriterLock);
+	if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end())
+	{
+		const uint32_t HandleIndex = It->second;
+		TOpenHandles&  OpenHandles = m_OpenFiles[HandleIndex];
+		if (BasicFile* File = OpenHandles.Pop(); File != nullptr)
+		{
+			m_OpenHandleCount--;
+			m_CacheHitCount++;
+			return std::unique_ptr<BasicFile>(File);
+		}
+	}
+	m_CacheMissCount++;
+	return nullptr;
+}
+
+void
+BufferedWriteFileCache::Put(uint32_t FileIndex, std::unique_ptr<BasicFile>&& Writer)
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::Put");
+
+	if (m_OpenHandleCount.load() >= MaxBufferedCount)
+	{
+		m_DroppedHandleCount++;
+		return;
+	}
+	RwLock::ExclusiveLockScope _(m_WriterLock);
+	if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end())
+	{
+		const uint32_t HandleIndex = It->second;
+		TOpenHandles&  OpenHandles = m_OpenFiles[HandleIndex];
+		if (OpenHandles.Push(Writer.get()))
+		{
+			Writer.release();
+			m_OpenHandleCount++;
+		}
+		else
+		{
+			m_DroppedHandleCount++;
+		}
+	}
+	else
+	{
+		const uint32_t HandleIndex = gsl::narrow<uint32_t>(m_OpenFiles.size());
+		m_OpenFiles.push_back(TOpenHandles{});
+		m_OpenFiles.back().Push(Writer.release());
+		m_ChunkWriters.insert_or_assign(FileIndex, HandleIndex);
+		m_OpenHandleCount++;
+	}
+}
+
+void
+BufferedWriteFileCache::Close(std::span<uint32_t> FileIndexes)
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::Close");
+
+	std::vector<std::unique_ptr<BasicFile>> FilesToClose;
+	FilesToClose.reserve(FileIndexes.size());
+	{
+		RwLock::ExclusiveLockScope _(m_WriterLock);
+		for (uint32_t FileIndex : FileIndexes)
+		{
+			if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end())
+			{
+				const uint32_t HandleIndex = It->second;
+				TOpenHandles&  OpenHandles = m_OpenFiles[HandleIndex];
+				while (BasicFile* File = OpenHandles.Pop())
+				{
+					FilesToClose.emplace_back(std::unique_ptr<BasicFile>(File));
+					m_OpenHandleCount--;
+				}
+				m_ChunkWriters.erase(It);
+			}
+		}
+	}
+	FilesToClose.clear();
+}
+
+BufferedWriteFileCache::Local::Local(BufferedWriteFileCache& Cache) : m_Cache(Cache)
+{
+}
+
+BufferedWriteFileCache::Local::Writer*
+BufferedWriteFileCache::Local::GetWriter(uint32_t FileIndex)
+{
+	if (auto It = m_FileIndexToWriterIndex.find(FileIndex); It != m_FileIndexToWriterIndex.end())
+	{
+		return m_ChunkWriters[It->second].get();
+	}
+	std::unique_ptr<BasicFile> File = m_Cache.Get(FileIndex);
+	if (File)
+	{
+		const uint32_t WriterIndex = gsl::narrow<uint32_t>(m_ChunkWriters.size());
+		m_FileIndexToWriterIndex.insert_or_assign(FileIndex, WriterIndex);
+		m_ChunkWriters.emplace_back(std::make_unique<Writer>(Writer{.File = std::move(File)}));
+		return m_ChunkWriters.back().get();
+	}
+	return nullptr;
+}
+
+BufferedWriteFileCache::Local::Writer*
+BufferedWriteFileCache::Local::PutWriter(uint32_t FileIndex, std::unique_ptr<Writer> Writer)
+{
+	ZEN_ASSERT(!m_FileIndexToWriterIndex.contains(FileIndex));
+	const uint32_t WriterIndex = gsl::narrow<uint32_t>(m_ChunkWriters.size());
+	m_FileIndexToWriterIndex.insert_or_assign(FileIndex, WriterIndex);
+	m_ChunkWriters.emplace_back(std::move(Writer));
+	return m_ChunkWriters.back().get();
+}
+
+BufferedWriteFileCache::Local::~Local()
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::~Local()");
+	try
+	{
+		for (auto& It : m_FileIndexToWriterIndex)
+		{
+			const uint32_t FileIndex   = It.first;
+			const uint32_t WriterIndex = It.second;
+			m_ChunkWriters[WriterIndex]->Writer.reset();
+			std::unique_ptr<BasicFile> File;
+			File.swap(m_ChunkWriters[WriterIndex]->File);
+			m_Cache.Put(FileIndex, std::move(File));
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("BufferedWriteFileCache::~Local() threw exeption: {}", Ex.what());
+	}
+}
+
+}  // namespace zen
diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp
new file mode 100644
index 000000000..88238effd
--- /dev/null
+++ b/src/zenutil/buildstoragecache.cpp
@@ -0,0 +1,407 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/buildstoragecache.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zencore/workthreadpool.h>
+#include <zenhttp/httpclient.h>
+#include <zenhttp/packageformat.h>
+#include <zenutil/workerpools.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+class ZenBuildStorageCache : public BuildStorageCache
+{
+public:
+	explicit ZenBuildStorageCache(HttpClient&					 HttpClient,
+								  BuildStorageCache::Statistics& Stats,
+								  std::string_view				 Namespace,
+								  std::string_view				 Bucket,
+								  const std::filesystem::path&	 TempFolderPath,
+								  bool							 BoostBackgroundThreadCount)
+	: m_HttpClient(HttpClient)
+	, m_Stats(Stats)
+	, m_Namespace(Namespace.empty() ? "none" : Namespace)
+	, m_Bucket(Bucket.empty() ? "none" : Bucket)
+	, m_TempFolderPath(std::filesystem::path(TempFolderPath).make_preferred())
+	, m_BoostBackgroundThreadCount(BoostBackgroundThreadCount)
+	, m_BackgroundWorkPool(m_BoostBackgroundThreadCount ? GetSmallWorkerPool(EWorkloadType::Background)
+														: GetTinyWorkerPool(EWorkloadType::Background))
+	, m_PendingBackgroundWorkCount(1)
+	, m_CancelBackgroundWork(false)
+	{
+	}
+
+	virtual ~ZenBuildStorageCache()
+	{
+		try
+		{
+			m_CancelBackgroundWork.store(true);
+			if (!IsFlushed)
+			{
+				m_PendingBackgroundWorkCount.CountDown();
+				m_PendingBackgroundWorkCount.Wait();
+			}
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_ERROR("~ZenBuildStorageCache() failed with: {}", Ex.what());
+		}
+	}
+
+	void ScheduleBackgroundWork(std::function<void()>&& Work)
+	{
+		m_PendingBackgroundWorkCount.AddCount(1);
+		try
+		{
+			m_BackgroundWorkPool.ScheduleWork([this, Work = std::move(Work)]() {
+				ZEN_TRACE_CPU("ZenBuildStorageCache::BackgroundWork");
+				auto _ = MakeGuard([this]() { m_PendingBackgroundWorkCount.CountDown(); });
+				if (!m_CancelBackgroundWork)
+				{
+					try
+					{
+						Work();
+					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("Failed executing background upload to build cache. Reason: {}", Ex.what());
+					}
+				}
+			});
+		}
+		catch (const std::exception& Ex)
+		{
+			m_PendingBackgroundWorkCount.CountDown();
+			ZEN_ERROR("Failed scheduling background upload to build cache. Reason: {}", Ex.what());
+		}
+	}
+
+	virtual void PutBuildBlob(const Oid&			 BuildId,
+							  const IoHash&			 RawHash,
+							  ZenContentType		 ContentType,
+							  const CompositeBuffer& Payload) override
+	{
+		ZEN_ASSERT(!IsFlushed);
+		ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
+		ScheduleBackgroundWork(
+			[this, BuildId = Oid(BuildId), RawHash = IoHash(RawHash), ContentType, Payload = CompositeBuffer(Payload)]() {
+				ZEN_TRACE_CPU("ZenBuildStorageCache::PutBuildBlob");
+				Stopwatch ExecutionTimer;
+				auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+				HttpClient::Response CacheResponse =
+					m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()),
+										Payload,
+										ContentType);
+				AddStatistic(CacheResponse);
+				if (!CacheResponse.IsSuccess())
+				{
+					ZEN_DEBUG("Failed posting blob to cache: {}", CacheResponse.ErrorMessage(""sv));
+				}
+			});
+	}
+
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
+	{
+		ZEN_TRACE_CPU("ZenBuildStorageCache::GetBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+		HttpClient::KeyValueMap Headers;
+		if (RangeOffset != 0 || RangeBytes != (uint64_t)-1)
+		{
+			Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", RangeOffset, RangeOffset + RangeBytes - 1)});
+		}
+		CreateDirectories(m_TempFolderPath);
+		HttpClient::Response CacheResponse =
+			m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()),
+								  m_TempFolderPath,
+								  Headers);
+		AddStatistic(CacheResponse);
+		if (CacheResponse.IsSuccess())
+		{
+			return CacheResponse.ResponsePayload;
+		}
+		return {};
+	}
+
+	virtual void PutBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes, std::span<const CbObject> MetaDatas) override
+	{
+		ZEN_ASSERT(!IsFlushed);
+		ScheduleBackgroundWork([this,
+								BuildId		  = Oid(BuildId),
+								BlobRawHashes = std::vector<IoHash>(BlobHashes.begin(), BlobHashes.end()),
+								MetaDatas	  = std::vector<CbObject>(MetaDatas.begin(), MetaDatas.end())]() {
+			ZEN_TRACE_CPU("ZenBuildStorageCache::PutBlobMetadatas");
+
+			Stopwatch ExecutionTimer;
+			auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+			const uint64_t BlobCount = BlobRawHashes.size();
+
+			CbPackage							   RequestPackage;
+			std::vector<CbAttachment>			   Attachments;
+			tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes;
+			Attachments.reserve(BlobCount);
+			AttachmentHashes.reserve(BlobCount);
+			{
+				CbObjectWriter RequestWriter;
+				RequestWriter.BeginArray("blobHashes");
+				for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+				{
+					RequestWriter.AddHash(BlobRawHashes[BlockHashIndex]);
+				}
+				RequestWriter.EndArray();  // blobHashes
+
+				RequestWriter.BeginArray("metadatas");
+				for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+				{
+					const IoHash ObjectHash = MetaDatas[BlockHashIndex].GetHash();
+					RequestWriter.AddBinaryAttachment(ObjectHash);
+					if (!AttachmentHashes.contains(ObjectHash))
+					{
+						Attachments.push_back(CbAttachment(MetaDatas[BlockHashIndex], ObjectHash));
+						AttachmentHashes.insert(ObjectHash);
+					}
+				}
+
+				RequestWriter.EndArray();  // metadatas
+
+				RequestPackage.SetObject(RequestWriter.Save());
+			}
+			RequestPackage.AddAttachments(Attachments);
+
+			CompositeBuffer RpcRequestBuffer = FormatPackageMessageBuffer(RequestPackage);
+
+			HttpClient::Response CacheResponse =
+				m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/putBlobMetadata", m_Namespace, m_Bucket, BuildId),
+								  RpcRequestBuffer,
+								  ZenContentType::kCbPackage);
+			AddStatistic(CacheResponse);
+			if (!CacheResponse.IsSuccess())
+			{
+				ZEN_DEBUG("Failed posting blob metadata to cache: {}", CacheResponse.ErrorMessage(""sv));
+			}
+		});
+	}
+
+	virtual std::vector<CbObject> GetBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes) override
+	{
+		ZEN_TRACE_CPU("ZenBuildStorageCache::GetBlobMetadatas");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+		CbObjectWriter Request;
+
+		Request.BeginArray("blobHashes"sv);
+		for (const IoHash& BlobHash : BlobHashes)
+		{
+			Request.AddHash(BlobHash);
+		}
+		Request.EndArray();
+
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+
+		HttpClient::Response Response =
+			m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/getBlobMetadata", m_Namespace, m_Bucket, BuildId),
+							  Payload,
+							  HttpClient::Accept(ZenContentType::kCbObject));
+		AddStatistic(Response);
+		if (Response.IsSuccess())
+		{
+			std::vector<CbObject> Result;
+
+			CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload);
+			CbObject  ResponseObject  = ResponsePackage.GetObject();
+
+			CbArrayView BlobHashArray  = ResponseObject["blobHashes"sv].AsArrayView();
+			CbArrayView MetadatasArray = ResponseObject["metadatas"sv].AsArrayView();
+			Result.reserve(MetadatasArray.Num());
+			auto BlobHashesIt	 = BlobHashes.begin();
+			auto BlobHashArrayIt = begin(BlobHashArray);
+			auto MetadataArrayIt = begin(MetadatasArray);
+			while (MetadataArrayIt != end(MetadatasArray))
+			{
+				const IoHash BlobHash = (*BlobHashArrayIt).AsHash();
+				while (BlobHash != *BlobHashesIt)
+				{
+					ZEN_ASSERT(BlobHashesIt != BlobHashes.end());
+					BlobHashesIt++;
+				}
+
+				ZEN_ASSERT(BlobHash == *BlobHashesIt);
+
+				const IoHash		MetaHash	   = (*MetadataArrayIt).AsAttachment();
+				const CbAttachment* MetaAttachment = ResponsePackage.FindAttachment(MetaHash);
+				ZEN_ASSERT(MetaAttachment);
+
+				CbObject Metadata = MetaAttachment->AsObject();
+				Result.emplace_back(std::move(Metadata));
+
+				BlobHashArrayIt++;
+				MetadataArrayIt++;
+				BlobHashesIt++;
+			}
+			return Result;
+		}
+		return {};
+	}
+
+	virtual std::vector<BlobExistsResult> BlobsExists(const Oid& BuildId, std::span<const IoHash> BlobHashes) override
+	{
+		ZEN_TRACE_CPU("ZenBuildStorageCache::BlobsExists");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+		CbObjectWriter Request;
+
+		Request.BeginArray("blobHashes"sv);
+		for (const IoHash& BlobHash : BlobHashes)
+		{
+			Request.AddHash(BlobHash);
+		}
+		Request.EndArray();
+
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+
+		HttpClient::Response Response = m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/exists", m_Namespace, m_Bucket, BuildId),
+														  Payload,
+														  HttpClient::Accept(ZenContentType::kCbObject));
+		AddStatistic(Response);
+		if (Response.IsSuccess())
+		{
+			CbObject ResponseObject = LoadCompactBinaryObject(Response.ResponsePayload);
+			if (!ResponseObject)
+			{
+				throw std::runtime_error("BlobExists reponse is invalid, failed to load payload as compact binary object");
+			}
+			CbArrayView BlobsExistsArray = ResponseObject["blobExists"sv].AsArrayView();
+			if (!BlobsExistsArray)
+			{
+				throw std::runtime_error("BlobExists reponse is invalid, 'blobExists' array is missing");
+			}
+			if (BlobsExistsArray.Num() != BlobHashes.size())
+			{
+				throw std::runtime_error(fmt::format("BlobExists reponse is invalid, 'blobExists' array contains {} entries, expected {}",
+													 BlobsExistsArray.Num(),
+													 BlobHashes.size()));
+			}
+
+			CbArrayView MetadatasExistsArray = ResponseObject["metadataExists"sv].AsArrayView();
+			if (!MetadatasExistsArray)
+			{
+				throw std::runtime_error("BlobExists reponse is invalid, 'metadataExists' array is missing");
+			}
+			if (MetadatasExistsArray.Num() != BlobHashes.size())
+			{
+				throw std::runtime_error(
+					fmt::format("BlobExists reponse is invalid, 'metadataExists' array contains {} entries, expected {}",
+								MetadatasExistsArray.Num(),
+								BlobHashes.size()));
+			}
+
+			std::vector<BlobExistsResult> Result;
+			Result.reserve(BlobHashes.size());
+			auto BlobExistsIt	  = begin(BlobsExistsArray);
+			auto MetadataExistsIt = begin(MetadatasExistsArray);
+			while (BlobExistsIt != end(BlobsExistsArray))
+			{
+				ZEN_ASSERT(MetadataExistsIt != end(MetadatasExistsArray));
+
+				const bool HasBody	   = (*BlobExistsIt).AsBool();
+				const bool HasMetadata = (*MetadataExistsIt).AsBool();
+
+				Result.push_back({.HasBody = HasBody, .HasMetadata = HasMetadata});
+
+				BlobExistsIt++;
+				MetadataExistsIt++;
+			}
+			return Result;
+		}
+		return {};
+	}
+
+	virtual void Flush(int32_t UpdateIntervalMS, std::function<bool(intptr_t Remaining)>&& UpdateCallback) override
+	{
+		if (IsFlushed)
+		{
+			return;
+		}
+		if (!IsFlushed)
+		{
+			m_PendingBackgroundWorkCount.CountDown();
+			IsFlushed = true;
+		}
+		if (m_PendingBackgroundWorkCount.Wait(100))
+		{
+			return;
+		}
+		while (true)
+		{
+			intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining();
+			if (UpdateCallback(Remaining))
+			{
+				if (m_PendingBackgroundWorkCount.Wait(UpdateIntervalMS))
+				{
+					UpdateCallback(0);
+					return;
+				}
+			}
+			else
+			{
+				m_CancelBackgroundWork.store(true);
+			}
+		}
+	}
+
+private:
+	void AddStatistic(const HttpClient::Response& Result)
+	{
+		m_Stats.TotalBytesWritten += Result.UploadedBytes;
+		m_Stats.TotalBytesRead += Result.DownloadedBytes;
+		m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0);
+		m_Stats.TotalRequestCount++;
+	}
+
+	HttpClient&					   m_HttpClient;
+	BuildStorageCache::Statistics& m_Stats;
+	const std::string			   m_Namespace;
+	const std::string			   m_Bucket;
+	const std::filesystem::path	   m_TempFolderPath;
+	const bool					   m_BoostBackgroundThreadCount;
+	bool						   IsFlushed = false;
+
+	WorkerThreadPool& m_BackgroundWorkPool;
+	Latch			  m_PendingBackgroundWorkCount;
+	std::atomic<bool> m_CancelBackgroundWork;
+};
+
+std::unique_ptr<BuildStorageCache>
+CreateZenBuildStorageCache(HttpClient&					  HttpClient,
+						   BuildStorageCache::Statistics& Stats,
+						   std::string_view				  Namespace,
+						   std::string_view				  Bucket,
+						   const std::filesystem::path&	  TempFolderPath,
+						   bool							  BoostBackgroundThreadCount)
+{
+	return std::make_unique<ZenBuildStorageCache>(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BoostBackgroundThreadCount);
+}
+
+}  // namespace zen
diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp
index 1f951167d..46e80f6b7 100644
--- a/src/zenutil/cache/rpcrecording.cpp
+++ b/src/zenutil/cache/rpcrecording.cpp
@@ -46,7 +46,7 @@ struct RecordedRequestsWriter
 	void BeginWrite(const std::filesystem::path& BasePath)
 	{
 		m_BasePath = BasePath;
-		std::filesystem::create_directories(m_BasePath);
+		CreateDirectories(m_BasePath);
 	}
 
 	void EndWrite()
@@ -366,6 +366,7 @@ private:
 	};
 
 	std::unique_ptr<std::thread> m_WriterThread;
+	std::atomic_bool			 m_IsWriterReady{false};
 	std::atomic_bool			 m_IsActive{false};
 	std::atomic_int64_t			 m_PendingRequests{0};
 	RwLock						 m_RequestQueueLock;
@@ -426,7 +427,7 @@ RecordedRequestsSegmentWriter::BeginWrite(const std::filesystem::path& BasePath,
 	m_BasePath		   = BasePath;
 	m_SegmentIndex	   = SegmentIndex;
 	m_RequestBaseIndex = RequestBaseIndex;
-	std::filesystem::create_directories(m_BasePath);
+	CreateDirectories(m_BasePath);
 }
 
 void
@@ -658,6 +659,8 @@ RecordedRequestsWriter::BeginWrite(const std::filesystem::path& BasePath)
 	m_IsActive = true;
 
 	m_WriterThread.reset(new std::thread(&RecordedRequestsWriter::WriterThreadMain, this));
+
+	m_IsWriterReady.wait(false);
 }
 
 void
@@ -707,6 +710,9 @@ RecordedRequestsWriter::WriterThreadMain()
 	SetCurrentThreadName("rpc_writer");
 	EnsureCurrentSegment();
 
+	m_IsWriterReady.store(true);
+	m_IsWriterReady.notify_all();
+
 	while (m_IsActive)
 	{
 		m_PendingRequests.wait(0);
@@ -1051,14 +1057,14 @@ public:
 
 	static bool IsCompatible(const std::filesystem::path& BasePath)
 	{
-		if (std::filesystem::exists(BasePath / "rpc_recording_info.zcb"))
+		if (IsFile(BasePath / "rpc_recording_info.zcb"))
 		{
 			return true;
 		}
 
 		const std::filesystem::path SegmentZero = BasePath / MakeSegmentPath(0);
 
-		if (std::filesystem::exists(SegmentZero / "rpc_segment_info.zcb") && std::filesystem::exists(SegmentZero / "index.bin"))
+		if (IsFile(SegmentZero / "rpc_segment_info.zcb") && IsFile(SegmentZero / "index.bin"))
 		{
 			// top-level metadata is missing, possibly because of premature exit
 			// on the recording side
diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp
index f3c14edc4..abfc0fb63 100644
--- a/src/zenutil/chunkblock.cpp
+++ b/src/zenutil/chunkblock.cpp
@@ -52,7 +52,7 @@ ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject)
 		return {};
 	}
 	std::vector<ChunkBlockDescription> Result;
-	CbArrayView						   Blocks = BlocksObject["blocks"].AsArrayView();
+	CbArrayView						   Blocks = BlocksObject["blocks"sv].AsArrayView();
 	Result.reserve(Blocks.Num());
 	for (CbFieldView BlockView : Blocks)
 	{
diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
index bb1ee5183..cd1bf7dd7 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenutil/chunkedcontent.cpp
@@ -11,7 +11,7 @@
 
 #include <zenutil/chunkedfile.h>
 #include <zenutil/chunkingcontroller.h>
-#include <zenutil/parallellwork.h>
+#include <zenutil/parallelwork.h>
 #include <zenutil/workerpools.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -140,8 +140,12 @@ namespace {
 			{
 				ZEN_TRACE_CPU("HashOnly");
 
-				IoBuffer	 Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred());
-				const IoHash Hash	= IoHash::HashBuffer(Buffer, &Stats.BytesHashed);
+				IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred());
+				if (Buffer.GetSize() != RawSize)
+				{
+					throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path));
+				}
+				const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed);
 
 				Lock.WithExclusiveLock([&]() {
 					if (!RawHashToSequenceRawHashIndex.contains(Hash))
@@ -301,17 +305,25 @@ FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutP
 }
 
 FolderContent
-GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPathIndexes)
+GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPaths)
 {
 	ZEN_TRACE_CPU("FolderContent::GetUpdatedContent");
-	FolderContent						  Result = {.Platform = Old.Platform};
+
+	const uint32_t NewPathCount = gsl::narrow<uint32_t>(New.Paths.size());
+
+	FolderContent Result = {.Platform = Old.Platform};
+	Result.Paths.reserve(NewPathCount);
+	Result.RawSizes.reserve(NewPathCount);
+	Result.Attributes.reserve(NewPathCount);
+	Result.ModificationTicks.reserve(NewPathCount);
+
 	tsl::robin_map<std::string, uint32_t> NewPathToIndex;
-	const uint32_t						  NewPathCount = gsl::narrow<uint32_t>(New.Paths.size());
 	NewPathToIndex.reserve(NewPathCount);
 	for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++)
 	{
 		NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex});
 	}
+
 	uint32_t OldPathCount = gsl::narrow<uint32_t>(Old.Paths.size());
 	for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++)
 	{
@@ -330,7 +342,7 @@ GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vecto
 		}
 		else
 		{
-			OutDeletedPathIndexes.push_back(Old.Paths[OldPathIndex]);
+			OutDeletedPaths.push_back(Old.Paths[OldPathIndex]);
 		}
 	}
 	return Result;
@@ -366,7 +378,7 @@ GetFolderContent(GetFolderContentStatistics&															  Stats,
 				 std::function<bool(const std::string_view& RelativePath)>&&							  AcceptDirectory,
 				 std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile,
 				 WorkerThreadPool&																		  WorkerPool,
-				 int32_t																				  UpdateInteralMS,
+				 int32_t																				  UpdateIntervalMS,
 				 std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						  UpdateCallback,
 				 std::atomic<bool>&																		  AbortFlag)
 {
@@ -455,7 +467,7 @@ GetFolderContent(GetFolderContentStatistics&															  Stats,
 						WorkerPool,
 						PendingWork);
 	PendingWork.CountDown();
-	while (!PendingWork.Wait(UpdateInteralMS))
+	while (!PendingWork.Wait(UpdateIntervalMS))
 	{
 		UpdateCallback(AbortFlag.load(), PendingWork.Remaining());
 	}
@@ -650,7 +662,9 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const Chu
 }
 
 ChunkedFolderContent
-DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span<const std::filesystem::path> DeletedPaths)
+DeletePathsFromChunkedContent(const ChunkedFolderContent&			 BaseContent,
+							  const ChunkedContentLookup&			 BaseContentLookup,
+							  std::span<const std::filesystem::path> DeletedPaths)
 {
 	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
 
@@ -664,8 +678,18 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span
 		{
 			DeletedPathSet.insert(PathCompareString(DeletedPath));
 		}
-		const ChunkedContentLookup						 BaseLookup = BuildChunkedContentLookup(BaseContent);
-		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+
+		const size_t		  BaseChunkCount = BaseContent.ChunkedContent.ChunkHashes.size();
+		std::vector<uint32_t> NewChunkIndexes(BaseChunkCount, (uint32_t)-1);
+
+		const size_t ExpectedPathCount = BaseContent.Paths.size() - DeletedPaths.size();
+		Result.Paths.reserve(ExpectedPathCount);
+		Result.RawSizes.reserve(ExpectedPathCount);
+		Result.Attributes.reserve(ExpectedPathCount);
+		Result.RawHashes.reserve(ExpectedPathCount);
+
+		Result.ChunkedContent.ChunkHashes.reserve(BaseChunkCount);
+		Result.ChunkedContent.ChunkRawSizes.reserve(BaseChunkCount);
 
 		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
 		for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++)
@@ -685,20 +709,33 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span
 					{
 						RawHashToSequenceRawHashIndex.insert(
 							{RawHash, gsl::narrow<uint32_t>(Result.ChunkedContent.SequenceRawHashes.size())});
-						const uint32_t			  SequenceRawHashIndex = BaseLookup.RawHashToSequenceIndex.at(RawHash);
-						const uint32_t			  OrderIndexOffset	   = BaseLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex];
-						const uint32_t			  ChunkCount		   = BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
-						ChunkingStatistics		  Stats;
+						const uint32_t SequenceRawHashIndex = BaseContentLookup.RawHashToSequenceIndex.at(RawHash);
+						const uint32_t OrderIndexOffset		= BaseContentLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex];
+						const uint32_t ChunkCount			= BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
+
 						std::span<const uint32_t> OriginalChunkOrder =
 							std::span<const uint32_t>(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
-						AddChunkSequence(Stats,
-										 Result.ChunkedContent,
-										 ChunkHashToChunkIndex,
-										 RawHash,
-										 OriginalChunkOrder,
-										 BaseContent.ChunkedContent.ChunkHashes,
-										 BaseContent.ChunkedContent.ChunkRawSizes);
-						Stats.UniqueSequencesFound++;
+
+						Result.ChunkedContent.ChunkCounts.push_back(gsl::narrow<uint32_t>(OriginalChunkOrder.size()));
+
+						for (uint32_t OldChunkIndex : OriginalChunkOrder)
+						{
+							if (uint32_t FoundChunkIndex = NewChunkIndexes[OldChunkIndex]; FoundChunkIndex != (uint32_t)-1)
+							{
+								Result.ChunkedContent.ChunkOrders.push_back(FoundChunkIndex);
+							}
+							else
+							{
+								const uint32_t NewChunkIndex   = gsl::narrow<uint32_t>(Result.ChunkedContent.ChunkHashes.size());
+								NewChunkIndexes[OldChunkIndex] = NewChunkIndex;
+								const IoHash&  ChunkHash	   = BaseContent.ChunkedContent.ChunkHashes[OldChunkIndex];
+								const uint64_t OldChunkSize	   = BaseContent.ChunkedContent.ChunkRawSizes[OldChunkIndex];
+								Result.ChunkedContent.ChunkHashes.push_back(ChunkHash);
+								Result.ChunkedContent.ChunkRawSizes.push_back(OldChunkSize);
+								Result.ChunkedContent.ChunkOrders.push_back(NewChunkIndex);
+							}
+						}
+						Result.ChunkedContent.SequenceRawHashes.push_back(RawHash);
 					}
 				}
 			}
@@ -708,14 +745,28 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span
 }
 
 ChunkedFolderContent
-ChunkFolderContent(ChunkingStatistics&												 Stats,
-				   WorkerThreadPool&												 WorkerPool,
-				   const std::filesystem::path&										 RootPath,
-				   const FolderContent&												 Content,
-				   const ChunkingController&										 InChunkingController,
-				   int32_t															 UpdateInteralMS,
-				   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
-				   std::atomic<bool>&												 AbortFlag)
+DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span<const std::filesystem::path> DeletedPaths)
+{
+	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
+	ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size());
+	if (DeletedPaths.size() == BaseContent.Paths.size())
+	{
+		return {};
+	}
+	const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent);
+	return DeletePathsFromChunkedContent(BaseContent, BaseLookup, DeletedPaths);
+}
+
+ChunkedFolderContent
+ChunkFolderContent(ChunkingStatistics&																Stats,
+				   WorkerThreadPool&																WorkerPool,
+				   const std::filesystem::path&														RootPath,
+				   const FolderContent&																Content,
+				   const ChunkingController&														InChunkingController,
+				   int32_t																			UpdateIntervalMS,
+				   std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+				   std::atomic<bool>&																AbortFlag,
+				   std::atomic<bool>&																PauseFlag)
 {
 	ZEN_TRACE_CPU("ChunkFolderContent");
 
@@ -754,7 +805,7 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 
 		RwLock Lock;
 
-		ParallellWork Work(AbortFlag);
+		ParallelWork Work(AbortFlag, PauseFlag);
 
 		for (uint32_t PathIndex : Order)
 		{
@@ -762,31 +813,28 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 			{
 				break;
 			}
-			Work.ScheduleWork(
-				WorkerPool,	 // GetSyncWorkerPool()
-				[&, PathIndex](std::atomic<bool>& AbortFlag) {
-					if (!AbortFlag)
-					{
-						IoHash RawHash = HashOneFile(Stats,
-													 InChunkingController,
-													 Result,
-													 ChunkHashToChunkIndex,
-													 RawHashToSequenceRawHashIndex,
-													 Lock,
-													 RootPath,
-													 PathIndex,
-													 AbortFlag);
-						Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; });
-						Stats.FilesProcessed++;
-					}
-				},
-				Work.DefaultErrorFunction());
-		}
-
-		Work.Wait(UpdateInteralMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-			ZEN_UNUSED(IsAborted);
+			Work.ScheduleWork(WorkerPool,  // GetSyncWorkerPool()
+							  [&, PathIndex](std::atomic<bool>& AbortFlag) {
+								  if (!AbortFlag)
+								  {
+									  IoHash RawHash = HashOneFile(Stats,
+																   InChunkingController,
+																   Result,
+																   ChunkHashToChunkIndex,
+																   RawHashToSequenceRawHashIndex,
+																   Lock,
+																   RootPath,
+																   PathIndex,
+																   AbortFlag);
+									  Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; });
+									  Stats.FilesProcessed++;
+								  }
+							  });
+		}
+
+		Work.Wait(UpdateIntervalMS, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
 			ZEN_UNUSED(PendingWork);
-			UpdateCallback(Work.IsAborted(), Work.PendingWork().Remaining());
+			UpdateCallback(IsAborted, IsPaused, Work.PendingWork().Remaining());
 		});
 	}
 	return Result;
@@ -799,8 +847,9 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 
 	struct ChunkLocationReference
 	{
-		uint32_t									ChunkIndex = (uint32_t)-1;
-		ChunkedContentLookup::ChunkSequenceLocation Location;
+		uint32_t ChunkIndex	   = (uint32_t)-1;
+		uint32_t SequenceIndex = (uint32_t)-1;
+		uint64_t Offset		   = (uint64_t)-1;
 	};
 
 	ChunkedContentLookup Result;
@@ -829,8 +878,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 		{
 			uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex];
 
-			Locations.push_back(
-				ChunkLocationReference{ChunkIndex, ChunkedContentLookup::ChunkSequenceLocation{SequenceIndex, LocationOffset}});
+			Locations.push_back(ChunkLocationReference{.ChunkIndex = ChunkIndex, .SequenceIndex = SequenceIndex, .Offset = LocationOffset});
 
 			LocationOffset += Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
 		}
@@ -845,15 +893,15 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 		{
 			return false;
 		}
-		if (Lhs.Location.SequenceIndex < Rhs.Location.SequenceIndex)
+		if (Lhs.SequenceIndex < Rhs.SequenceIndex)
 		{
 			return true;
 		}
-		if (Lhs.Location.SequenceIndex > Rhs.Location.SequenceIndex)
+		if (Lhs.SequenceIndex > Rhs.SequenceIndex)
 		{
 			return false;
 		}
-		return Lhs.Location.Offset < Rhs.Location.Offset;
+		return Lhs.Offset < Rhs.Offset;
 	});
 
 	Result.ChunkSequenceLocations.reserve(Locations.size());
@@ -866,7 +914,10 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 		uint32_t Count = 0;
 		while ((RangeOffset + Count < Locations.size()) && (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex))
 		{
-			Result.ChunkSequenceLocations.push_back(Locations[RangeOffset + Count].Location);
+			const ChunkLocationReference& LocationReference = Locations[RangeOffset + Count];
+			Result.ChunkSequenceLocations.push_back(
+				ChunkedContentLookup::ChunkSequenceLocation{.SequenceIndex = LocationReference.SequenceIndex,
+															.Offset		   = LocationReference.Offset});
 			Count++;
 		}
 		Result.ChunkSequenceLocationOffset.push_back(RangeOffset);
@@ -875,8 +926,12 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 	}
 
 	Result.SequenceIndexFirstPathIndex.resize(Content.ChunkedContent.SequenceRawHashes.size(), (uint32_t)-1);
+	Result.PathExtensionHash.resize(Content.Paths.size());
 	for (uint32_t PathIndex = 0; PathIndex < Content.Paths.size(); PathIndex++)
 	{
+		std::string LowercaseExtension = Content.Paths[PathIndex].extension().string();
+		std::transform(LowercaseExtension.begin(), LowercaseExtension.end(), LowercaseExtension.begin(), ::tolower);
+		Result.PathExtensionHash[PathIndex] = HashStringDjb2(LowercaseExtension);
 		if (Content.RawSizes[PathIndex] > 0)
 		{
 			const IoHash& RawHash		  = Content.RawHashes[PathIndex];
diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp
index 2a7057a46..6fb4182c0 100644
--- a/src/zenutil/chunkingcontroller.cpp
+++ b/src/zenutil/chunkingcontroller.cpp
@@ -4,6 +4,7 @@
 
 #include <zencore/basicfile.h>
 #include <zencore/compactbinarybuilder.h>
+#include <zencore/filesystem.h>
 #include <zencore/trace.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -35,26 +36,54 @@ namespace {
 		return ChunkedParams{.UseThreshold = UseThreshold, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize};
 	}
 
-}  // namespace
+	void WriteChunkParams(CbObjectWriter& Writer, const ChunkedParams& Params)
+	{
+		Writer.BeginObject("ChunkingParams"sv);
+		{
+			Writer.AddBool("UseThreshold"sv, Params.UseThreshold);
 
-class BasicChunkingController : public ChunkingController
-{
-public:
-	BasicChunkingController(std::span<const std::string_view> ExcludeExtensions,
-							uint64_t						  ChunkFileSizeLimit,
-							const ChunkedParams&			  ChunkingParams)
-	: m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end())
-	, m_ChunkFileSizeLimit(ChunkFileSizeLimit)
-	, m_ChunkingParams(ChunkingParams)
+			Writer.AddInteger("MinSize"sv, (uint64_t)Params.MinSize);
+			Writer.AddInteger("MaxSize"sv, (uint64_t)Params.MaxSize);
+			Writer.AddInteger("AvgSize"sv, (uint64_t)Params.AvgSize);
+		}
+		Writer.EndObject();	 // ChunkingParams
+	}
+
+	bool IsElfFile(BasicFile& Buffer)
 	{
+		if (Buffer.FileSize() > 4)
+		{
+			uint32_t ElfCheck = 0;
+			Buffer.Read(&ElfCheck, 4, 0);
+			if (ElfCheck == 0x464c457f)
+			{
+				return true;
+			}
+		}
+		return false;
 	}
 
-	BasicChunkingController(CbObjectView Parameters)
-	: m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()))
-	, m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit))
-	, m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()))
+	bool IsMachOFile(BasicFile& Buffer)
 	{
+		if (Buffer.FileSize() > 4)
+		{
+			uint32_t MachOCheck = 0;
+			Buffer.Read(&MachOCheck, 4, 0);
+			if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe))
+			{
+				return true;
+			}
+		}
+		return false;
 	}
+}  // namespace
+
+class BasicChunkingController : public ChunkingController
+{
+public:
+	BasicChunkingController(const BasicChunkingControllerSettings& Settings) : m_Settings(Settings) {}
+
+	BasicChunkingController(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {}
 
 	virtual bool ProcessFile(const std::filesystem::path& InputPath,
 							 uint64_t					  RawSize,
@@ -64,16 +93,25 @@ public:
 	{
 		ZEN_TRACE_CPU("BasicChunkingController::ProcessFile");
 		const bool ExcludeFromChunking =
-			std::find(m_ChunkExcludeExtensions.begin(), m_ChunkExcludeExtensions.end(), InputPath.extension()) !=
-			m_ChunkExcludeExtensions.end();
+			std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) !=
+			m_Settings.ExcludeExtensions.end();
 
-		if (ExcludeFromChunking || (RawSize < m_ChunkFileSizeLimit))
+		if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit))
 		{
 			return false;
 		}
 
 		BasicFile Buffer(InputPath, BasicFile::Mode::kRead);
-		OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag);
+		if (m_Settings.ExcludeElfFiles && IsElfFile(Buffer))
+		{
+			return false;
+		}
+		if (m_Settings.ExcludeMachOFiles && IsMachOFile(Buffer))
+		{
+			return false;
+		}
+
+		OutChunked = ChunkData(Buffer, 0, RawSize, m_Settings.ChunkingParams, &BytesProcessed, &AbortFlag);
 		return true;
 	}
 
@@ -84,53 +122,43 @@ public:
 		CbObjectWriter Writer;
 		Writer.BeginArray("ChunkExcludeExtensions"sv);
 		{
-			for (const std::string& Extension : m_ChunkExcludeExtensions)
+			for (const std::string& Extension : m_Settings.ExcludeExtensions)
 			{
 				Writer.AddString(Extension);
 			}
 		}
 		Writer.EndArray();	// ChunkExcludeExtensions
-		Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit);
-		Writer.BeginObject("ChunkingParams"sv);
-		{
-			Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold);
 
-			Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize);
-			Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize);
-			Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize);
-		}
-		Writer.EndObject();	 // ChunkingParams
+		Writer.AddBool("ExcludeElfFiles"sv, m_Settings.ExcludeElfFiles);
+		Writer.AddBool("ExcludeMachOFiles"sv, m_Settings.ExcludeMachOFiles);
+		Writer.AddInteger("ChunkFileSizeLimit"sv, m_Settings.ChunkFileSizeLimit);
+
+		WriteChunkParams(Writer, m_Settings.ChunkingParams);
+
 		return Writer.Save();
 	}
 	static constexpr std::string_view Name = "BasicChunkingController"sv;
 
-protected:
-	const std::vector<std::string> m_ChunkExcludeExtensions;
-	const uint64_t				   m_ChunkFileSizeLimit;
-	const ChunkedParams			   m_ChunkingParams;
+private:
+	static BasicChunkingControllerSettings ReadSettings(CbObjectView Parameters)
+	{
+		return BasicChunkingControllerSettings{
+			.ExcludeExtensions	= ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()),
+			.ExcludeElfFiles	= Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles),
+			.ExcludeMachOFiles	= Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles),
+			.ChunkFileSizeLimit = Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit),
+			.ChunkingParams		= ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())};
+	}
+
+	const BasicChunkingControllerSettings m_Settings;
 };
 
 class ChunkingControllerWithFixedChunking : public ChunkingController
 {
 public:
-	ChunkingControllerWithFixedChunking(std::span<const std::string_view> FixedChunkingExtensions,
-										uint64_t						  ChunkFileSizeLimit,
-										const ChunkedParams&			  ChunkingParams,
-										uint32_t						  FixedChunkingChunkSize)
-	: m_FixedChunkingExtensions(FixedChunkingExtensions.begin(), FixedChunkingExtensions.end())
-	, m_ChunkFileSizeLimit(ChunkFileSizeLimit)
-	, m_ChunkingParams(ChunkingParams)
-	, m_FixedChunkingChunkSize(FixedChunkingChunkSize)
-	{
-	}
+	ChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Settings) : m_Settings(Settings) {}
 
-	ChunkingControllerWithFixedChunking(CbObjectView Parameters)
-	: m_FixedChunkingExtensions(ReadStringArray(Parameters["FixedChunkingExtensions"sv].AsArrayView()))
-	, m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit))
-	, m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()))
-	, m_FixedChunkingChunkSize(Parameters["FixedChunkingChunkSize"sv].AsUInt32(16u * 1024u * 1024u))
-	{
-	}
+	ChunkingControllerWithFixedChunking(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {}
 
 	virtual bool ProcessFile(const std::filesystem::path& InputPath,
 							 uint64_t					  RawSize,
@@ -139,33 +167,71 @@ public:
 							 std::atomic<bool>&			  AbortFlag) const override
 	{
 		ZEN_TRACE_CPU("ChunkingControllerWithFixedChunking::ProcessFile");
-		if (RawSize < m_ChunkFileSizeLimit)
+		const bool ExcludeFromChunking =
+			std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) !=
+			m_Settings.ExcludeExtensions.end();
+
+		if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit))
 		{
 			return false;
 		}
-		const bool FixedChunking = std::find(m_FixedChunkingExtensions.begin(), m_FixedChunkingExtensions.end(), InputPath.extension()) !=
-								   m_FixedChunkingExtensions.end();
 
-		if (FixedChunking)
+		const bool FixedChunkingExtension =
+			std::find(m_Settings.FixedChunkingExtensions.begin(), m_Settings.FixedChunkingExtensions.end(), InputPath.extension()) !=
+			m_Settings.FixedChunkingExtensions.end();
+
+		if (FixedChunkingExtension)
 		{
+			if (RawSize < m_Settings.MinSizeForFixedChunking)
+			{
+				return false;
+			}
 			ZEN_TRACE_CPU("FixedChunking");
-			IoHashStream									 FullHash;
-			IoBuffer										 Source = IoBufferBuilder::MakeFromFile(InputPath);
+			IoHashStream									 FullHasher;
+			BasicFile										 Source(InputPath, BasicFile::Mode::kRead);
 			uint64_t										 Offset = 0;
 			tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
-			ChunkHashToChunkIndex.reserve(1 + (RawSize / m_FixedChunkingChunkSize));
+			const uint64_t									 ExpectedChunkCount = 1 + (RawSize / m_Settings.FixedChunkingChunkSize);
+			ChunkHashToChunkIndex.reserve(ExpectedChunkCount);
+			OutChunked.Info.ChunkHashes.reserve(ExpectedChunkCount);
+			OutChunked.Info.ChunkSequence.reserve(ExpectedChunkCount);
+			OutChunked.ChunkSources.reserve(ExpectedChunkCount);
+
+			static const uint64_t BufferingSize = 256u * 1024u;
+
+			IoHashStream ChunkHasher;
+
 			while (Offset < RawSize)
 			{
 				if (AbortFlag)
 				{
 					return false;
 				}
-				uint64_t   ChunkSize = std::min<uint64_t>(RawSize - Offset, m_FixedChunkingChunkSize);
-				IoBuffer   Chunk(Source, Offset, ChunkSize);
-				MemoryView ChunkData = Chunk.GetView();
-				FullHash.Append(ChunkData);
 
-				IoHash ChunkHash = IoHash::HashBuffer(ChunkData);
+				ChunkHasher.Reset();
+
+				uint64_t ChunkSize = std::min<uint64_t>(RawSize - Offset, m_Settings.FixedChunkingChunkSize);
+				if (ChunkSize >= (BufferingSize + BufferingSize / 2))
+				{
+					ScanFile(Source.Handle(),
+							 Offset,
+							 ChunkSize,
+							 BufferingSize,
+							 [&FullHasher, &ChunkHasher, &BytesProcessed](const void* Data, size_t Size) {
+								 FullHasher.Append(Data, Size);
+								 ChunkHasher.Append(Data, Size);
+								 BytesProcessed.fetch_add(Size);
+							 });
+				}
+				else
+				{
+					IoBuffer ChunkData = Source.ReadRange(Offset, ChunkSize);
+					FullHasher.Append(ChunkData);
+					ChunkHasher.Append(ChunkData);
+					BytesProcessed.fetch_add(ChunkSize);
+				}
+
+				const IoHash ChunkHash = ChunkHasher.GetHash();
 				if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end())
 				{
 					OutChunked.Info.ChunkSequence.push_back(It->second);
@@ -178,16 +244,24 @@ public:
 					OutChunked.ChunkSources.push_back({.Offset = Offset, .Size = gsl::narrow<uint32_t>(ChunkSize)});
 				}
 				Offset += ChunkSize;
-				BytesProcessed.fetch_add(ChunkSize);
 			}
 			OutChunked.Info.RawSize = RawSize;
-			OutChunked.Info.RawHash = FullHash.GetHash();
+			OutChunked.Info.RawHash = FullHasher.GetHash();
 			return true;
 		}
 		else
 		{
 			BasicFile Buffer(InputPath, BasicFile::Mode::kRead);
-			OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed);
+			if (m_Settings.ExcludeElfFiles && IsElfFile(Buffer))
+			{
+				return false;
+			}
+			if (m_Settings.ExcludeMachOFiles && IsMachOFile(Buffer))
+			{
+				return false;
+			}
+
+			OutChunked = ChunkData(Buffer, 0, RawSize, m_Settings.ChunkingParams, &BytesProcessed, &AbortFlag);
 			return true;
 		}
 	}
@@ -199,41 +273,57 @@ public:
 		CbObjectWriter Writer;
 		Writer.BeginArray("FixedChunkingExtensions");
 		{
-			for (const std::string& Extension : m_FixedChunkingExtensions)
+			for (const std::string& Extension : m_Settings.FixedChunkingExtensions)
 			{
 				Writer.AddString(Extension);
 			}
 		}
 		Writer.EndArray();	// ChunkExcludeExtensions
-		Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit);
-		Writer.BeginObject("ChunkingParams"sv);
-		{
-			Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold);
 
-			Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize);
-			Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize);
-			Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize);
+		Writer.BeginArray("ChunkExcludeExtensions"sv);
+		{
+			for (const std::string& Extension : m_Settings.ExcludeExtensions)
+			{
+				Writer.AddString(Extension);
+			}
 		}
-		Writer.EndObject();	 // ChunkingParams
-		Writer.AddInteger("FixedChunkingChunkSize"sv, m_FixedChunkingChunkSize);
+		Writer.EndArray();	// ChunkExcludeExtensions
+
+		Writer.AddBool("ExcludeElfFiles"sv, m_Settings.ExcludeElfFiles);
+		Writer.AddBool("ExcludeMachOFiles"sv, m_Settings.ExcludeMachOFiles);
+
+		Writer.AddInteger("ChunkFileSizeLimit"sv, m_Settings.ChunkFileSizeLimit);
+
+		WriteChunkParams(Writer, m_Settings.ChunkingParams);
+
+		Writer.AddInteger("FixedChunkingChunkSize"sv, m_Settings.FixedChunkingChunkSize);
+		Writer.AddInteger("MinSizeForFixedChunking"sv, m_Settings.MinSizeForFixedChunking);
 		return Writer.Save();
 	}
 
 	static constexpr std::string_view Name = "ChunkingControllerWithFixedChunking"sv;
 
-protected:
-	const std::vector<std::string> m_FixedChunkingExtensions;
-	const uint64_t				   m_ChunkFileSizeLimit;
-	const ChunkedParams			   m_ChunkingParams;
-	const uint32_t				   m_FixedChunkingChunkSize;
+private:
+	static ChunkingControllerWithFixedChunkingSettings ReadSettings(CbObjectView Parameters)
+	{
+		return ChunkingControllerWithFixedChunkingSettings{
+			.FixedChunkingExtensions = ReadStringArray(Parameters["FixedChunkingExtensions"sv].AsArrayView()),
+			.ExcludeExtensions		 = ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()),
+			.ExcludeElfFiles		 = Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles),
+			.ExcludeMachOFiles		 = Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles),
+			.ChunkFileSizeLimit		 = Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit),
+			.ChunkingParams			 = ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()),
+			.FixedChunkingChunkSize	 = Parameters["FixedChunkingChunkSize"sv].AsUInt64(DefaultFixedChunkingChunkSize),
+			.MinSizeForFixedChunking = Parameters["MinSizeForFixedChunking"sv].AsUInt64(DefaultFixedChunkingChunkSize)};
+	}
+
+	const ChunkingControllerWithFixedChunkingSettings m_Settings;
 };
 
 std::unique_ptr<ChunkingController>
-CreateBasicChunkingController(std::span<const std::string_view> ExcludeExtensions,
-							  uint64_t							ChunkFileSizeLimit,
-							  const ChunkedParams&				ChunkingParams)
+CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings)
 {
-	return std::make_unique<BasicChunkingController>(ExcludeExtensions, ChunkFileSizeLimit, ChunkingParams);
+	return std::make_unique<BasicChunkingController>(Settings);
 }
 std::unique_ptr<ChunkingController>
 CreateBasicChunkingController(CbObjectView Parameters)
@@ -242,15 +332,9 @@ CreateBasicChunkingController(CbObjectView Parameters)
 }
 
 std::unique_ptr<ChunkingController>
-CreateChunkingControllerWithFixedChunking(std::span<const std::string_view> FixedChunkingExtensions,
-										  uint64_t							ChunkFileSizeLimit,
-										  const ChunkedParams&				ChunkingParams,
-										  uint32_t							FixedChunkingChunkSize)
+CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting)
 {
-	return std::make_unique<ChunkingControllerWithFixedChunking>(FixedChunkingExtensions,
-																 ChunkFileSizeLimit,
-																 ChunkingParams,
-																 FixedChunkingChunkSize);
+	return std::make_unique<ChunkingControllerWithFixedChunking>(Setting);
 }
 std::unique_ptr<ChunkingController>
 CreateChunkingControllerWithFixedChunking(CbObjectView Parameters)
diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp
new file mode 100644
index 000000000..afef7f6f2
--- /dev/null
+++ b/src/zenutil/commandlineoptions.cpp
@@ -0,0 +1,221 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/commandlineoptions.h>
+
+#include <filesystem>
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#endif	// ZEN_WITH_TESTS
+
+void
+cxxopts::values::parse_value(const std::string& text, std::filesystem::path& value)
+{
+	value = zen::StringToPath(text);
+}
+
+namespace zen {
+
+std::vector<std::string>
+ParseCommandLine(std::string_view CommandLine)
+{
+	auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) {
+		if (Pos == CommandLine.length())
+		{
+			return true;
+		}
+		if (CommandLine[Pos] == ' ')
+		{
+			return true;
+		}
+		return false;
+	};
+
+	bool IsParsingArg = false;
+	bool IsInQuote	  = false;
+
+	std::string::size_type	 Pos	  = 0;
+	std::string::size_type	 ArgStart = 0;
+	std::vector<std::string> Args;
+	while (Pos < CommandLine.length())
+	{
+		if (IsInQuote)
+		{
+			if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1))
+			{
+				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1)));
+				Pos++;
+				IsInQuote	 = false;
+				IsParsingArg = false;
+			}
+			else
+			{
+				Pos++;
+			}
+		}
+		else if (IsParsingArg)
+		{
+			ZEN_ASSERT(Pos > ArgStart);
+			if (CommandLine[Pos] == ' ')
+			{
+				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart)));
+				Pos++;
+				IsParsingArg = false;
+			}
+			else if (CommandLine[Pos] == '"')
+			{
+				IsInQuote = true;
+				Pos++;
+			}
+			else
+			{
+				Pos++;
+			}
+		}
+		else if (CommandLine[Pos] == '"')
+		{
+			IsInQuote	 = true;
+			IsParsingArg = true;
+			ArgStart	 = Pos;
+			Pos++;
+		}
+		else if (CommandLine[Pos] != ' ')
+		{
+			IsParsingArg = true;
+			ArgStart	 = Pos;
+			Pos++;
+		}
+		else
+		{
+			Pos++;
+		}
+	}
+	if (IsParsingArg)
+	{
+		ZEN_ASSERT(Pos > ArgStart);
+		Args.push_back(std::string(CommandLine.substr(ArgStart)));
+	}
+
+	return Args;
+}
+
+std::vector<char*>
+StripCommandlineQuotes(std::vector<std::string>& InOutArgs)
+{
+	std::vector<char*> RawArgs;
+	RawArgs.reserve(InOutArgs.size());
+	for (std::string& Arg : InOutArgs)
+	{
+		std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1);
+		while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos)
+		{
+			Arg.erase(EscapedQuotePos, 1);
+			EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos);
+		}
+
+		if (Arg.starts_with("\""))
+		{
+			if (Arg.find('"', 1) == Arg.length() - 1)
+			{
+				if (Arg.find(' ', 1) == std::string::npos)
+				{
+					Arg = Arg.substr(1, Arg.length() - 2);
+				}
+			}
+		}
+		RawArgs.push_back(const_cast<char*>(Arg.c_str()));
+	}
+	return RawArgs;
+}
+
+void
+MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path)
+{
+	if (!Path.empty())
+	{
+		std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred();
+#if ZEN_PLATFORM_WINDOWS
+		const std::string_view Prefix = "\\\\?\\";
+		const std::u8string	   PrefixU8(Prefix.begin(), Prefix.end());
+		std::u8string		   PathString = AbsolutePath.u8string();
+		if (!PathString.empty() && !PathString.starts_with(PrefixU8))
+		{
+			PathString.insert(0, PrefixU8);
+			Path = PathString;
+		}
+#endif	// ZEN_PLATFORM_WINDOWS
+	}
+}
+
+std::filesystem::path
+MakeSafeAbsolutePath(const std::filesystem::path& Path)
+{
+	std::filesystem::path Tmp(Path);
+	MakeSafeAbsolutePathÍnPlace(Tmp);
+	return Tmp;
+}
+
+std::filesystem::path
+StringToPath(const std::string_view& Path)
+{
+	std::string_view UnquotedPath = RemoveQuotes(Path);
+
+	if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator))
+	{
+		UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1);
+	}
+
+	return std::filesystem::path(UnquotedPath).make_preferred();
+}
+
+std::string_view
+RemoveQuotes(const std::string_view& Arg)
+{
+	if (Arg.length() > 2)
+	{
+		if (Arg[0] == '"' && Arg[Arg.length() - 1] == '"')
+		{
+			return Arg.substr(1, Arg.length() - 2);
+		}
+	}
+	return Arg;
+}
+
+#if ZEN_WITH_TESTS
+
+void
+commandlineoptions_forcelink()
+{
+}
+
+TEST_CASE("CommandLine")
+{
+	std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\"");
+	CHECK_EQ(v1[0], "c:\\my\\exe.exe");
+	CHECK_EQ(v1[1], "\"quoted arg\"");
+	CHECK_EQ(v1[2], "\"one\",two,\"three\\\"");
+
+	std::vector<std::string> v2 = ParseCommandLine(
+		"--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog "
+		"--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" "
+		"--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" "
+		"--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5");
+
+	std::vector<char*> v2Stripped = StripCommandlineQuotes(v2);
+	CHECK_EQ(v2Stripped[0], std::string("--tracehost"));
+	CHECK_EQ(v2Stripped[1], std::string("127.0.0.1"));
+	CHECK_EQ(v2Stripped[2], std::string("builds"));
+	CHECK_EQ(v2Stripped[3], std::string("download"));
+	CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com"));
+	CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog"));
+	CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows"));
+	CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path"));
+	CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\""));
+	CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\""));
+	CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\""));
+	CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab"));
+	CHECK_EQ(v2Stripped[12], std::string("--build-part-name"));
+	CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5"));
+}
+
+#endif
+}  // namespace zen
diff --git a/src/zenutil/environmentoptions.cpp b/src/zenutil/environmentoptions.cpp
new file mode 100644
index 000000000..1b7ce8029
--- /dev/null
+++ b/src/zenutil/environmentoptions.cpp
@@ -0,0 +1,84 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/environmentoptions.h>
+
+#include <zencore/filesystem.h>
+
+namespace zen {
+
+EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value)
+{
+}
+void
+EnvironmentOptions::StringOption::Parse(std::string_view Value)
+{
+	RefValue = std::string(Value);
+}
+
+EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value)
+{
+}
+void
+EnvironmentOptions::FilePathOption::Parse(std::string_view Value)
+{
+	RefValue = MakeSafeAbsolutePath(Value);
+}
+
+EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value)
+{
+}
+void
+EnvironmentOptions::BoolOption::Parse(std::string_view Value)
+{
+	const std::string Lower = ToLower(Value);
+	if (Lower == "true" || Lower == "y" || Lower == "yes")
+	{
+		RefValue = true;
+	}
+	else if (Lower == "false" || Lower == "n" || Lower == "no")
+	{
+		RefValue = false;
+	}
+}
+
+std::shared_ptr<EnvironmentOptions::OptionValue>
+EnvironmentOptions::MakeOption(std::string& Value)
+{
+	return std::make_shared<StringOption>(Value);
+}
+
+std::shared_ptr<EnvironmentOptions::OptionValue>
+EnvironmentOptions::MakeOption(std::filesystem::path& Value)
+{
+	return std::make_shared<FilePathOption>(Value);
+}
+
+std::shared_ptr<EnvironmentOptions::OptionValue>
+EnvironmentOptions::MakeOption(bool& Value)
+{
+	return std::make_shared<BoolOption>(Value);
+}
+
+EnvironmentOptions::EnvironmentOptions()
+{
+}
+
+void
+EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult)
+{
+	for (auto& It : OptionMap)
+	{
+		std::string_view EnvName = It.first;
+		const Option&	 Opt	 = It.second;
+		if (CmdLineResult.count(Opt.CommandLineOptionName) == 0)
+		{
+			std::string EnvValue = GetEnvVariable(It.first);
+			if (!EnvValue.empty())
+			{
+				Opt.Value->Parse(EnvValue);
+			}
+		}
+	}
+}
+
+}  // namespace zen
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index 47a4e1cc4..c2cc5ab3c 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -35,6 +35,27 @@ public:
 
 	virtual ~FileBuildStorage() {}
 
+	virtual CbObject ListNamespaces(bool bRecursive) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::ListNamespaces");
+		ZEN_UNUSED(bRecursive);
+
+		SimulateLatency(0, 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObjectWriter Writer;
+		Writer.BeginArray("results");
+		{
+		}
+		Writer.EndArray();	// results
+		Writer.Save();
+		SimulateLatency(Writer.GetSaveSize(), 0);
+		return Writer.Save();
+	}
+
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::ListBuilds");
@@ -66,7 +87,7 @@ public:
 				}
 			}
 		}
-		Writer.EndArray();	// builds
+		Writer.EndArray();	// results
 		Writer.Save();
 		SimulateLatency(Writer.GetSaveSize(), 0);
 		return Writer.Save();
@@ -235,7 +256,7 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (!std::filesystem::is_regular_file(BlockPath))
+		if (!IsFile(BlockPath))
 		{
 			CreateDirectories(BlockPath.parent_path());
 			TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView());
@@ -260,7 +281,7 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (!std::filesystem::is_regular_file(BlockPath))
+		if (!IsFile(BlockPath))
 		{
 			CreateDirectories(BlockPath.parent_path());
 
@@ -346,7 +367,7 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (std::filesystem::is_regular_file(BlockPath))
+		if (IsFile(BlockPath))
 		{
 			BasicFile File(BlockPath, BasicFile::Mode::kRead);
 			IoBuffer  Payload;
@@ -369,11 +390,11 @@ public:
 		return IoBuffer{};
 	}
 
-	virtual std::vector<std::function<void()>> GetLargeBuildBlob(
-		const Oid&																			   BuildId,
-		const IoHash&																		   RawHash,
-		uint64_t																			   ChunkSize,
-		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid&													   BuildId,
+																 const IoHash&												   RawHash,
+																 uint64_t													   ChunkSize,
+																 std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+																 std::function<void()>&& OnComplete) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob");
 		ZEN_UNUSED(BuildId);
@@ -383,20 +404,22 @@ public:
 		m_Stats.TotalRequestCount++;
 
 		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
-		if (std::filesystem::is_regular_file(BlockPath))
+		if (IsFile(BlockPath))
 		{
 			struct WorkloadData
 			{
-				std::atomic<uint64_t>																 BytesRemaining;
-				BasicFile																			 BlobFile;
-				std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)> Receiver;
+				std::atomic<uint64_t>										BytesRemaining;
+				BasicFile													BlobFile;
+				std::function<void(uint64_t Offset, const IoBuffer& Chunk)> OnReceive;
+				std::function<void()>										OnComplete;
 			};
 
 			std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
 			Workload->BlobFile.Open(BlockPath, BasicFile::Mode::kRead);
 			const uint64_t BlobSize = Workload->BlobFile.FileSize();
 
-			Workload->Receiver		 = std::move(Receiver);
+			Workload->OnReceive		 = std::move(OnReceive);
+			Workload->OnComplete	 = std::move(OnComplete);
 			Workload->BytesRemaining = BlobSize;
 
 			std::vector<std::function<void()>> WorkItems;
@@ -410,8 +433,12 @@ public:
 					IoBuffer PartPayload(Size);
 					Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset);
 					m_Stats.TotalBytesRead += PartPayload.GetSize();
+					Workload->OnReceive(Offset, PartPayload);
 					uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size);
-					Workload->Receiver(Offset, PartPayload, ByteRemaning);
+					if (ByteRemaning == Size)
+					{
+						Workload->OnComplete();
+					}
 					SimulateLatency(Size, PartPayload.GetSize());
 				});
 
@@ -423,7 +450,7 @@ public:
 		return {};
 	}
 
-	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata");
 		ZEN_UNUSED(BuildId);
@@ -440,68 +467,107 @@ public:
 		m_Stats.TotalBytesWritten += MetaData.GetSize();
 		WriteAsJson(BlockMetaDataPath, MetaData);
 		SimulateLatency(0, 0);
+		return true;
 	}
 
-	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
+	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::FindBlocks");
 		ZEN_UNUSED(BuildId);
-		SimulateLatency(0, 0);
+		SimulateLatency(sizeof(BuildId), 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		m_Stats.TotalRequestCount++;
 
+		uint64_t FoundCount = 0;
+
 		DirectoryContent Content;
 		GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content);
-		std::vector<ChunkBlockDescription> Result;
+		CbObjectWriter Writer;
+		Writer.BeginArray("blocks");
 		for (const std::filesystem::path& MetaDataFile : Content.Files)
 		{
 			IoHash ChunkHash;
 			if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash))
 			{
 				std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash);
-				if (std::filesystem::is_regular_file(BlockPath))
+				if (IsFile(BlockPath))
 				{
 					IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
 
 					m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
 
 					CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
-					Result.emplace_back(ParseChunkBlockDescription(BlockObject));
+					Writer.AddObject(BlockObject);
+					FoundCount++;
+					if (FoundCount == MaxBlockCount)
+					{
+						break;
+					}
 				}
 			}
 		}
-		SimulateLatency(0, sizeof(IoHash) * Result.size());
+		Writer.EndArray();	// blocks
+		CbObject Result = Writer.Save();
+		SimulateLatency(0, Result.GetSize());
 		return Result;
 	}
 
-	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata");
 		ZEN_UNUSED(BuildId);
-		SimulateLatency(0, 0);
+		SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		m_Stats.TotalRequestCount++;
 
-		std::vector<ChunkBlockDescription> Result;
+		CbObjectWriter Writer;
+		Writer.BeginArray("blocks");
+
 		for (const IoHash& BlockHash : BlockHashes)
 		{
 			std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash);
-			if (std::filesystem::is_regular_file(MetaDataFile))
+			if (IsFile(MetaDataFile))
 			{
 				IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
 
 				m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
 
 				CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
-				Result.emplace_back(ParseChunkBlockDescription(BlockObject));
+				Writer.AddObject(BlockObject);
 			}
 		}
-		SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size());
+		Writer.EndArray();	// blocks
+		CbObject Result = Writer.Save();
+		SimulateLatency(0, Result.GetSize());
 		return Result;
 	}
 
+	virtual void PutBuildPartStats(const Oid&								  BuildId,
+								   const Oid&								  BuildPartId,
+								   const tsl::robin_map<std::string, double>& FloatStats) override
+	{
+		CbObjectWriter Request;
+		Request.BeginObject("floatStats"sv);
+		for (auto It : FloatStats)
+		{
+			Request.AddFloat(It.first, It.second);
+		}
+		Request.EndObject();
+		CbObject Payload = Request.Save();
+
+		SimulateLatency(Payload.GetSize(), 0);
+
+		const std::filesystem::path BuildPartStatsDataPath = GetBuildPartStatsPath(BuildId, BuildPartId);
+		CreateDirectories(BuildPartStatsDataPath.parent_path());
+
+		TemporaryFile::SafeWriteFile(BuildPartStatsDataPath, Payload.GetView());
+		WriteAsJson(BuildPartStatsDataPath, Payload);
+
+		SimulateLatency(0, 0);
+	}
+
 protected:
 	std::filesystem::path GetBuildsFolder() const { return m_StoragePath / "builds"; }
 	std::filesystem::path GetBlobsFolder() const { return m_StoragePath / "blobs"; }
@@ -520,6 +586,11 @@ protected:
 		return GetBuildPartFolder(BuildId, BuildPartId) / "metadata.cb";
 	}
 
+	std::filesystem::path GetBuildPartStatsPath(const Oid& BuildId, const Oid& BuildPartId) const
+	{
+		return GetBuildPartFolder(BuildId, BuildPartId) / fmt::format("stats_{}.cb", Oid::NewOid());
+	}
+
 	std::filesystem::path GetBlobPayloadPath(const IoHash& RawHash) const { return GetBlobsFolder() / fmt::format("{}.cbz", RawHash); }
 
 	std::filesystem::path GetBlobMetadataPath(const IoHash& RawHash) const
@@ -587,7 +658,7 @@ protected:
 		BuildPartObject.IterateAttachments([&](CbFieldView FieldView) {
 			const IoHash				AttachmentHash = FieldView.AsBinaryAttachment();
 			const std::filesystem::path BlockPath	   = GetBlobPayloadPath(AttachmentHash);
-			if (!std::filesystem::is_regular_file(BlockPath))
+			if (!IsFile(BlockPath))
 			{
 				NeededAttachments.push_back(AttachmentHash);
 			}
@@ -608,13 +679,24 @@ protected:
 		{
 			return false;
 		}
-		CompositeBuffer Decompressed = ValidateBuffer.DecompressToComposite();
-		if (!Decompressed)
+
+		IoHashStream Hash;
+		bool		 CouldDecompress = ValidateBuffer.DecompressToStream(
+			0,
+			(uint64_t)-1,
+			[&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+				ZEN_UNUSED(SourceOffset, SourceSize, Offset);
+				for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+				{
+					Hash.Append(Segment.GetView());
+				}
+				return true;
+			});
+		if (!CouldDecompress)
 		{
 			return false;
 		}
-		IoHash Hash = IoHash::HashBuffer(Decompressed);
-		if (Hash != RawHash)
+		if (Hash.GetHash() != VerifyHash)
 		{
 			return false;
 		}
diff --git a/src/zenutil/include/zenutil/bufferedwritefilecache.h b/src/zenutil/include/zenutil/bufferedwritefilecache.h
new file mode 100644
index 000000000..68d6c375e
--- /dev/null
+++ b/src/zenutil/include/zenutil/bufferedwritefilecache.h
@@ -0,0 +1,106 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/basicfile.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+class CompositeBuffer;
+
+class BufferedWriteFileCache
+{
+public:
+	BufferedWriteFileCache(const BufferedWriteFileCache&) = delete;
+	BufferedWriteFileCache& operator=(const BufferedWriteFileCache&) = delete;
+
+	BufferedWriteFileCache();
+
+	~BufferedWriteFileCache();
+
+	std::unique_ptr<BasicFile> Get(uint32_t FileIndex);
+
+	void Put(uint32_t FileIndex, std::unique_ptr<BasicFile>&& Writer);
+
+	void Close(std::span<uint32_t> FileIndexes);
+
+	class Local
+	{
+	public:
+		struct Writer
+		{
+			std::unique_ptr<BasicFile>		 File;
+			std::unique_ptr<BasicFileWriter> Writer;
+
+			inline void Write(const CompositeBuffer& Chunk, uint64_t FileOffset)
+			{
+				if (Writer)
+				{
+					Writer->Write(Chunk, FileOffset);
+				}
+				else
+				{
+					File->Write(Chunk, FileOffset);
+				}
+			}
+		};
+
+		Local(const Local&) = delete;
+		Local& operator=(const Local&) = delete;
+
+		explicit Local(BufferedWriteFileCache& Cache);
+		~Local();
+
+		Writer* GetWriter(uint32_t FileIndex);
+		Writer* PutWriter(uint32_t FileIndex, std::unique_ptr<Writer> Writer);
+
+	private:
+		tsl::robin_map<uint32_t, uint32_t>	 m_FileIndexToWriterIndex;
+		std::vector<std::unique_ptr<Writer>> m_ChunkWriters;
+		BufferedWriteFileCache&				 m_Cache;
+	};
+
+private:
+	static constexpr size_t MaxHandlesPerPath = 7;
+	static constexpr size_t MaxBufferedCount  = 1024;
+	struct TOpenHandles
+	{
+		BasicFile*		  Files[MaxHandlesPerPath];
+		uint64_t		  Size = 0;
+		inline BasicFile* Pop()
+		{
+			if (Size > 0)
+			{
+				return Files[--Size];
+			}
+			else
+			{
+				return nullptr;
+			}
+		}
+		inline bool Push(BasicFile* File)
+		{
+			if (Size < MaxHandlesPerPath)
+			{
+				Files[Size++] = File;
+				return true;
+			}
+			return false;
+		}
+	};
+	static_assert(sizeof(TOpenHandles) == 64);
+
+	RwLock							   m_WriterLock;
+	tsl::robin_map<uint32_t, uint32_t> m_ChunkWriters;
+	std::vector<TOpenHandles>		   m_OpenFiles;
+	std::atomic<uint32_t>			   m_CacheHitCount;
+	std::atomic<uint32_t>			   m_CacheMissCount;
+	std::atomic<uint32_t>			   m_OpenHandleCount;
+	std::atomic<uint32_t>			   m_DroppedHandleCount;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
index 9d2bab170..f49d4b42a 100644
--- a/src/zenutil/include/zenutil/buildstorage.h
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -5,6 +5,10 @@
 #include <zencore/compactbinary.h>
 #include <zenutil/chunkblock.h>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 namespace zen {
 
 class BuildStorage
@@ -21,6 +25,7 @@ public:
 
 	virtual ~BuildStorage() {}
 
+	virtual CbObject ListNamespaces(bool bRecursive = false)				= 0;
 	virtual CbObject ListBuilds(CbObject Query)								= 0;
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0;
 	virtual CbObject GetBuild(const Oid& BuildId)							= 0;
@@ -43,16 +48,18 @@ public:
 	virtual IoBuffer						   GetBuildBlob(const Oid&	  BuildId,
 															const IoHash& RawHash,
 															uint64_t	  RangeOffset = 0,
-															uint64_t	  RangeBytes  = (uint64_t)-1) = 0;
-	virtual std::vector<std::function<void()>> GetLargeBuildBlob(
-		const Oid&																			   BuildId,
-		const IoHash&																		   RawHash,
-		uint64_t																			   ChunkSize,
-		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) = 0;
-
-	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData)				 = 0;
-	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId)											 = 0;
-	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0;
+															uint64_t	  RangeBytes  = (uint64_t)-1)		 = 0;
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid&													   BuildId,
+																 const IoHash&												   RawHash,
+																 uint64_t													   ChunkSize,
+																 std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+																 std::function<void()>&& OnComplete) = 0;
+
+	[[nodiscard]] virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0;
+	virtual CbObject		   FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount)									  = 0;
+	virtual CbObject		   GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes)				  = 0;
+
+	virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map<std::string, double>& FloatStats) = 0;
 };
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h
new file mode 100644
index 000000000..e1fb73fd4
--- /dev/null
+++ b/src/zenutil/include/zenutil/buildstoragecache.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/compositebuffer.h>
+#include <zenutil/chunkblock.h>
+
+namespace zen {
+
+class HttpClient;
+
+class BuildStorageCache
+{
+public:
+	struct Statistics
+	{
+		std::atomic<uint64_t> TotalBytesRead	   = 0;
+		std::atomic<uint64_t> TotalBytesWritten	   = 0;
+		std::atomic<uint64_t> TotalRequestCount	   = 0;
+		std::atomic<uint64_t> TotalRequestTimeUs   = 0;
+		std::atomic<uint64_t> TotalExecutionTimeUs = 0;
+	};
+
+	virtual ~BuildStorageCache() {}
+
+	virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0;
+	virtual IoBuffer GetBuildBlob(const Oid&	BuildId,
+								  const IoHash& RawHash,
+								  uint64_t		RangeOffset = 0,
+								  uint64_t		RangeBytes	= (uint64_t)-1)																 = 0;
+
+	virtual void PutBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes, std::span<const CbObject> MetaDatas) = 0;
+	virtual std::vector<CbObject> GetBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes)					   = 0;
+
+	struct BlobExistsResult
+	{
+		bool HasBody	 = 0;
+		bool HasMetadata = 0;
+	};
+
+	virtual std::vector<BlobExistsResult> BlobsExists(const Oid& BuildId, std::span<const IoHash> BlobHashes) = 0;
+
+	virtual void Flush(
+		int32_t									  UpdateIntervalMS,
+		std::function<bool(intptr_t Remaining)>&& UpdateCallback = [](intptr_t) { return true; }) = 0;
+};
+
+std::unique_ptr<BuildStorageCache> CreateZenBuildStorageCache(HttpClient&					 HttpClient,
+															  BuildStorageCache::Statistics& Stats,
+															  std::string_view				 Namespace,
+															  std::string_view				 Bucket,
+															  const std::filesystem::path&	 TempFolderPath,
+															  bool							 BoostBackgroundThreadCount);
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
index 57b55cb8e..306a5d990 100644
--- a/src/zenutil/include/zenutil/chunkedcontent.h
+++ b/src/zenutil/include/zenutil/chunkedcontent.h
@@ -67,7 +67,7 @@ FolderContent GetFolderContent(GetFolderContentStatistics&																Stats,
 							   std::function<bool(const std::string_view& RelativePath)>&&								AcceptDirectory,
 							   std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile,
 							   WorkerThreadPool&																		WorkerPool,
-							   int32_t																					UpdateInteralMS,
+							   int32_t																					UpdateIntervalMS,
 							   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						UpdateCallback,
 							   std::atomic<bool>&																		AbortFlag);
 
@@ -94,10 +94,31 @@ struct ChunkedFolderContent
 	ChunkedContentData				   ChunkedContent;
 };
 
+struct ChunkedContentLookup
+{
+	struct ChunkSequenceLocation
+	{
+		uint32_t SequenceIndex = (uint32_t)-1;
+		uint64_t Offset		   = (uint64_t)-1;
+	};
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex;
+	std::vector<uint32_t>							 SequenceIndexChunkOrderOffset;
+	std::vector<ChunkSequenceLocation>				 ChunkSequenceLocations;
+	std::vector<size_t>
+		ChunkSequenceLocationOffset;  // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex
+	std::vector<uint32_t> ChunkSequenceLocationCounts;	// ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex
+	std::vector<uint32_t> SequenceIndexFirstPathIndex;	// SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash
+	std::vector<uint32_t> PathExtensionHash;
+};
+
 void				 SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output);
 ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input);
 
 ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays);
+ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent&			  Base,
+												   const ChunkedContentLookup&			  BaseContentLookup,
+												   std::span<const std::filesystem::path> DeletedPaths);
 ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span<const std::filesystem::path> DeletedPaths);
 
 struct ChunkingStatistics
@@ -111,31 +132,15 @@ struct ChunkingStatistics
 	uint64_t			  ElapsedWallTimeUS	   = 0;
 };
 
-ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&												  Stats,
-										WorkerThreadPool&												  WorkerPool,
-										const std::filesystem::path&									  RootPath,
-										const FolderContent&											  Content,
-										const ChunkingController&										  InChunkingController,
-										int32_t															  UpdateInteralMS,
-										std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
-										std::atomic<bool>&												  AbortFlag);
-
-struct ChunkedContentLookup
-{
-	struct ChunkSequenceLocation
-	{
-		uint32_t SequenceIndex = (uint32_t)-1;
-		uint64_t Offset		   = (uint64_t)-1;
-	};
-	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
-	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex;
-	std::vector<uint32_t>							 SequenceIndexChunkOrderOffset;
-	std::vector<ChunkSequenceLocation>				 ChunkSequenceLocations;
-	std::vector<size_t>
-		ChunkSequenceLocationOffset;  // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex
-	std::vector<uint32_t> ChunkSequenceLocationCounts;	// ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex
-	std::vector<uint32_t> SequenceIndexFirstPathIndex;	// SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash
-};
+ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&			 Stats,
+										WorkerThreadPool&			 WorkerPool,
+										const std::filesystem::path& RootPath,
+										const FolderContent&		 Content,
+										const ChunkingController&	 InChunkingController,
+										int32_t						 UpdateIntervalMS,
+										std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+										std::atomic<bool>&																 AbortFlag,
+										std::atomic<bool>&																 PauseFlag);
 
 ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content);
 
diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h
index 246f4498a..315502265 100644
--- a/src/zenutil/include/zenutil/chunkingcontroller.h
+++ b/src/zenutil/include/zenutil/chunkingcontroller.h
@@ -11,7 +11,11 @@
 
 namespace zen {
 
-const std::vector<std::string_view> DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"};
+const std::vector<std::string> DefaultChunkingExcludeExtensions =
+	{".exe", ".dll", ".pdb", ".self", ".mp4", ".zip", ".7z", ".bzip", ".rar", ".gzip"};
+const std::vector<std::string> DefaultFixedChunkingExtensions	= {".apk", ".nsp", ".xvc", ".pkg", ".dmg", ".ipa"};
+const bool					   DefaultChunkingExcludeElfFiles	= true;
+const bool					   DefaultChunkingExcludeMachOFiles = true;
 
 const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u,
 											.MaxSize = 128u * 1024u,
@@ -19,7 +23,8 @@ const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128
 
 const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize;
 
-const uint32_t DefaultFixedChunkingChunkSize = 16u * 1024u * 1024u;
+const uint64_t DefaultFixedChunkingChunkSize  = 32u * 1024u * 1024u;
+const uint64_t DefaultMinSizeForFixedChunking = DefaultFixedChunkingChunkSize * 8u;
 
 struct ChunkedInfoWithSource;
 
@@ -38,17 +43,31 @@ public:
 	virtual CbObject		 GetParameters() const							 = 0;
 };
 
-std::unique_ptr<ChunkingController> CreateBasicChunkingController(
-	std::span<const std::string_view> ExcludeExtensions	 = DefaultChunkingExcludeExtensions,
-	uint64_t						  ChunkFileSizeLimit = DefaultChunkingFileSizeLimit,
-	const ChunkedParams&			  ChunkingParams	 = DefaultChunkedParams);
+struct BasicChunkingControllerSettings
+{
+	std::vector<std::string> ExcludeExtensions	= DefaultChunkingExcludeExtensions;
+	bool					 ExcludeElfFiles	= DefaultChunkingExcludeElfFiles;
+	bool					 ExcludeMachOFiles	= DefaultChunkingExcludeMachOFiles;
+	uint64_t				 ChunkFileSizeLimit = DefaultChunkingFileSizeLimit;
+	ChunkedParams			 ChunkingParams		= DefaultChunkedParams;
+};
+
+std::unique_ptr<ChunkingController> CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings);
 std::unique_ptr<ChunkingController> CreateBasicChunkingController(CbObjectView Parameters);
 
-std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(
-	std::span<const std::string_view> ExcludeExtensions		 = DefaultChunkingExcludeExtensions,
-	uint64_t						  ChunkFileSizeLimit	 = DefaultChunkingFileSizeLimit,
-	const ChunkedParams&			  ChunkingParams		 = DefaultChunkedParams,
-	uint32_t						  FixedChunkingChunkSize = DefaultFixedChunkingChunkSize);
+struct ChunkingControllerWithFixedChunkingSettings
+{
+	std::vector<std::string> FixedChunkingExtensions = DefaultFixedChunkingExtensions;
+	std::vector<std::string> ExcludeExtensions		 = DefaultChunkingExcludeExtensions;
+	bool					 ExcludeElfFiles		 = DefaultChunkingExcludeElfFiles;
+	bool					 ExcludeMachOFiles		 = DefaultChunkingExcludeMachOFiles;
+	uint64_t				 ChunkFileSizeLimit		 = DefaultChunkingFileSizeLimit;
+	ChunkedParams			 ChunkingParams			 = DefaultChunkedParams;
+	uint64_t				 FixedChunkingChunkSize	 = DefaultFixedChunkingChunkSize;
+	uint64_t				 MinSizeForFixedChunking = DefaultMinSizeForFixedChunking;
+};
+
+std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting);
 std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(CbObjectView Parameters);
 
 std::unique_ptr<ChunkingController> CreateChunkingController(std::string_view Name, CbObjectView Parameters);
diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h
new file mode 100644
index 000000000..f927d41e5
--- /dev/null
+++ b/src/zenutil/include/zenutil/commandlineoptions.h
@@ -0,0 +1,29 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+#include <filesystem>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+
+namespace cxxopts::values {
+// We declare this specialization before including cxxopts to make it stick
+void parse_value(const std::string& text, std::filesystem::path& value);
+}  // namespace cxxopts::values
+
+#include <cxxopts.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+std::vector<std::string>			ParseCommandLine(std::string_view CommandLine);
+std::vector<char*>					StripCommandlineQuotes(std::vector<std::string>& InOutArgs);
+void								MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path);
+[[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path);
+std::filesystem::path				StringToPath(const std::string_view& Path);
+std::string_view					RemoveQuotes(const std::string_view& Arg);
+
+void commandlineoptions_forcelink();  // internal
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h
new file mode 100644
index 000000000..7418608e4
--- /dev/null
+++ b/src/zenutil/include/zenutil/environmentoptions.h
@@ -0,0 +1,92 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/string.h>
+#include <zenutil/commandlineoptions.h>
+
+namespace zen {
+
+class EnvironmentOptions
+{
+public:
+	class OptionValue
+	{
+	public:
+		virtual void Parse(std::string_view Value) = 0;
+
+		virtual ~OptionValue() {}
+	};
+
+	class StringOption : public OptionValue
+	{
+	public:
+		explicit StringOption(std::string& Value);
+		virtual void Parse(std::string_view Value) override;
+		std::string& RefValue;
+	};
+
+	class FilePathOption : public OptionValue
+	{
+	public:
+		explicit FilePathOption(std::filesystem::path& Value);
+		virtual void		   Parse(std::string_view Value) override;
+		std::filesystem::path& RefValue;
+	};
+
+	class BoolOption : public OptionValue
+	{
+	public:
+		explicit BoolOption(bool& Value);
+		virtual void Parse(std::string_view Value);
+		bool&		 RefValue;
+	};
+
+	template<Integral T>
+	class NumberOption : public OptionValue
+	{
+	public:
+		explicit NumberOption(T& Value) : RefValue(Value) {}
+		virtual void Parse(std::string_view Value) override
+		{
+			if (std::optional<T> OptionalValue = ParseInt<T>(Value); OptionalValue.has_value())
+			{
+				RefValue = OptionalValue.value();
+			}
+		}
+		T& RefValue;
+	};
+
+	struct Option
+	{
+		std::string					 CommandLineOptionName;
+		std::shared_ptr<OptionValue> Value;
+	};
+
+	std::shared_ptr<OptionValue> MakeOption(std::string& Value);
+	std::shared_ptr<OptionValue> MakeOption(std::filesystem::path& Value);
+
+	template<Integral T>
+	std::shared_ptr<OptionValue> MakeOption(T& Value)
+	{
+		return std::make_shared<NumberOption<T>>(Value);
+	};
+
+	std::shared_ptr<OptionValue> MakeOption(bool& Value);
+
+	template<typename T>
+	void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "")
+	{
+		OptionMap.insert_or_assign(std::string(EnvName),
+								   Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)});
+	};
+
+	EnvironmentOptions();
+
+	void Parse(const cxxopts::ParseResult& CmdLineResult);
+
+private:
+	std::unordered_map<std::string, Option> OptionMap;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h
index 89fc70140..bbf070993 100644
--- a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h
+++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h
@@ -13,5 +13,6 @@ std::unique_ptr<BuildStorage> CreateJupiterBuildStorage(LoggerRef					 InLog,
 														BuildStorage::Statistics&	 Stats,
 														std::string_view			 Namespace,
 														std::string_view			 Bucket,
+														bool						 AllowRedirect,
 														const std::filesystem::path& TempFolderPath);
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
index 2c5fc73b8..b79790f25 100644
--- a/src/zenutil/include/zenutil/jupiter/jupitersession.h
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -65,7 +65,7 @@ struct FinalizeBuildPartResult : JupiterResult
 class JupiterSession
 {
 public:
-	JupiterSession(LoggerRef InLog, HttpClient& InHttpClient);
+	JupiterSession(LoggerRef InLog, HttpClient& InHttpClient, bool AllowRedirect);
 	~JupiterSession();
 
 	JupiterResult Authenticate();
@@ -102,6 +102,8 @@ public:
 
 	std::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes);
 
+	JupiterResult	   ListBuildNamespaces();
+	JupiterResult	   ListBuildBuckets(std::string_view Namespace);
 	JupiterResult	   ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload);
 	JupiterResult	   PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload);
 	JupiterResult	   GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
@@ -135,13 +137,14 @@ public:
 												  uint64_t														  PayloadSize,
 												  std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&&		  Transmitter,
 												  std::vector<std::function<JupiterResult(bool& OutIsComplete)>>& OutWorkItems);
-	JupiterResult			GetMultipartBuildBlob(std::string_view																		 Namespace,
-												  std::string_view																		 BucketId,
-												  const Oid&																			 BuildId,
-												  const IoHash&																			 Hash,
-												  uint64_t																				 ChunkSize,
-												  std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver,
-												  std::vector<std::function<JupiterResult()>>& OutWorkItems);
+	JupiterResult			GetMultipartBuildBlob(std::string_view												Namespace,
+												  std::string_view												BucketId,
+												  const Oid&													BuildId,
+												  const IoHash&													Hash,
+												  uint64_t														ChunkSize,
+												  std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+												  std::function<void()>&&										OnComplete,
+												  std::vector<std::function<JupiterResult()>>&					OutWorkItems);
 	JupiterResult			PutBlockMetadata(std::string_view Namespace,
 											 std::string_view BucketId,
 											 const Oid&		  BuildId,
@@ -152,9 +155,15 @@ public:
 											  const Oid&	   BuildId,
 											  const Oid&	   PartId,
 											  const IoHash&	   RawHash);
-	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
+	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount);
 	JupiterResult			GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload);
 
+	JupiterResult PutBuildPartStats(std::string_view Namespace,
+									std::string_view BucketId,
+									const Oid&		 BuildId,
+									const Oid&		 BuildPartId,
+									IoBuffer		 Payload);
+
 private:
 	inline LoggerRef Log() { return m_Log; }
 
@@ -164,6 +173,7 @@ private:
 
 	LoggerRef	m_Log;
 	HttpClient& m_HttpClient;
+	const bool	m_AllowRedirect = false;
 };
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h
index 758722156..cd28bdcb2 100644
--- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h
+++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h
@@ -27,7 +27,6 @@ public:
 	{
 		ZEN_MEMSCOPE(ELLMTag::Logging);
 
-		ZEN_MEMSCOPE(ELLMTag::Logging);
 		std::error_code Ec;
 		if (RotateOnOpen)
 		{
diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h
deleted file mode 100644
index 79798fc8d..000000000
--- a/src/zenutil/include/zenutil/parallellwork.h
+++ /dev/null
@@ -1,117 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/except.h>
-#include <zencore/fmtutils.h>
-#include <zencore/thread.h>
-#include <zencore/workthreadpool.h>
-
-#include <atomic>
-
-namespace zen {
-
-class ParallellWork
-{
-public:
-	ParallellWork(std::atomic<bool>& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) {}
-
-	~ParallellWork()
-	{
-		// Make sure to call Wait before destroying
-		ZEN_ASSERT(m_PendingWork.Remaining() == 0);
-	}
-
-	std::function<void(const std::exception& Ex, std::atomic<bool>& AbortFlag)> DefaultErrorFunction()
-	{
-		return [&](const std::exception& Ex, std::atomic<bool>& AbortFlag) {
-			m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex.what()); });
-			AbortFlag = true;
-		};
-	}
-
-	void ScheduleWork(WorkerThreadPool&																WorkerPool,
-					  std::function<void(std::atomic<bool>& AbortFlag)>&&							Work,
-					  std::function<void(const std::exception& Ex, std::atomic<bool>& AbortFlag)>&& OnError)
-	{
-		m_PendingWork.AddCount(1);
-		try
-		{
-			WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = std::move(OnError)] {
-				try
-				{
-					Work(m_AbortFlag);
-				}
-				catch (const AssertException& AssertEx)
-				{
-					OnError(
-						std::runtime_error(fmt::format("Caught assert exception while handling request: {}", AssertEx.FullDescription())),
-						m_AbortFlag);
-				}
-				catch (const std::system_error& SystemError)
-				{
-					if (IsOOM(SystemError.code()))
-					{
-						OnError(std::runtime_error(fmt::format("Out of memory. Reason: {}", SystemError.what())), m_AbortFlag);
-					}
-					else if (IsOOD(SystemError.code()))
-					{
-						OnError(std::runtime_error(fmt::format("Out of disk. Reason: {}", SystemError.what())), m_AbortFlag);
-					}
-					else
-					{
-						OnError(std::runtime_error(fmt::format("System error. Reason: {}", SystemError.what())), m_AbortFlag);
-					}
-				}
-				catch (const std::exception& Ex)
-				{
-					OnError(Ex, m_AbortFlag);
-				}
-				m_PendingWork.CountDown();
-			});
-		}
-		catch (const std::exception&)
-		{
-			m_PendingWork.CountDown();
-			throw;
-		}
-	}
-
-	void Abort() { m_AbortFlag = true; }
-
-	bool IsAborted() const { return m_AbortFlag.load(); }
-
-	void Wait(int32_t UpdateInteralMS, std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback)
-	{
-		ZEN_ASSERT(m_PendingWork.Remaining() > 0);
-		m_PendingWork.CountDown();
-		while (!m_PendingWork.Wait(UpdateInteralMS))
-		{
-			UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining());
-		}
-		if (m_Errors.size() == 1)
-		{
-			throw std::runtime_error(m_Errors.front());
-		}
-		else if (m_Errors.size() > 1)
-		{
-			ExtendableStringBuilder<128> SB;
-			SB.Append("Multiple errors:");
-			for (const std::string& Error : m_Errors)
-			{
-				SB.Append(fmt::format("\n    {}", Error));
-			}
-			throw std::runtime_error(SB.ToString());
-		}
-	}
-	Latch& PendingWork() { return m_PendingWork; }
-
-private:
-	std::atomic<bool>& m_AbortFlag;
-	Latch			   m_PendingWork;
-
-	RwLock					 m_ErrorLock;
-	std::vector<std::string> m_Errors;
-};
-
-}  // namespace zen
diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h
new file mode 100644
index 000000000..639c6968c
--- /dev/null
+++ b/src/zenutil/include/zenutil/parallelwork.h
@@ -0,0 +1,77 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/scopeguard.h>
+#include <zencore/thread.h>
+#include <zencore/workthreadpool.h>
+
+#include <atomic>
+
+namespace zen {
+
+class ParallelWork
+{
+public:
+	ParallelWork(std::atomic<bool>& AbortFlag, std::atomic<bool>& PauseFlag);
+
+	~ParallelWork();
+
+	typedef std::function<void(std::atomic<bool>& AbortFlag)>							   WorkCallback;
+	typedef std::function<void(std::exception_ptr Ex, std::atomic<bool>& AbortFlag)>	   ExceptionCallback;
+	typedef std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)> UpdateCallback;
+
+	void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {})
+	{
+		m_PendingWork.AddCount(1);
+		try
+		{
+			WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] {
+				auto _ = MakeGuard([this]() { m_PendingWork.CountDown(); });
+				try
+				{
+					while (m_PauseFlag && !m_AbortFlag)
+					{
+						Sleep(2000);
+					}
+					Work(m_AbortFlag);
+				}
+				catch (...)
+				{
+					OnError(std::current_exception(), m_AbortFlag);
+				}
+			});
+		}
+		catch (const std::exception&)
+		{
+			m_PendingWork.CountDown();
+			throw;
+		}
+	}
+
+	void Abort() { m_AbortFlag = true; }
+
+	bool IsAborted() const { return m_AbortFlag.load(); }
+
+	void Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback);
+
+	void Wait();
+
+	Latch& PendingWork() { return m_PendingWork; }
+
+private:
+	ExceptionCallback DefaultErrorFunction();
+	void			  RethrowErrors();
+
+	std::atomic<bool>& m_AbortFlag;
+	std::atomic<bool>& m_PauseFlag;
+	bool			   m_DispatchComplete = false;
+	Latch			   m_PendingWork;
+
+	RwLock							m_ErrorLock;
+	std::vector<std::exception_ptr> m_Errors;
+};
+
+void parallellwork_forcelink();
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h
index 9683ad720..df2033bca 100644
--- a/src/zenutil/include/zenutil/workerpools.h
+++ b/src/zenutil/include/zenutil/workerpools.h
@@ -21,6 +21,9 @@ WorkerThreadPool& GetMediumWorkerPool(EWorkloadType WorkloadType);
 // Worker pool with std::thread::hardware_concurrency() / 8 worker threads, but at least one thread
 WorkerThreadPool& GetSmallWorkerPool(EWorkloadType WorkloadType);
 
+// Worker pool with minimum number of worker threads, but at least one thread
+WorkerThreadPool& GetTinyWorkerPool(EWorkloadType WorkloadType);
+
 // Special worker pool that does not use worker thread but issues all scheduled work on the calling thread
 // This is useful for debugging when multiple async thread can make stepping in debugger complicated
 WorkerThreadPool& GetSyncWorkerPool();
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index bf89ce785..9974725ff 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -25,8 +25,9 @@ public:
 						Statistics&					 Stats,
 						std::string_view			 Namespace,
 						std::string_view			 Bucket,
+						bool						 AllowRedirect,
 						const std::filesystem::path& TempFolderPath)
-	: m_Session(InLog, InHttpClient)
+	: m_Session(InLog, InHttpClient, AllowRedirect)
 	, m_Stats(Stats)
 	, m_Namespace(Namespace)
 	, m_Bucket(Bucket)
@@ -35,6 +36,61 @@ public:
 	}
 	virtual ~JupiterBuildStorage() {}
 
+	virtual CbObject ListNamespaces(bool bRecursive) override
+	{
+		ZEN_TRACE_CPU("Jupiter::ListNamespaces");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult ListResult = m_Session.ListBuildNamespaces();
+		AddStatistic(ListResult);
+		if (!ListResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed listing namespaces: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+		}
+		CbObject NamespaceResponse = PayloadToCbObject("Failed listing namespaces"sv, ListResult.Response);
+
+		CbObjectWriter Response;
+		Response.BeginArray("results"sv);
+		for (CbFieldView NamespaceField : NamespaceResponse["namespaces"])
+		{
+			std::string_view Namespace = NamespaceField.AsString();
+			if (!Namespace.empty())
+			{
+				Response.BeginObject();
+				Response.AddString("name", Namespace);
+
+				if (bRecursive)
+				{
+					JupiterResult BucketsResult = m_Session.ListBuildBuckets(Namespace);
+					AddStatistic(BucketsResult);
+					if (!BucketsResult.Success)
+					{
+						throw std::runtime_error(
+							fmt::format("Failed listing namespaces: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode));
+					}
+					CbObject BucketResponse = PayloadToCbObject("Failed listing namespaces"sv, BucketsResult.Response);
+
+					Response.BeginArray("items");
+					for (CbFieldView BucketField : BucketResponse["buckets"])
+					{
+						std::string_view Bucket = BucketField.AsString();
+						if (!Bucket.empty())
+						{
+							Response.AddString(Bucket);
+						}
+					}
+					Response.EndArray();
+				}
+
+				Response.EndObject();
+			}
+		}
+		Response.EndArray();
+
+		return Response.Save();
+	}
+
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
 		ZEN_TRACE_CPU("Jupiter::ListBuilds");
@@ -49,7 +105,7 @@ public:
 		{
 			throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode));
 		}
-		return PayloadToJson("Failed listing builds"sv, ListResult.Response);
+		return PayloadToCbObject("Failed listing builds"sv, ListResult.Response);
 	}
 
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
@@ -66,7 +122,7 @@ public:
 		{
 			throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode));
 		}
-		return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
+		return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
 	}
 
 	virtual CbObject GetBuild(const Oid& BuildId) override
@@ -81,7 +137,7 @@ public:
 		{
 			throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode));
 		}
-		return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
+		return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
 	}
 
 	virtual void FinalizeBuild(const Oid& BuildId) override
@@ -134,7 +190,7 @@ public:
 												 GetBuildPartResult.Reason,
 												 GetBuildPartResult.ErrorCode));
 		}
-		return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
+		return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
 	}
 
 	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
@@ -235,19 +291,25 @@ public:
 		return std::move(GetBuildBlobResult.Response);
 	}
 
-	virtual std::vector<std::function<void()>> GetLargeBuildBlob(
-		const Oid&																			   BuildId,
-		const IoHash&																		   RawHash,
-		uint64_t																			   ChunkSize,
-		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid&													   BuildId,
+																 const IoHash&												   RawHash,
+																 uint64_t													   ChunkSize,
+																 std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+																 std::function<void()>&& OnComplete) override
 	{
 		ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob");
 
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		std::vector<std::function<JupiterResult()>> WorkItems;
-		JupiterResult								GetMultipartBlobResult =
-			m_Session.GetMultipartBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ChunkSize, std::move(Receiver), WorkItems);
+		JupiterResult								GetMultipartBlobResult = m_Session.GetMultipartBuildBlob(m_Namespace,
+																				 m_Bucket,
+																				 BuildId,
+																				 RawHash,
+																				 ChunkSize,
+																				 std::move(OnReceive),
+																				 std::move(OnComplete),
+																				 WorkItems);
 
 		AddStatistic(GetMultipartBlobResult);
 		if (!GetMultipartBlobResult.Success)
@@ -272,7 +334,7 @@ public:
 		return WorkList;
 	}
 
-	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
 		ZEN_TRACE_CPU("Jupiter::PutBlockMetadata");
 
@@ -284,27 +346,32 @@ public:
 		AddStatistic(PutMetaResult);
 		if (!PutMetaResult.Success)
 		{
+			if (PutMetaResult.ErrorCode == int32_t(HttpResponseCode::NotFound))
+			{
+				return false;
+			}
 			throw std::runtime_error(
 				fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode));
 		}
+		return true;
 	}
 
-	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
+	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override
 	{
 		ZEN_TRACE_CPU("Jupiter::FindBlocks");
 
 		Stopwatch	  ExecutionTimer;
 		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
-		JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId);
+		JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId, MaxBlockCount);
 		AddStatistic(FindResult);
 		if (!FindResult.Success)
 		{
 			throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode));
 		}
-		return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response));
+		return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response);
 	}
 
-	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
 		ZEN_TRACE_CPU("Jupiter::GetBlockMetadata");
 
@@ -328,28 +395,35 @@ public:
 			throw std::runtime_error(
 				fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode));
 		}
-		std::vector<ChunkBlockDescription> UnorderedList =
-			ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response));
-		tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup;
-		for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++)
+		return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response);
+	}
+
+	virtual void PutBuildPartStats(const Oid&								  BuildId,
+								   const Oid&								  BuildPartId,
+								   const tsl::robin_map<std::string, double>& FloatStats) override
+	{
+		ZEN_UNUSED(BuildId, BuildPartId, FloatStats);
+		CbObjectWriter Request;
+		Request.BeginObject("floatStats"sv);
+		for (auto It : FloatStats)
 		{
-			const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex];
-			BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex);
+			Request.AddFloat(It.first, It.second);
 		}
-		std::vector<ChunkBlockDescription> SortedBlockDescriptions;
-		SortedBlockDescriptions.reserve(BlockDescriptionLookup.size());
-		for (const IoHash& BlockHash : BlockHashes)
+		Request.EndObject();
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult PutBuildPartStatsResult = m_Session.PutBuildPartStats(m_Namespace, m_Bucket, BuildId, BuildPartId, Payload);
+		AddStatistic(PutBuildPartStatsResult);
+		if (!PutBuildPartStatsResult.Success)
 		{
-			if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end())
-			{
-				SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second]));
-			}
+			throw std::runtime_error(fmt::format("Failed posting build part statistics: {} ({})",
+												 PutBuildPartStatsResult.Reason,
+												 PutBuildPartStatsResult.ErrorCode));
 		}
-		return SortedBlockDescriptions;
 	}
 
 private:
-	static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload)
+	static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload)
 	{
 		if (Payload.GetContentType() == ZenContentType::kJSON)
 		{
@@ -394,11 +468,12 @@ CreateJupiterBuildStorage(LoggerRef					   InLog,
 						  BuildStorage::Statistics&	   Stats,
 						  std::string_view			   Namespace,
 						  std::string_view			   Bucket,
+						  bool						   AllowRedirect,
 						  const std::filesystem::path& TempFolderPath)
 {
 	ZEN_TRACE_CPU("CreateJupiterBuildStorage");
 
-	return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath);
+	return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, AllowRedirect, TempFolderPath);
 }
 
 }  // namespace zen
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index 68f214c06..1fd59acdf 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -5,6 +5,7 @@
 #include <zencore/compactbinary.h>
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compositebuffer.h>
+#include <zencore/compress.h>
 #include <zencore/fmtutils.h>
 #include <zencore/trace.h>
 
@@ -48,7 +49,10 @@ namespace detail {
 	}
 }  // namespace detail
 
-JupiterSession::JupiterSession(LoggerRef InLog, HttpClient& InHttpClient) : m_Log(InLog), m_HttpClient(InHttpClient)
+JupiterSession::JupiterSession(LoggerRef InLog, HttpClient& InHttpClient, bool AllowRedirect)
+: m_Log(InLog)
+, m_HttpClient(InHttpClient)
+, m_AllowRedirect(AllowRedirect)
 {
 }
 
@@ -357,12 +361,28 @@ JupiterSession::CacheTypeExists(std::string_view Namespace, std::string_view Typ
 }
 
 JupiterResult
+JupiterSession::ListBuildNamespaces()
+{
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds"), {HttpClient::Accept(ZenContentType::kJSON)});
+	return detail::ConvertResponse(Response, "JupiterSession::ListBuildNamespaces"sv);
+}
+
+JupiterResult
+JupiterSession::ListBuildBuckets(std::string_view Namespace)
+{
+	HttpClient::Response Response =
+		m_HttpClient.Get(fmt::format("/api/v2/builds/{}", Namespace), {HttpClient::Accept(ZenContentType::kJSON)});
+	return detail::ConvertResponse(Response, "JupiterSession::ListBuildBuckets"sv);
+}
+
+JupiterResult
 JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
-	HttpClient::Response Response = m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/search", Namespace, BucketId),
-													  Payload,
-													  {HttpClient::Accept(ZenContentType::kCbObject)});
+	std::string			 OptionalBucketPath = BucketId.empty() ? "" : fmt::format("/{}", BucketId);
+	HttpClient::Response Response			= m_HttpClient.Post(fmt::format("/api/v2/builds/{}{}/search", Namespace, OptionalBucketPath),
+														Payload,
+														{HttpClient::Accept(ZenContentType::kCbObject)});
 	return detail::ConvertResponse(Response, "JupiterSession::ListBuilds"sv);
 }
 
@@ -527,12 +547,14 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 									  bool& OutIsComplete) -> JupiterResult {
 			const MultipartUploadResponse::Part& Part		 = Workload->PartDescription.Parts[PartIndex];
 			IoBuffer							 PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte);
-			std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
-																		   Namespace,
-																		   BucketId,
-																		   BuildId,
-																		   Hash.ToHexString(),
-																		   Part.QueryString);
+			std::string							 MultipartUploadResponseRequestString =
+				fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}&supportsRedirect={}",
+							Namespace,
+							BucketId,
+							BuildId,
+							Hash.ToHexString(),
+							Part.QueryString,
+							m_AllowRedirect ? "true"sv : "false"sv);
 			// ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString);
 			HttpClient::Response MultipartUploadResponse = m_HttpClient.Put(MultipartUploadResponseRequestString, PartPayload);
 			if (!MultipartUploadResponse.IsSuccess())
@@ -590,12 +612,13 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 								IoBuffer							 RetryPartPayload =
 									Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte - 1);
 								std::string RetryMultipartUploadResponseRequestString =
-									fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
+									fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}&supportsRedirect={}",
 												Namespace,
 												BucketId,
 												BuildId,
 												Hash.ToHexString(),
-												RetryPart.QueryString);
+												RetryPart.QueryString,
+												m_AllowRedirect ? "true"sv : "false"sv);
 
 								MultipartUploadResponse = m_HttpClient.Put(RetryMultipartUploadResponseRequestString, RetryPartPayload);
 								TotalUploadedBytes		= MultipartUploadResponse.UploadedBytes;
@@ -626,15 +649,21 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 }
 
 JupiterResult
-JupiterSession::GetMultipartBuildBlob(std::string_view																		 Namespace,
-									  std::string_view																		 BucketId,
-									  const Oid&																			 BuildId,
-									  const IoHash&																			 Hash,
-									  uint64_t																				 ChunkSize,
-									  std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver,
-									  std::vector<std::function<JupiterResult()>>&											 OutWorkItems)
-{
-	std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
+JupiterSession::GetMultipartBuildBlob(std::string_view												Namespace,
+									  std::string_view												BucketId,
+									  const Oid&													BuildId,
+									  const IoHash&													Hash,
+									  uint64_t														ChunkSize,
+									  std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+									  std::function<void()>&&										OnComplete,
+									  std::vector<std::function<JupiterResult()>>&					OutWorkItems)
+{
+	std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}",
+										  Namespace,
+										  BucketId,
+										  BuildId,
+										  Hash.ToHexString(),
+										  m_AllowRedirect ? "true"sv : "false"sv);
 	HttpClient::Response Response =
 		m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}}));
 	if (Response.IsSuccess())
@@ -649,46 +678,68 @@ JupiterSession::GetMultipartBuildBlob(std::string_view																		 Namespa
 					uint64_t TotalSize	 = TotalSizeMaybe.value();
 					uint64_t PayloadSize = Response.ResponsePayload.GetSize();
 
-					Receiver(0, Response.ResponsePayload, TotalSize);
+					OnReceive(0, Response.ResponsePayload);
 
 					if (TotalSize > PayloadSize)
 					{
 						struct WorkloadData
 						{
-							std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)> Receiver;
-							std::atomic<uint64_t>																 BytesRemaining;
+							std::function<void(uint64_t Offset, const IoBuffer& Chunk)> OnReceive;
+							std::function<void()>										OnComplete;
+							std::atomic<uint64_t>										BytesRemaining;
 						};
 
 						std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
-						Workload->Receiver		 = std::move(Receiver);
+						Workload->OnReceive		 = std::move(OnReceive);
+						Workload->OnComplete	 = std::move(OnComplete);
 						Workload->BytesRemaining = TotalSize - PayloadSize;
 
 						uint64_t Offset = PayloadSize;
 						while (Offset < TotalSize)
 						{
 							uint64_t PartSize = Min(ChunkSize, TotalSize - Offset);
-							OutWorkItems.emplace_back(
-								[this, Namespace, BucketId, BuildId, Hash, TotalSize, Workload, Offset, PartSize]() -> JupiterResult {
-									std::string RequestUrl =
-										fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
-									HttpClient::Response Response = m_HttpClient.Get(
-										RequestUrl,
-										HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
-									if (Response.IsSuccess())
+							OutWorkItems.emplace_back([this,
+													   Namespace = std::string(Namespace),
+													   BucketId	 = std::string(BucketId),
+													   BuildId	 = Oid(BuildId),
+													   Hash		 = IoHash(Hash),
+													   TotalSize,
+													   Workload,
+													   Offset,
+													   PartSize]() -> JupiterResult {
+								std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}",
+																	  Namespace,
+																	  BucketId,
+																	  BuildId,
+																	  Hash.ToHexString(),
+																	  m_AllowRedirect ? "true"sv : "false"sv);
+								HttpClient::Response Response	= m_HttpClient.Get(
+									  RequestUrl,
+									  HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
+								if (Response.IsSuccess())
+								{
+									Workload->OnReceive(Offset, Response.ResponsePayload);
+									uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize());
+									if (ByteRemaning == Response.ResponsePayload.GetSize())
 									{
-										uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize());
-										Workload->Receiver(Offset, Response.ResponsePayload, ByteRemaning);
+										Workload->OnComplete();
 									}
-									return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv);
-								});
+								}
+								return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv);
+							});
 							Offset += PartSize;
 						}
 					}
+					else
+					{
+						OnComplete();
+					}
 					return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv);
 				}
 			}
 		}
-		Receiver(0, Response.ResponsePayload, Response.ResponsePayload.GetSize());
+		OnReceive(0, Response.ResponsePayload);
+		OnComplete();
 	}
 	return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv);
 }
@@ -707,11 +758,28 @@ JupiterSession::GetBuildBlob(std::string_view	   Namespace,
 	{
 		Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", Offset, Offset + Size - 1)});
 	}
-	HttpClient::Response Response =
-		m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
-							  TempFolderPath,
-							  Headers);
-
+	HttpClient::Response Response = m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}",
+																	  Namespace,
+																	  BucketId,
+																	  BuildId,
+																	  Hash.ToHexString(),
+																	  m_AllowRedirect ? "true"sv : "false"sv),
+														  TempFolderPath,
+														  Headers);
+	if (Response.IsSuccess())
+	{
+		// If we get a redirect to S3 or a non-Jupiter endpoint the content type will not be correct, validate it and set it
+		if (m_AllowRedirect && (Response.ResponsePayload.GetContentType() == HttpContentType::kBinary))
+		{
+			IoHash	 ValidateRawHash;
+			uint64_t ValidateRawSize = 0;
+			ZEN_ASSERT_SLOW(CompressedBuffer::ValidateCompressedHeader(Response.ResponsePayload, ValidateRawHash, ValidateRawSize));
+			ZEN_ASSERT_SLOW(ValidateRawHash == Hash);
+			ZEN_ASSERT_SLOW(ValidateRawSize > 0);
+			ZEN_UNUSED(ValidateRawHash, ValidateRawSize);
+			Response.ResponsePayload.SetContentType(ZenContentType::kCompressedBinary);
+		}
+	}
 	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
 }
 
@@ -758,10 +826,12 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace,
 }
 
 JupiterResult
-JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId)
+JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount)
 {
-	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId),
-													 HttpClient::Accept(ZenContentType::kCbObject));
+	const std::string	 Parameters = MaxBlockCount == (uint64_t)-1 ? "" : fmt::format("?count={}", MaxBlockCount);
+	HttpClient::Response Response =
+		m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks{}", Namespace, BucketId, BuildId, Parameters),
+						 HttpClient::Accept(ZenContentType::kCbObject));
 	return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv);
 }
 
@@ -776,4 +846,19 @@ JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view Bu
 	return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv);
 }
 
+JupiterResult
+JupiterSession::PutBuildPartStats(std::string_view Namespace,
+								  std::string_view BucketId,
+								  const Oid&	   BuildId,
+								  const Oid&	   BuildPartId,
+								  IoBuffer		   Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+	HttpClient::Response Response =
+		m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/stats", Namespace, BucketId, BuildId, BuildPartId),
+						 Payload,
+						 HttpClient::Accept(ZenContentType::kCbObject));
+	return detail::ConvertResponse(Response, "JupiterSession::PutBuildPartStats"sv);
+}
+
 }  // namespace zen
diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp
new file mode 100644
index 000000000..aa806438b
--- /dev/null
+++ b/src/zenutil/parallelwork.cpp
@@ -0,0 +1,225 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/parallelwork.h>
+
+#include <zencore/callstack.h>
+#include <zencore/except.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+
+#include <typeinfo>
+
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#endif	// ZEN_WITH_TESTS
+
+namespace zen {
+
+ParallelWork::ParallelWork(std::atomic<bool>& AbortFlag, std::atomic<bool>& PauseFlag)
+: m_AbortFlag(AbortFlag)
+, m_PauseFlag(PauseFlag)
+, m_PendingWork(1)
+{
+}
+
+ParallelWork::~ParallelWork()
+{
+	try
+	{
+		if (!m_DispatchComplete)
+		{
+			ZEN_ASSERT(m_PendingWork.Remaining() > 0);
+			ZEN_WARN(
+				"ParallelWork disposed without explicit wait for completion, likely caused by an exception, waiting for dispatched threads "
+				"to complete");
+			m_PendingWork.CountDown();
+		}
+		m_PendingWork.Wait();
+		ptrdiff_t RemainingWork = m_PendingWork.Remaining();
+		if (RemainingWork != 0)
+		{
+			void*			 Frames[8];
+			uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+			CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+			ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}",
+					  RemainingWork,
+					  CallstackToString(Callstack, "  "));
+			FreeCallstack(Callstack);
+
+			uint32_t WaitedMs = 0;
+			while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000)
+			{
+				Sleep(50);
+				WaitedMs += 50;
+			}
+			RemainingWork = m_PendingWork.Remaining();
+			if (RemainingWork != 0)
+			{
+				ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork)
+			}
+			else
+			{
+				ZEN_INFO("ParallelWork destructor safety wait succeeded");
+			}
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("Exception in ~ParallelWork: {}", Ex.what());
+	}
+}
+
+ParallelWork::ExceptionCallback
+ParallelWork::DefaultErrorFunction()
+{
+	return [&](std::exception_ptr Ex, std::atomic<bool>& AbortFlag) {
+		m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex); });
+		AbortFlag = true;
+	};
+}
+
+void
+ParallelWork::Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback)
+{
+	ZEN_ASSERT(!m_DispatchComplete);
+	m_DispatchComplete = true;
+
+	ZEN_ASSERT(m_PendingWork.Remaining() > 0);
+	m_PendingWork.CountDown();
+
+	while (!m_PendingWork.Wait(UpdateIntervalMS))
+	{
+		UpdateCallback(m_AbortFlag.load(), m_PauseFlag.load(), m_PendingWork.Remaining());
+	}
+
+	RethrowErrors();
+}
+
+void
+ParallelWork::Wait()
+{
+	ZEN_ASSERT(!m_DispatchComplete);
+	m_DispatchComplete = true;
+
+	ZEN_ASSERT(m_PendingWork.Remaining() > 0);
+	m_PendingWork.CountDown();
+	m_PendingWork.Wait();
+
+	RethrowErrors();
+}
+
+void
+ParallelWork::RethrowErrors()
+{
+	if (!m_Errors.empty())
+	{
+		if (m_Errors.size() > 1)
+		{
+			ZEN_INFO("Multiple exceptions thrown during ParallelWork execution, dropping the following exceptions:");
+			auto It = m_Errors.begin() + 1;
+			while (It != m_Errors.end())
+			{
+				try
+				{
+					std::rethrow_exception(*It);
+				}
+				catch (const std::exception& Ex)
+				{
+					ZEN_INFO("    {}", Ex.what());
+				}
+				It++;
+			}
+		}
+		std::exception_ptr Ex = m_Errors.front();
+		m_Errors.clear();
+		std::rethrow_exception(Ex);
+	}
+}
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("parallellwork.nowork")
+{
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	Work.Wait();
+}
+
+TEST_CASE("parallellwork.basic")
+{
+	WorkerThreadPool WorkerPool(2);
+
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	for (uint32_t I = 0; I < 5; I++)
+	{
+		Work.ScheduleWork(WorkerPool, [](std::atomic<bool>& AbortFlag) { CHECK(!AbortFlag); });
+	}
+	Work.Wait();
+}
+
+TEST_CASE("parallellwork.throws_in_work")
+{
+	WorkerThreadPool WorkerPool(2);
+
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	for (uint32_t I = 0; I < 10; I++)
+	{
+		Work.ScheduleWork(WorkerPool, [I](std::atomic<bool>& AbortFlag) {
+			ZEN_UNUSED(AbortFlag);
+			if (I > 3)
+			{
+				throw std::runtime_error("We throw in async thread");
+			}
+			else
+			{
+				Sleep(10);
+			}
+		});
+	}
+	CHECK_THROWS_WITH(Work.Wait(), "We throw in async thread");
+}
+
+TEST_CASE("parallellwork.throws_in_dispatch")
+{
+	WorkerThreadPool	  WorkerPool(2);
+	std::atomic<uint32_t> ExecutedCount;
+	try
+	{
+		std::atomic<bool> AbortFlag;
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
+		for (uint32_t I = 0; I < 5; I++)
+		{
+			Work.ScheduleWork(WorkerPool, [I, &ExecutedCount](std::atomic<bool>& AbortFlag) {
+				if (AbortFlag.load())
+				{
+					return;
+				}
+				ExecutedCount++;
+			});
+			if (I == 3)
+			{
+				throw std::runtime_error("We throw in dispatcher thread");
+			}
+		}
+		CHECK(false);
+	}
+	catch (const std::runtime_error& Ex)
+	{
+		CHECK_EQ("We throw in dispatcher thread", std::string(Ex.what()));
+		CHECK_LE(ExecutedCount.load(), 4);
+	}
+}
+
+void
+parallellwork_forcelink()
+{
+}
+#endif	// ZEN_WITH_TESTS
+
+}  // namespace zen
diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp
index e3165e838..797034978 100644
--- a/src/zenutil/workerpools.cpp
+++ b/src/zenutil/workerpools.cpp
@@ -11,9 +11,10 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 namespace {
-	const int LargeWorkerThreadPoolTreadCount  = gsl::narrow<int>(std::thread::hardware_concurrency());
+	const int LargeWorkerThreadPoolTreadCount  = gsl::narrow<int>(Max(std::thread::hardware_concurrency() - 1u, 2u));
 	const int MediumWorkerThreadPoolTreadCount = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 4u), 2u));
 	const int SmallWorkerThreadPoolTreadCount  = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 8u), 1u));
+	const int TinyWorkerThreadPoolTreadCount   = 1;
 
 	static bool IsShutDown = false;
 
@@ -35,6 +36,9 @@ namespace {
 	WorkerPool BurstSmallWorkerPool		 = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(burst)"};
 	WorkerPool BackgroundSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(bkg)"};
 
+	WorkerPool BurstTinyWorkerPool		= {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(burst)"};
+	WorkerPool BackgroundTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(bkg)"};
+
 	WorkerPool SyncWorkerPool = {.TreadCount = 0, .Name = "SyncThreadPool"};
 
 	WorkerThreadPool& EnsurePoolPtr(WorkerPool& Pool)
@@ -75,6 +79,12 @@ GetSmallWorkerPool(EWorkloadType WorkloadType)
 }
 
 WorkerThreadPool&
+GetTinyWorkerPool(EWorkloadType WorkloadType)
+{
+	return EnsurePoolPtr(WorkloadType == EWorkloadType::Burst ? BurstTinyWorkerPool : BackgroundTinyWorkerPool);
+}
+
+WorkerThreadPool&
 GetSyncWorkerPool()
 {
 	return EnsurePoolPtr(SyncWorkerPool);
@@ -91,6 +101,8 @@ ShutdownWorkerPools()
 	BackgroundMediumWorkerPool.Pool.reset();
 	BurstSmallWorkerPool.Pool.reset();
 	BackgroundSmallWorkerPool.Pool.reset();
+	BurstTinyWorkerPool.Pool.reset();
+	BackgroundTinyWorkerPool.Pool.reset();
 	SyncWorkerPool.Pool.reset();
 }
 }  // namespace zen
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 0409cb976..4fd1cef9a 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -547,7 +547,7 @@ ZenServerEnvironment::CreateNewTestDir()
 	TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1));
 
 	std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str();
-	ZEN_ASSERT(!std::filesystem::exists(TestPath));
+	ZEN_ASSERT(!IsDir(TestPath));
 
 	ZEN_INFO("Creating new test dir @ '{}'", TestPath);
 
@@ -581,7 +581,7 @@ ZenServerInstance::~ZenServerInstance()
 	{
 		Shutdown();
 		std::error_code DummyEc;
-		std::filesystem::remove(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc);
+		RemoveFile(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc);
 	}
 	catch (const std::exception& Err)
 	{
@@ -632,6 +632,7 @@ ZenServerInstance::Shutdown()
 				std::error_code Ec;
 				if (SignalShutdown(Ec))
 				{
+					Stopwatch Timer;
 					ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
 					while (!m_Process.Wait(1000))
 					{
@@ -645,7 +646,10 @@ ZenServerInstance::Shutdown()
 							ZEN_WARN("Wait abandoned by exited process");
 							return 0;
 						}
-						ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid());
+						ZEN_WARN("Waited for zenserver process {} ({}) to exit for {}",
+								 m_Name,
+								 m_Process.Pid(),
+								 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 					}
 					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 					int ExitCode = m_Process.GetExitCode();
@@ -1046,7 +1050,7 @@ std::string
 ZenServerInstance::GetLogOutput() const
 {
 	std::filesystem::path OutputPath = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
-	if (std::filesystem::is_regular_file(OutputPath))
+	if (IsFile(OutputPath))
 	{
 		FileContents Contents = ReadFile(OutputPath);
 		if (!Contents.ErrorCode)
@@ -1068,7 +1072,7 @@ ZenServerInstance::Terminate()
 	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
 	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
 	ProcessHandle				RunningProcess;
-	std::error_code				Ec = FindProcess(Executable, RunningProcess);
+	std::error_code				Ec = FindProcess(Executable, RunningProcess, /*IncludeSelf*/ false);
 	if (Ec)
 	{
 		throw std::system_error(Ec, fmt::format("failed to look up running server executable '{}'", Executable));
@@ -1136,7 +1140,7 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason)
 		OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort);
 		return false;
 	}
-	if (!std::filesystem::is_directory(Info.DataDir))
+	if (!IsDir(Info.DataDir))
 	{
 		OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir);
 		return false;
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
index 19eb63ce9..fe23b00c1 100644
--- a/src/zenutil/zenutil.cpp
+++ b/src/zenutil/zenutil.cpp
@@ -7,6 +7,8 @@
 #	include <zenutil/cache/cacherequests.h>
 #	include <zenutil/cache/rpcrecording.h>
 #	include <zenutil/chunkedfile.h>
+#	include <zenutil/commandlineoptions.h>
+#	include <zenutil/parallelwork.h>
 
 namespace zen {
 
@@ -17,6 +19,8 @@ zenutil_forcelinktests()
 	cache::rpcrecord_forcelink();
 	cacherequests_forcelink();
 	chunkedfile_forcelink();
+	commandlineoptions_forcelink();
+	parallellwork_forcelink();
 }
 
 }  // namespace zen
diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp
index a3cfe9d15..9cec5372a 100644
--- a/src/zenvfs/vfsprovider.cpp
+++ b/src/zenvfs/vfsprovider.cpp
@@ -373,13 +373,12 @@ VfsProvider::Initialize()
 	std::filesystem::path ManifestPath = Root / ".zen_vfs";
 	bool				  HaveManifest = false;
 
-	if (std::filesystem::exists(Root))
+	if (IsFile(Root))
+	{
+		throw std::runtime_error("specified VFS root exists but is not a directory");
+	}
+	if (IsDir(Root))
 	{
-		if (!std::filesystem::is_directory(Root))
-		{
-			throw std::runtime_error("specified VFS root exists but is not a directory");
-		}
-
 		std::error_code Ec;
 		m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str());
 
@@ -388,7 +387,7 @@ VfsProvider::Initialize()
 			throw std::system_error(Ec);
 		}
 
-		if (std::filesystem::exists(ManifestPath))
+		if (IsFile(ManifestPath))
 		{
 			FileContents ManifestData = zen::ReadFile(ManifestPath);
 			CbObject	 Manifest	  = LoadCompactBinaryObject(ManifestData.Flatten());