Merge branch 'main' into zs/put-overwrite-policy

39 files changed, 6287 insertions, 78 deletions

diff --git a/src/zenutil/bufferedwritefilecache.cpp b/src/zenutil/bufferedwritefilecache.cpp
new file mode 100644
index 000000000..a52850314
--- /dev/null
+++ b/src/zenutil/bufferedwritefilecache.cpp
@@ -0,0 +1,177 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/bufferedwritefilecache.h>
+
+#include <zencore/logging.h>
+#include <zencore/trace.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+BufferedWriteFileCache::BufferedWriteFileCache() : m_CacheHitCount(0), m_CacheMissCount(0), m_OpenHandleCount(0), m_DroppedHandleCount(0)
+{
+}
+
+BufferedWriteFileCache::~BufferedWriteFileCache()
+{
+	ZEN_TRACE_CPU("~BufferedWriteFileCache()");
+
+	try
+	{
+		for (TOpenHandles& OpenHandles : m_OpenFiles)
+		{
+			while (BasicFile* File = OpenHandles.Pop())
+			{
+				std::unique_ptr<BasicFile> FileToClose(File);
+				m_OpenHandleCount--;
+			}
+		}
+		m_OpenFiles.clear();
+		m_ChunkWriters.clear();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("~BufferedWriteFileCache() threw exeption: {}", Ex.what());
+	}
+}
+
+std::unique_ptr<BasicFile>
+BufferedWriteFileCache::Get(uint32_t FileIndex)
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::Get");
+
+	RwLock::ExclusiveLockScope _(m_WriterLock);
+	if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end())
+	{
+		const uint32_t HandleIndex = It->second;
+		TOpenHandles&  OpenHandles = m_OpenFiles[HandleIndex];
+		if (BasicFile* File = OpenHandles.Pop(); File != nullptr)
+		{
+			m_OpenHandleCount--;
+			m_CacheHitCount++;
+			return std::unique_ptr<BasicFile>(File);
+		}
+	}
+	m_CacheMissCount++;
+	return nullptr;
+}
+
+void
+BufferedWriteFileCache::Put(uint32_t FileIndex, std::unique_ptr<BasicFile>&& Writer)
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::Put");
+
+	if (m_OpenHandleCount.load() >= MaxBufferedCount)
+	{
+		m_DroppedHandleCount++;
+		return;
+	}
+	RwLock::ExclusiveLockScope _(m_WriterLock);
+	if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end())
+	{
+		const uint32_t HandleIndex = It->second;
+		TOpenHandles&  OpenHandles = m_OpenFiles[HandleIndex];
+		if (OpenHandles.Push(Writer.get()))
+		{
+			Writer.release();
+			m_OpenHandleCount++;
+		}
+		else
+		{
+			m_DroppedHandleCount++;
+		}
+	}
+	else
+	{
+		const uint32_t HandleIndex = gsl::narrow<uint32_t>(m_OpenFiles.size());
+		m_OpenFiles.push_back(TOpenHandles{});
+		m_OpenFiles.back().Push(Writer.release());
+		m_ChunkWriters.insert_or_assign(FileIndex, HandleIndex);
+		m_OpenHandleCount++;
+	}
+}
+
+void
+BufferedWriteFileCache::Close(std::span<uint32_t> FileIndexes)
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::Close");
+
+	std::vector<std::unique_ptr<BasicFile>> FilesToClose;
+	FilesToClose.reserve(FileIndexes.size());
+	{
+		RwLock::ExclusiveLockScope _(m_WriterLock);
+		for (uint32_t FileIndex : FileIndexes)
+		{
+			if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end())
+			{
+				const uint32_t HandleIndex = It->second;
+				TOpenHandles&  OpenHandles = m_OpenFiles[HandleIndex];
+				while (BasicFile* File = OpenHandles.Pop())
+				{
+					FilesToClose.emplace_back(std::unique_ptr<BasicFile>(File));
+					m_OpenHandleCount--;
+				}
+				m_ChunkWriters.erase(It);
+			}
+		}
+	}
+	FilesToClose.clear();
+}
+
+BufferedWriteFileCache::Local::Local(BufferedWriteFileCache& Cache) : m_Cache(Cache)
+{
+}
+
+BufferedWriteFileCache::Local::Writer*
+BufferedWriteFileCache::Local::GetWriter(uint32_t FileIndex)
+{
+	if (auto It = m_FileIndexToWriterIndex.find(FileIndex); It != m_FileIndexToWriterIndex.end())
+	{
+		return m_ChunkWriters[It->second].get();
+	}
+	std::unique_ptr<BasicFile> File = m_Cache.Get(FileIndex);
+	if (File)
+	{
+		const uint32_t WriterIndex = gsl::narrow<uint32_t>(m_ChunkWriters.size());
+		m_FileIndexToWriterIndex.insert_or_assign(FileIndex, WriterIndex);
+		m_ChunkWriters.emplace_back(std::make_unique<Writer>(Writer{.File = std::move(File)}));
+		return m_ChunkWriters.back().get();
+	}
+	return nullptr;
+}
+
+BufferedWriteFileCache::Local::Writer*
+BufferedWriteFileCache::Local::PutWriter(uint32_t FileIndex, std::unique_ptr<Writer> Writer)
+{
+	ZEN_ASSERT(!m_FileIndexToWriterIndex.contains(FileIndex));
+	const uint32_t WriterIndex = gsl::narrow<uint32_t>(m_ChunkWriters.size());
+	m_FileIndexToWriterIndex.insert_or_assign(FileIndex, WriterIndex);
+	m_ChunkWriters.emplace_back(std::move(Writer));
+	return m_ChunkWriters.back().get();
+}
+
+BufferedWriteFileCache::Local::~Local()
+{
+	ZEN_TRACE_CPU("BufferedWriteFileCache::~Local()");
+	try
+	{
+		for (auto& It : m_FileIndexToWriterIndex)
+		{
+			const uint32_t FileIndex   = It.first;
+			const uint32_t WriterIndex = It.second;
+			m_ChunkWriters[WriterIndex]->Writer.reset();
+			std::unique_ptr<BasicFile> File;
+			File.swap(m_ChunkWriters[WriterIndex]->File);
+			m_Cache.Put(FileIndex, std::move(File));
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("BufferedWriteFileCache::~Local() threw exeption: {}", Ex.what());
+	}
+}
+
+}  // namespace zen
diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp
new file mode 100644
index 000000000..88238effd
--- /dev/null
+++ b/src/zenutil/buildstoragecache.cpp
@@ -0,0 +1,407 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/buildstoragecache.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zencore/workthreadpool.h>
+#include <zenhttp/httpclient.h>
+#include <zenhttp/packageformat.h>
+#include <zenutil/workerpools.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+class ZenBuildStorageCache : public BuildStorageCache
+{
+public:
+	explicit ZenBuildStorageCache(HttpClient&					 HttpClient,
+								  BuildStorageCache::Statistics& Stats,
+								  std::string_view				 Namespace,
+								  std::string_view				 Bucket,
+								  const std::filesystem::path&	 TempFolderPath,
+								  bool							 BoostBackgroundThreadCount)
+	: m_HttpClient(HttpClient)
+	, m_Stats(Stats)
+	, m_Namespace(Namespace.empty() ? "none" : Namespace)
+	, m_Bucket(Bucket.empty() ? "none" : Bucket)
+	, m_TempFolderPath(std::filesystem::path(TempFolderPath).make_preferred())
+	, m_BoostBackgroundThreadCount(BoostBackgroundThreadCount)
+	, m_BackgroundWorkPool(m_BoostBackgroundThreadCount ? GetSmallWorkerPool(EWorkloadType::Background)
+														: GetTinyWorkerPool(EWorkloadType::Background))
+	, m_PendingBackgroundWorkCount(1)
+	, m_CancelBackgroundWork(false)
+	{
+	}
+
+	virtual ~ZenBuildStorageCache()
+	{
+		try
+		{
+			m_CancelBackgroundWork.store(true);
+			if (!IsFlushed)
+			{
+				m_PendingBackgroundWorkCount.CountDown();
+				m_PendingBackgroundWorkCount.Wait();
+			}
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_ERROR("~ZenBuildStorageCache() failed with: {}", Ex.what());
+		}
+	}
+
+	void ScheduleBackgroundWork(std::function<void()>&& Work)
+	{
+		m_PendingBackgroundWorkCount.AddCount(1);
+		try
+		{
+			m_BackgroundWorkPool.ScheduleWork([this, Work = std::move(Work)]() {
+				ZEN_TRACE_CPU("ZenBuildStorageCache::BackgroundWork");
+				auto _ = MakeGuard([this]() { m_PendingBackgroundWorkCount.CountDown(); });
+				if (!m_CancelBackgroundWork)
+				{
+					try
+					{
+						Work();
+					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("Failed executing background upload to build cache. Reason: {}", Ex.what());
+					}
+				}
+			});
+		}
+		catch (const std::exception& Ex)
+		{
+			m_PendingBackgroundWorkCount.CountDown();
+			ZEN_ERROR("Failed scheduling background upload to build cache. Reason: {}", Ex.what());
+		}
+	}
+
+	virtual void PutBuildBlob(const Oid&			 BuildId,
+							  const IoHash&			 RawHash,
+							  ZenContentType		 ContentType,
+							  const CompositeBuffer& Payload) override
+	{
+		ZEN_ASSERT(!IsFlushed);
+		ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
+		ScheduleBackgroundWork(
+			[this, BuildId = Oid(BuildId), RawHash = IoHash(RawHash), ContentType, Payload = CompositeBuffer(Payload)]() {
+				ZEN_TRACE_CPU("ZenBuildStorageCache::PutBuildBlob");
+				Stopwatch ExecutionTimer;
+				auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+				HttpClient::Response CacheResponse =
+					m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()),
+										Payload,
+										ContentType);
+				AddStatistic(CacheResponse);
+				if (!CacheResponse.IsSuccess())
+				{
+					ZEN_DEBUG("Failed posting blob to cache: {}", CacheResponse.ErrorMessage(""sv));
+				}
+			});
+	}
+
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
+	{
+		ZEN_TRACE_CPU("ZenBuildStorageCache::GetBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+		HttpClient::KeyValueMap Headers;
+		if (RangeOffset != 0 || RangeBytes != (uint64_t)-1)
+		{
+			Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", RangeOffset, RangeOffset + RangeBytes - 1)});
+		}
+		CreateDirectories(m_TempFolderPath);
+		HttpClient::Response CacheResponse =
+			m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()),
+								  m_TempFolderPath,
+								  Headers);
+		AddStatistic(CacheResponse);
+		if (CacheResponse.IsSuccess())
+		{
+			return CacheResponse.ResponsePayload;
+		}
+		return {};
+	}
+
+	virtual void PutBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes, std::span<const CbObject> MetaDatas) override
+	{
+		ZEN_ASSERT(!IsFlushed);
+		ScheduleBackgroundWork([this,
+								BuildId		  = Oid(BuildId),
+								BlobRawHashes = std::vector<IoHash>(BlobHashes.begin(), BlobHashes.end()),
+								MetaDatas	  = std::vector<CbObject>(MetaDatas.begin(), MetaDatas.end())]() {
+			ZEN_TRACE_CPU("ZenBuildStorageCache::PutBlobMetadatas");
+
+			Stopwatch ExecutionTimer;
+			auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+			const uint64_t BlobCount = BlobRawHashes.size();
+
+			CbPackage							   RequestPackage;
+			std::vector<CbAttachment>			   Attachments;
+			tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes;
+			Attachments.reserve(BlobCount);
+			AttachmentHashes.reserve(BlobCount);
+			{
+				CbObjectWriter RequestWriter;
+				RequestWriter.BeginArray("blobHashes");
+				for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+				{
+					RequestWriter.AddHash(BlobRawHashes[BlockHashIndex]);
+				}
+				RequestWriter.EndArray();  // blobHashes
+
+				RequestWriter.BeginArray("metadatas");
+				for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++)
+				{
+					const IoHash ObjectHash = MetaDatas[BlockHashIndex].GetHash();
+					RequestWriter.AddBinaryAttachment(ObjectHash);
+					if (!AttachmentHashes.contains(ObjectHash))
+					{
+						Attachments.push_back(CbAttachment(MetaDatas[BlockHashIndex], ObjectHash));
+						AttachmentHashes.insert(ObjectHash);
+					}
+				}
+
+				RequestWriter.EndArray();  // metadatas
+
+				RequestPackage.SetObject(RequestWriter.Save());
+			}
+			RequestPackage.AddAttachments(Attachments);
+
+			CompositeBuffer RpcRequestBuffer = FormatPackageMessageBuffer(RequestPackage);
+
+			HttpClient::Response CacheResponse =
+				m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/putBlobMetadata", m_Namespace, m_Bucket, BuildId),
+								  RpcRequestBuffer,
+								  ZenContentType::kCbPackage);
+			AddStatistic(CacheResponse);
+			if (!CacheResponse.IsSuccess())
+			{
+				ZEN_DEBUG("Failed posting blob metadata to cache: {}", CacheResponse.ErrorMessage(""sv));
+			}
+		});
+	}
+
+	virtual std::vector<CbObject> GetBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes) override
+	{
+		ZEN_TRACE_CPU("ZenBuildStorageCache::GetBlobMetadatas");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+		CbObjectWriter Request;
+
+		Request.BeginArray("blobHashes"sv);
+		for (const IoHash& BlobHash : BlobHashes)
+		{
+			Request.AddHash(BlobHash);
+		}
+		Request.EndArray();
+
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+
+		HttpClient::Response Response =
+			m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/getBlobMetadata", m_Namespace, m_Bucket, BuildId),
+							  Payload,
+							  HttpClient::Accept(ZenContentType::kCbObject));
+		AddStatistic(Response);
+		if (Response.IsSuccess())
+		{
+			std::vector<CbObject> Result;
+
+			CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload);
+			CbObject  ResponseObject  = ResponsePackage.GetObject();
+
+			CbArrayView BlobHashArray  = ResponseObject["blobHashes"sv].AsArrayView();
+			CbArrayView MetadatasArray = ResponseObject["metadatas"sv].AsArrayView();
+			Result.reserve(MetadatasArray.Num());
+			auto BlobHashesIt	 = BlobHashes.begin();
+			auto BlobHashArrayIt = begin(BlobHashArray);
+			auto MetadataArrayIt = begin(MetadatasArray);
+			while (MetadataArrayIt != end(MetadatasArray))
+			{
+				const IoHash BlobHash = (*BlobHashArrayIt).AsHash();
+				while (BlobHash != *BlobHashesIt)
+				{
+					ZEN_ASSERT(BlobHashesIt != BlobHashes.end());
+					BlobHashesIt++;
+				}
+
+				ZEN_ASSERT(BlobHash == *BlobHashesIt);
+
+				const IoHash		MetaHash	   = (*MetadataArrayIt).AsAttachment();
+				const CbAttachment* MetaAttachment = ResponsePackage.FindAttachment(MetaHash);
+				ZEN_ASSERT(MetaAttachment);
+
+				CbObject Metadata = MetaAttachment->AsObject();
+				Result.emplace_back(std::move(Metadata));
+
+				BlobHashArrayIt++;
+				MetadataArrayIt++;
+				BlobHashesIt++;
+			}
+			return Result;
+		}
+		return {};
+	}
+
+	virtual std::vector<BlobExistsResult> BlobsExists(const Oid& BuildId, std::span<const IoHash> BlobHashes) override
+	{
+		ZEN_TRACE_CPU("ZenBuildStorageCache::BlobsExists");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+		CbObjectWriter Request;
+
+		Request.BeginArray("blobHashes"sv);
+		for (const IoHash& BlobHash : BlobHashes)
+		{
+			Request.AddHash(BlobHash);
+		}
+		Request.EndArray();
+
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+
+		HttpClient::Response Response = m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/exists", m_Namespace, m_Bucket, BuildId),
+														  Payload,
+														  HttpClient::Accept(ZenContentType::kCbObject));
+		AddStatistic(Response);
+		if (Response.IsSuccess())
+		{
+			CbObject ResponseObject = LoadCompactBinaryObject(Response.ResponsePayload);
+			if (!ResponseObject)
+			{
+				throw std::runtime_error("BlobExists reponse is invalid, failed to load payload as compact binary object");
+			}
+			CbArrayView BlobsExistsArray = ResponseObject["blobExists"sv].AsArrayView();
+			if (!BlobsExistsArray)
+			{
+				throw std::runtime_error("BlobExists reponse is invalid, 'blobExists' array is missing");
+			}
+			if (BlobsExistsArray.Num() != BlobHashes.size())
+			{
+				throw std::runtime_error(fmt::format("BlobExists reponse is invalid, 'blobExists' array contains {} entries, expected {}",
+													 BlobsExistsArray.Num(),
+													 BlobHashes.size()));
+			}
+
+			CbArrayView MetadatasExistsArray = ResponseObject["metadataExists"sv].AsArrayView();
+			if (!MetadatasExistsArray)
+			{
+				throw std::runtime_error("BlobExists reponse is invalid, 'metadataExists' array is missing");
+			}
+			if (MetadatasExistsArray.Num() != BlobHashes.size())
+			{
+				throw std::runtime_error(
+					fmt::format("BlobExists reponse is invalid, 'metadataExists' array contains {} entries, expected {}",
+								MetadatasExistsArray.Num(),
+								BlobHashes.size()));
+			}
+
+			std::vector<BlobExistsResult> Result;
+			Result.reserve(BlobHashes.size());
+			auto BlobExistsIt	  = begin(BlobsExistsArray);
+			auto MetadataExistsIt = begin(MetadatasExistsArray);
+			while (BlobExistsIt != end(BlobsExistsArray))
+			{
+				ZEN_ASSERT(MetadataExistsIt != end(MetadatasExistsArray));
+
+				const bool HasBody	   = (*BlobExistsIt).AsBool();
+				const bool HasMetadata = (*MetadataExistsIt).AsBool();
+
+				Result.push_back({.HasBody = HasBody, .HasMetadata = HasMetadata});
+
+				BlobExistsIt++;
+				MetadataExistsIt++;
+			}
+			return Result;
+		}
+		return {};
+	}
+
+	virtual void Flush(int32_t UpdateIntervalMS, std::function<bool(intptr_t Remaining)>&& UpdateCallback) override
+	{
+		if (IsFlushed)
+		{
+			return;
+		}
+		if (!IsFlushed)
+		{
+			m_PendingBackgroundWorkCount.CountDown();
+			IsFlushed = true;
+		}
+		if (m_PendingBackgroundWorkCount.Wait(100))
+		{
+			return;
+		}
+		while (true)
+		{
+			intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining();
+			if (UpdateCallback(Remaining))
+			{
+				if (m_PendingBackgroundWorkCount.Wait(UpdateIntervalMS))
+				{
+					UpdateCallback(0);
+					return;
+				}
+			}
+			else
+			{
+				m_CancelBackgroundWork.store(true);
+			}
+		}
+	}
+
+private:
+	void AddStatistic(const HttpClient::Response& Result)
+	{
+		m_Stats.TotalBytesWritten += Result.UploadedBytes;
+		m_Stats.TotalBytesRead += Result.DownloadedBytes;
+		m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0);
+		m_Stats.TotalRequestCount++;
+	}
+
+	HttpClient&					   m_HttpClient;
+	BuildStorageCache::Statistics& m_Stats;
+	const std::string			   m_Namespace;
+	const std::string			   m_Bucket;
+	const std::filesystem::path	   m_TempFolderPath;
+	const bool					   m_BoostBackgroundThreadCount;
+	bool						   IsFlushed = false;
+
+	WorkerThreadPool& m_BackgroundWorkPool;
+	Latch			  m_PendingBackgroundWorkCount;
+	std::atomic<bool> m_CancelBackgroundWork;
+};
+
+std::unique_ptr<BuildStorageCache>
+CreateZenBuildStorageCache(HttpClient&					  HttpClient,
+						   BuildStorageCache::Statistics& Stats,
+						   std::string_view				  Namespace,
+						   std::string_view				  Bucket,
+						   const std::filesystem::path&	  TempFolderPath,
+						   bool							  BoostBackgroundThreadCount)
+{
+	return std::make_unique<ZenBuildStorageCache>(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BoostBackgroundThreadCount);
+}
+
+}  // namespace zen
diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp
index 1f951167d..46e80f6b7 100644
--- a/src/zenutil/cache/rpcrecording.cpp
+++ b/src/zenutil/cache/rpcrecording.cpp
@@ -46,7 +46,7 @@ struct RecordedRequestsWriter
 	void BeginWrite(const std::filesystem::path& BasePath)
 	{
 		m_BasePath = BasePath;
-		std::filesystem::create_directories(m_BasePath);
+		CreateDirectories(m_BasePath);
 	}
 
 	void EndWrite()
@@ -366,6 +366,7 @@ private:
 	};
 
 	std::unique_ptr<std::thread> m_WriterThread;
+	std::atomic_bool			 m_IsWriterReady{false};
 	std::atomic_bool			 m_IsActive{false};
 	std::atomic_int64_t			 m_PendingRequests{0};
 	RwLock						 m_RequestQueueLock;
@@ -426,7 +427,7 @@ RecordedRequestsSegmentWriter::BeginWrite(const std::filesystem::path& BasePath,
 	m_BasePath		   = BasePath;
 	m_SegmentIndex	   = SegmentIndex;
 	m_RequestBaseIndex = RequestBaseIndex;
-	std::filesystem::create_directories(m_BasePath);
+	CreateDirectories(m_BasePath);
 }
 
 void
@@ -658,6 +659,8 @@ RecordedRequestsWriter::BeginWrite(const std::filesystem::path& BasePath)
 	m_IsActive = true;
 
 	m_WriterThread.reset(new std::thread(&RecordedRequestsWriter::WriterThreadMain, this));
+
+	m_IsWriterReady.wait(false);
 }
 
 void
@@ -707,6 +710,9 @@ RecordedRequestsWriter::WriterThreadMain()
 	SetCurrentThreadName("rpc_writer");
 	EnsureCurrentSegment();
 
+	m_IsWriterReady.store(true);
+	m_IsWriterReady.notify_all();
+
 	while (m_IsActive)
 	{
 		m_PendingRequests.wait(0);
@@ -1051,14 +1057,14 @@ public:
 
 	static bool IsCompatible(const std::filesystem::path& BasePath)
 	{
-		if (std::filesystem::exists(BasePath / "rpc_recording_info.zcb"))
+		if (IsFile(BasePath / "rpc_recording_info.zcb"))
 		{
 			return true;
 		}
 
 		const std::filesystem::path SegmentZero = BasePath / MakeSegmentPath(0);
 
-		if (std::filesystem::exists(SegmentZero / "rpc_segment_info.zcb") && std::filesystem::exists(SegmentZero / "index.bin"))
+		if (IsFile(SegmentZero / "rpc_segment_info.zcb") && IsFile(SegmentZero / "index.bin"))
 		{
 			// top-level metadata is missing, possibly because of premature exit
 			// on the recording side
diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp
new file mode 100644
index 000000000..abfc0fb63
--- /dev/null
+++ b/src/zenutil/chunkblock.cpp
@@ -0,0 +1,257 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/chunkblock.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+
+#include <vector>
+
+namespace zen {
+
+using namespace std::literals;
+
+ChunkBlockDescription
+ParseChunkBlockDescription(const CbObjectView& BlockObject)
+{
+	ChunkBlockDescription Result;
+	Result.BlockHash = BlockObject["rawHash"sv].AsHash();
+	if (Result.BlockHash != IoHash::Zero)
+	{
+		Result.HeaderSize		= BlockObject["headerSize"sv].AsUInt64();
+		CbArrayView ChunksArray = BlockObject["rawHashes"sv].AsArrayView();
+		Result.ChunkRawHashes.reserve(ChunksArray.Num());
+		for (CbFieldView ChunkView : ChunksArray)
+		{
+			Result.ChunkRawHashes.push_back(ChunkView.AsHash());
+		}
+
+		CbArrayView ChunkRawLengthsArray = BlockObject["chunkRawLengths"sv].AsArrayView();
+		Result.ChunkRawLengths.reserve(ChunkRawLengthsArray.Num());
+		for (CbFieldView ChunkView : ChunkRawLengthsArray)
+		{
+			Result.ChunkRawLengths.push_back(ChunkView.AsUInt32());
+		}
+
+		CbArrayView ChunkCompressedLengthsArray = BlockObject["chunkCompressedLengths"sv].AsArrayView();
+		Result.ChunkCompressedLengths.reserve(ChunkCompressedLengthsArray.Num());
+		for (CbFieldView ChunkView : ChunkCompressedLengthsArray)
+		{
+			Result.ChunkCompressedLengths.push_back(ChunkView.AsUInt32());
+		}
+	}
+	return Result;
+}
+
+std::vector<ChunkBlockDescription>
+ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject)
+{
+	if (!BlocksObject)
+	{
+		return {};
+	}
+	std::vector<ChunkBlockDescription> Result;
+	CbArrayView						   Blocks = BlocksObject["blocks"sv].AsArrayView();
+	Result.reserve(Blocks.Num());
+	for (CbFieldView BlockView : Blocks)
+	{
+		CbObjectView BlockObject = BlockView.AsObjectView();
+		Result.emplace_back(ParseChunkBlockDescription(BlockObject));
+	}
+	return Result;
+}
+
+CbObject
+BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData)
+{
+	ZEN_ASSERT(Block.BlockHash != IoHash::Zero);
+	ZEN_ASSERT(Block.HeaderSize > 0);
+	ZEN_ASSERT(Block.ChunkRawLengths.size() == Block.ChunkRawHashes.size());
+	ZEN_ASSERT(Block.ChunkCompressedLengths.size() == Block.ChunkRawHashes.size());
+
+	CbObjectWriter Writer;
+	Writer.AddHash("rawHash"sv, Block.BlockHash);
+	Writer.AddInteger("headerSize"sv, Block.HeaderSize);
+	Writer.BeginArray("rawHashes"sv);
+	{
+		for (const IoHash& ChunkHash : Block.ChunkRawHashes)
+		{
+			Writer.AddHash(ChunkHash);
+		}
+	}
+	Writer.EndArray();
+
+	Writer.BeginArray("chunkRawLengths");
+	{
+		for (uint32_t ChunkSize : Block.ChunkRawLengths)
+		{
+			Writer.AddInteger(ChunkSize);
+		}
+	}
+	Writer.EndArray();
+
+	Writer.BeginArray("chunkCompressedLengths");
+	{
+		for (uint32_t ChunkSize : Block.ChunkCompressedLengths)
+		{
+			Writer.AddInteger(ChunkSize);
+		}
+	}
+	Writer.EndArray();
+
+	Writer.AddObject("metadata", MetaData);
+
+	return Writer.Save();
+}
+
+ChunkBlockDescription
+GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash)
+{
+	ChunkBlockDescription BlockDescription = {{.BlockHash = IoHash::HashBuffer(BlockPayload)}};
+	if (BlockDescription.BlockHash != RawHash)
+	{
+		throw std::runtime_error(fmt::format("Block {} content hash {} does not match block hash", RawHash, BlockDescription.BlockHash));
+	}
+	if (IterateChunkBlock(
+			BlockPayload,
+			[&BlockDescription, RawHash](CompressedBuffer&& Chunk, const IoHash& AttachmentHash) {
+				if (CompositeBuffer Decompressed = Chunk.DecompressToComposite(); Decompressed)
+				{
+					IoHash ChunkHash = IoHash::HashBuffer(Decompressed.Flatten());
+					if (ChunkHash != AttachmentHash)
+					{
+						throw std::runtime_error(
+							fmt::format("Chunk {} in block {} content hash {} does not match chunk", AttachmentHash, RawHash, ChunkHash));
+					}
+					BlockDescription.ChunkRawHashes.push_back(AttachmentHash);
+					BlockDescription.ChunkRawLengths.push_back(gsl::narrow<uint32_t>(Decompressed.GetSize()));
+					BlockDescription.ChunkCompressedLengths.push_back(gsl::narrow<uint32_t>(Chunk.GetCompressedSize()));
+				}
+				else
+				{
+					throw std::runtime_error(fmt::format("Chunk {} in block {} is not a compressed buffer", AttachmentHash, RawHash));
+				}
+			},
+			BlockDescription.HeaderSize))
+	{
+		return BlockDescription;
+	}
+	else
+	{
+		throw std::runtime_error(fmt::format("Block {} is malformed", RawHash));
+	}
+}
+
+CompressedBuffer
+GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock)
+{
+	const size_t ChunkCount = FetchChunks.size();
+
+	std::vector<SharedBuffer> ChunkSegments;
+	ChunkSegments.resize(1);
+	ChunkSegments.reserve(1 + ChunkCount);
+	OutBlock.ChunkRawHashes.reserve(ChunkCount);
+	OutBlock.ChunkRawLengths.reserve(ChunkCount);
+	OutBlock.ChunkCompressedLengths.reserve(ChunkCount);
+	{
+		IoBuffer		  TempBuffer(ChunkCount * 9);
+		MutableMemoryView View			 = TempBuffer.GetMutableView();
+		uint8_t*		  BufferStartPtr = reinterpret_cast<uint8_t*>(View.GetData());
+		uint8_t*		  BufferEndPtr	 = BufferStartPtr;
+		BufferEndPtr += WriteVarUInt(gsl::narrow<uint64_t>(ChunkCount), BufferEndPtr);
+		for (const auto& It : FetchChunks)
+		{
+			std::pair<uint64_t, CompressedBuffer> Chunk		= It.second(It.first);
+			uint64_t							  ChunkSize = 0;
+			std::span<const SharedBuffer>		  Segments	= Chunk.second.GetCompressed().GetSegments();
+			for (const SharedBuffer& Segment : Segments)
+			{
+				ZEN_ASSERT(Segment.IsOwned());
+				ChunkSize += Segment.GetSize();
+				ChunkSegments.push_back(Segment);
+			}
+			BufferEndPtr += WriteVarUInt(ChunkSize, BufferEndPtr);
+			OutBlock.ChunkRawHashes.push_back(It.first);
+			OutBlock.ChunkRawLengths.push_back(gsl::narrow<uint32_t>(Chunk.first));
+			OutBlock.ChunkCompressedLengths.push_back(gsl::narrow<uint32_t>(ChunkSize));
+		}
+		ZEN_ASSERT(BufferEndPtr <= View.GetDataEnd());
+		ptrdiff_t TempBufferLength = std::distance(BufferStartPtr, BufferEndPtr);
+		ChunkSegments[0]		   = SharedBuffer(IoBuffer(TempBuffer, 0, gsl::narrow<size_t>(TempBufferLength)));
+		OutBlock.HeaderSize		   = TempBufferLength;
+	}
+	CompressedBuffer CompressedBlock =
+		CompressedBuffer::Compress(CompositeBuffer(std::move(ChunkSegments)), OodleCompressor::Mermaid, OodleCompressionLevel::None);
+	OutBlock.BlockHash = CompressedBlock.DecodeRawHash();
+	return CompressedBlock;
+}
+
+std::vector<uint32_t>
+ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize)
+{
+	const uint8_t* ReadPtr = reinterpret_cast<const uint8_t*>(BlockView.GetData());
+	uint32_t	   NumberSize;
+	uint64_t	   ChunkCount = ReadVarUInt(ReadPtr, NumberSize);
+	ReadPtr += NumberSize;
+	std::vector<uint32_t> ChunkSizes;
+	ChunkSizes.reserve(ChunkCount);
+	while (ChunkCount--)
+	{
+		if (ReadPtr >= BlockView.GetDataEnd())
+		{
+			throw std::runtime_error("Invalid block header, block data ended unexpectedly");
+		}
+		uint64_t ChunkSize = ReadVarUInt(ReadPtr, NumberSize);
+		if (ChunkSize > std::numeric_limits<uint32_t>::max())
+		{
+			throw std::runtime_error("Invalid block header, header data is corrupt");
+		}
+		if (ChunkSize < 1)
+		{
+			throw std::runtime_error("Invalid block header, header data is corrupt");
+		}
+		ChunkSizes.push_back(gsl::narrow<uint32_t>(ChunkSize));
+		ReadPtr += NumberSize;
+	}
+	uint64_t Offset = std::distance((const uint8_t*)BlockView.GetData(), ReadPtr);
+	OutHeaderSize	= Offset;
+	return ChunkSizes;
+}
+
+bool
+IterateChunkBlock(const SharedBuffer&														  BlockPayload,
+				  std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
+				  uint64_t&																	  OutHeaderSize)
+{
+	ZEN_ASSERT(BlockPayload);
+	if (BlockPayload.GetSize() < 1)
+	{
+		return false;
+	}
+
+	MemoryView BlockView = BlockPayload.GetView();
+
+	std::vector<uint32_t> ChunkSizes = ReadChunkBlockHeader(BlockView, OutHeaderSize);
+	uint64_t			  Offset	 = OutHeaderSize;
+	OutHeaderSize					 = Offset;
+	for (uint64_t ChunkSize : ChunkSizes)
+	{
+		IoBuffer		 Chunk(BlockPayload.AsIoBuffer(), Offset, ChunkSize);
+		IoHash			 AttachmentRawHash;
+		uint64_t		 AttachmentRawSize;
+		CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), AttachmentRawHash, AttachmentRawSize);
+		ZEN_ASSERT_SLOW(IoHash::HashBuffer(CompressedChunk.DecompressToComposite()) == AttachmentRawHash);
+		if (!CompressedChunk)
+		{
+			ZEN_ERROR("Invalid chunk in block");
+			return false;
+		}
+		Visitor(std::move(CompressedChunk), AttachmentRawHash);
+		Offset += ChunkSize;
+		ZEN_ASSERT(Offset <= BlockView.GetSize());
+	}
+	return true;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
new file mode 100644
index 000000000..cd1bf7dd7
--- /dev/null
+++ b/src/zenutil/chunkedcontent.cpp
@@ -0,0 +1,951 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/chunkedcontent.h>
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+
+#include <zenutil/chunkedfile.h>
+#include <zenutil/chunkingcontroller.h>
+#include <zenutil/parallelwork.h>
+#include <zenutil/workerpools.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+namespace {
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  std::span<const uint32_t>							ChunkSequence,
+						  std::span<const IoHash>							ChunkHashes,
+						  std::span<const uint64_t>							ChunkRawSizes)
+	{
+		ZEN_ASSERT(ChunkHashes.size() == ChunkRawSizes.size());
+		InOutChunkedContent.ChunkCounts.push_back(gsl::narrow<uint32_t>(ChunkSequence.size()));
+		InOutChunkedContent.ChunkOrders.reserve(InOutChunkedContent.ChunkOrders.size() + ChunkSequence.size());
+
+		for (uint32_t ChunkedSequenceIndex : ChunkSequence)
+		{
+			const IoHash& ChunkHash = ChunkHashes[ChunkedSequenceIndex];
+			if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end())
+			{
+				uint32_t ChunkIndex = gsl::narrow<uint32_t>(It->second);
+				InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+			}
+			else
+			{
+				uint32_t ChunkIndex = gsl::narrow<uint32_t>(InOutChunkedContent.ChunkHashes.size());
+				ChunkHashToChunkIndex.insert_or_assign(ChunkHash, ChunkIndex);
+				InOutChunkedContent.ChunkHashes.push_back(ChunkHash);
+				InOutChunkedContent.ChunkRawSizes.push_back(ChunkRawSizes[ChunkedSequenceIndex]);
+				InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+				Stats.UniqueChunksFound++;
+				Stats.UniqueBytesFound += ChunkRawSizes[ChunkedSequenceIndex];
+			}
+		}
+		InOutChunkedContent.SequenceRawHashes.push_back(RawHash);
+		Stats.UniqueSequencesFound++;
+	}
+
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  const uint64_t									RawSize)
+	{
+		InOutChunkedContent.ChunkCounts.push_back(1);
+
+		if (auto It = ChunkHashToChunkIndex.find(RawHash); It != ChunkHashToChunkIndex.end())
+		{
+			uint32_t ChunkIndex = gsl::narrow<uint32_t>(It->second);
+			InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+		}
+		else
+		{
+			uint32_t ChunkIndex = gsl::narrow<uint32_t>(InOutChunkedContent.ChunkHashes.size());
+			ChunkHashToChunkIndex.insert_or_assign(RawHash, ChunkIndex);
+			InOutChunkedContent.ChunkHashes.push_back(RawHash);
+			InOutChunkedContent.ChunkRawSizes.push_back(RawSize);
+			InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+			Stats.UniqueChunksFound++;
+			Stats.UniqueBytesFound += RawSize;
+		}
+		InOutChunkedContent.SequenceRawHashes.push_back(RawHash);
+		Stats.UniqueSequencesFound++;
+	}
+
+	IoHash HashOneFile(ChunkingStatistics&								 Stats,
+					   const ChunkingController&						 InChunkingController,
+					   ChunkedFolderContent&							 OutChunkedContent,
+					   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+					   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& RawHashToSequenceRawHashIndex,
+					   RwLock&											 Lock,
+					   const std::filesystem::path&						 FolderPath,
+					   uint32_t											 PathIndex,
+					   std::atomic<bool>&								 AbortFlag)
+	{
+		ZEN_TRACE_CPU("ChunkFolderContent");
+
+		const uint64_t				 RawSize = OutChunkedContent.RawSizes[PathIndex];
+		const std::filesystem::path& Path	 = OutChunkedContent.Paths[PathIndex];
+
+		if (RawSize == 0)
+		{
+			return IoHash::Zero;
+		}
+		else
+		{
+			ChunkedInfoWithSource Chunked;
+			const bool			  DidChunking =
+				InChunkingController.ProcessFile((FolderPath / Path).make_preferred(), RawSize, Chunked, Stats.BytesHashed, AbortFlag);
+			if (DidChunking)
+			{
+				Lock.WithExclusiveLock([&]() {
+					if (!RawHashToSequenceRawHashIndex.contains(Chunked.Info.RawHash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{Chunked.Info.RawHash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())});
+						std::vector<uint64_t> ChunkSizes;
+						ChunkSizes.reserve(Chunked.ChunkSources.size());
+						for (const ChunkSource& Source : Chunked.ChunkSources)
+						{
+							ChunkSizes.push_back(Source.Size);
+						}
+						AddChunkSequence(Stats,
+										 OutChunkedContent.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 Chunked.Info.RawHash,
+										 Chunked.Info.ChunkSequence,
+										 Chunked.Info.ChunkHashes,
+										 ChunkSizes);
+						Stats.UniqueSequencesFound++;
+					}
+				});
+				Stats.FilesChunked++;
+				return Chunked.Info.RawHash;
+			}
+			else
+			{
+				ZEN_TRACE_CPU("HashOnly");
+
+				IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred());
+				if (Buffer.GetSize() != RawSize)
+				{
+					throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path));
+				}
+				const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed);
+
+				Lock.WithExclusiveLock([&]() {
+					if (!RawHashToSequenceRawHashIndex.contains(Hash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{Hash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())});
+						AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
+						Stats.UniqueSequencesFound++;
+					}
+				});
+				return Hash;
+			}
+		}
+	}
+
+	std::string PathCompareString(const std::filesystem::path& Path) { return ToLower(Path.generic_string()); }
+
+}  // namespace
+
+std::string_view FolderContentSourcePlatformNames[(size_t)SourcePlatform::_Count] = {"Windows"sv, "Linux"sv, "MacOS"sv};
+
+std::string_view
+ToString(SourcePlatform Platform)
+{
+	return FolderContentSourcePlatformNames[(size_t)Platform];
+}
+
+SourcePlatform
+FromString(std::string_view Platform, SourcePlatform Default)
+{
+	for (size_t Index = 0; Index < (size_t)SourcePlatform::_Count; Index++)
+	{
+		if (Platform == FolderContentSourcePlatformNames[Index])
+		{
+			return (SourcePlatform)Index;
+		}
+	}
+	return Default;
+}
+
+SourcePlatform
+GetSourceCurrentPlatform()
+{
+#if ZEN_PLATFORM_WINDOWS
+	return SourcePlatform::Windows;
+#endif
+#if ZEN_PLATFORM_MAC
+	return SourcePlatform::MacOS;
+#endif
+#if ZEN_PLATFORM_LINUX
+	return SourcePlatform::Linux;
+#endif
+}
+
+bool
+FolderContent::AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return (Lhs & 0xff) == (Rhs & 0xff);
+#endif
+#if ZEN_PLATFORM_MAC
+	return Lhs == Rhs;
+#endif
+#if ZEN_PLATFORM_LINUX
+	return Lhs == Rhs;
+#endif
+}
+
+bool
+FolderContent::operator==(const FolderContent& Rhs) const
+{
+	if ((Platform == Rhs.Platform) && (RawSizes == Rhs.RawSizes) && (Attributes == Rhs.Attributes) &&
+		(ModificationTicks == Rhs.ModificationTicks) && (Paths.size() == Rhs.Paths.size()))
+	{
+		size_t PathCount = 0;
+		for (size_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
+		{
+			if (Paths[PathIndex].generic_string() != Rhs.Paths[PathIndex].generic_string())
+			{
+				return false;
+			}
+		}
+		return true;
+	}
+	return false;
+}
+
+bool
+FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const
+{
+	ZEN_TRACE_CPU("FolderContent::AreKnownFilesEqual");
+	tsl::robin_map<std::string, size_t> RhsPathToIndex;
+	const size_t						RhsPathCount = Rhs.Paths.size();
+	RhsPathToIndex.reserve(RhsPathCount);
+	for (size_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++)
+	{
+		RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex});
+	}
+	const size_t PathCount = Paths.size();
+	for (size_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
+	{
+		if (auto It = RhsPathToIndex.find(Paths[PathIndex].generic_string()); It != RhsPathToIndex.end())
+		{
+			const size_t RhsPathIndex = It->second;
+			if ((RawSizes[PathIndex] != Rhs.RawSizes[RhsPathIndex]) ||
+				(!AreFileAttributesEqual(Attributes[PathIndex], Rhs.Attributes[RhsPathIndex])) ||
+				(ModificationTicks[PathIndex] != Rhs.ModificationTicks[RhsPathIndex]))
+			{
+				return false;
+			}
+		}
+		else
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
+void
+FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutPathIndexesOufOfDate)
+{
+	ZEN_TRACE_CPU("FolderContent::UpdateState");
+	tsl::robin_map<std::string, uint32_t> RhsPathToIndex;
+	const uint32_t						  RhsPathCount = gsl::narrow<uint32_t>(Rhs.Paths.size());
+	RhsPathToIndex.reserve(RhsPathCount);
+	for (uint32_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++)
+	{
+		RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex});
+	}
+	uint32_t PathCount = gsl::narrow<uint32_t>(Paths.size());
+	for (uint32_t PathIndex = 0; PathIndex < PathCount;)
+	{
+		if (auto It = RhsPathToIndex.find(Paths[PathIndex].generic_string()); It != RhsPathToIndex.end())
+		{
+			const uint32_t RhsPathIndex = It->second;
+
+			if ((RawSizes[PathIndex] != Rhs.RawSizes[RhsPathIndex]) ||
+				(ModificationTicks[PathIndex] != Rhs.ModificationTicks[RhsPathIndex]))
+			{
+				RawSizes[PathIndex]			 = Rhs.RawSizes[RhsPathIndex];
+				ModificationTicks[PathIndex] = Rhs.ModificationTicks[RhsPathIndex];
+				OutPathIndexesOufOfDate.push_back(PathIndex);
+			}
+			Attributes[PathIndex] = Rhs.Attributes[RhsPathIndex];
+			PathIndex++;
+		}
+		else
+		{
+			Paths.erase(Paths.begin() + PathIndex);
+			RawSizes.erase(RawSizes.begin() + PathIndex);
+			Attributes.erase(Attributes.begin() + PathIndex);
+			ModificationTicks.erase(ModificationTicks.begin() + PathIndex);
+			PathCount--;
+		}
+	}
+}
+
+FolderContent
+GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPaths)
+{
+	ZEN_TRACE_CPU("FolderContent::GetUpdatedContent");
+
+	const uint32_t NewPathCount = gsl::narrow<uint32_t>(New.Paths.size());
+
+	FolderContent Result = {.Platform = Old.Platform};
+	Result.Paths.reserve(NewPathCount);
+	Result.RawSizes.reserve(NewPathCount);
+	Result.Attributes.reserve(NewPathCount);
+	Result.ModificationTicks.reserve(NewPathCount);
+
+	tsl::robin_map<std::string, uint32_t> NewPathToIndex;
+	NewPathToIndex.reserve(NewPathCount);
+	for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++)
+	{
+		NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex});
+	}
+
+	uint32_t OldPathCount = gsl::narrow<uint32_t>(Old.Paths.size());
+	for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++)
+	{
+		if (auto It = NewPathToIndex.find(Old.Paths[OldPathIndex].generic_string()); It != NewPathToIndex.end())
+		{
+			const uint32_t NewPathIndex = It->second;
+
+			if ((Old.RawSizes[OldPathIndex] != New.RawSizes[NewPathIndex]) ||
+				(Old.ModificationTicks[OldPathIndex] != New.ModificationTicks[NewPathIndex]))
+			{
+				Result.Paths.push_back(New.Paths[NewPathIndex]);
+				Result.RawSizes.push_back(New.RawSizes[NewPathIndex]);
+				Result.Attributes.push_back(New.Attributes[NewPathIndex]);
+				Result.ModificationTicks.push_back(New.ModificationTicks[NewPathIndex]);
+			}
+		}
+		else
+		{
+			OutDeletedPaths.push_back(Old.Paths[OldPathIndex]);
+		}
+	}
+	return Result;
+}
+
+void
+SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output)
+{
+	ZEN_TRACE_CPU("SaveFolderContentToCompactBinary");
+	Output.AddString("platform"sv, ToString(Content.Platform));
+	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
+	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.Attributes, "attributes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ModificationTicks, "modificationTimes"sv, Output);
+}
+
+FolderContent
+LoadFolderContentToCompactBinary(CbObjectView Input)
+{
+	ZEN_TRACE_CPU("LoadFolderContentToCompactBinary");
+	FolderContent Content;
+	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
+	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
+	compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes);
+	compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes);
+	compactbinary_helpers::ReadArray("modificationTimes"sv, Input, Content.ModificationTicks);
+	return Content;
+}
+
+FolderContent
+GetFolderContent(GetFolderContentStatistics&															  Stats,
+				 const std::filesystem::path&															  RootPath,
+				 std::function<bool(const std::string_view& RelativePath)>&&							  AcceptDirectory,
+				 std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile,
+				 WorkerThreadPool&																		  WorkerPool,
+				 int32_t																				  UpdateIntervalMS,
+				 std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						  UpdateCallback,
+				 std::atomic<bool>&																		  AbortFlag)
+{
+	ZEN_TRACE_CPU("GetFolderContent");
+
+	Stopwatch Timer;
+	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
+
+	FolderContent Content;
+	struct AsyncVisitor : public GetDirectoryContentVisitor
+	{
+		AsyncVisitor(GetFolderContentStatistics&															  Stats,
+					 std::atomic<bool>&																		  AbortFlag,
+					 FolderContent&																			  Content,
+					 std::function<bool(const std::string_view& RelativePath)>&&							  AcceptDirectory,
+					 std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile)
+		: m_Stats(Stats)
+		, m_AbortFlag(AbortFlag)
+		, m_FoundContent(Content)
+		, m_AcceptDirectory(std::move(AcceptDirectory))
+		, m_AcceptFile(std::move(AcceptFile))
+		{
+		}
+		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+		{
+			if (!m_AbortFlag)
+			{
+				m_Stats.FoundFileCount += Content.FileNames.size();
+				for (uint64_t FileSize : Content.FileSizes)
+				{
+					m_Stats.FoundFileByteCount += FileSize;
+				}
+				std::string RelativeDirectoryPath = RelativeRoot.generic_string();
+				if (m_AcceptDirectory(RelativeDirectoryPath))
+				{
+					std::vector<std::filesystem::path> Paths;
+					std::vector<uint64_t>			   RawSizes;
+					std::vector<uint32_t>			   Attributes;
+					std::vector<uint64_t>			   ModificatonTicks;
+					Paths.reserve(Content.FileNames.size());
+					RawSizes.reserve(Content.FileNames.size());
+					Attributes.reserve(Content.FileNames.size());
+					ModificatonTicks.reserve(Content.FileModificationTicks.size());
+
+					for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+					{
+						const std::filesystem::path& FileName	  = Content.FileNames[FileIndex];
+						std::string					 RelativePath = (RelativeRoot / FileName).generic_string();
+						std::replace(RelativePath.begin(), RelativePath.end(), '\\', '/');
+						if (m_AcceptFile(RelativePath, Content.FileSizes[FileIndex], Content.FileAttributes[FileIndex]))
+						{
+							Paths.emplace_back(std::move(RelativePath));
+							RawSizes.emplace_back(Content.FileSizes[FileIndex]);
+							Attributes.emplace_back(Content.FileAttributes[FileIndex]);
+							ModificatonTicks.emplace_back(Content.FileModificationTicks[FileIndex]);
+
+							m_Stats.AcceptedFileCount++;
+							m_Stats.AcceptedFileByteCount += Content.FileSizes[FileIndex];
+						}
+					}
+					m_Lock.WithExclusiveLock([&]() {
+						m_FoundContent.Paths.insert(m_FoundContent.Paths.end(), Paths.begin(), Paths.end());
+						m_FoundContent.RawSizes.insert(m_FoundContent.RawSizes.end(), RawSizes.begin(), RawSizes.end());
+						m_FoundContent.Attributes.insert(m_FoundContent.Attributes.end(), Attributes.begin(), Attributes.end());
+						m_FoundContent.ModificationTicks.insert(m_FoundContent.ModificationTicks.end(),
+																ModificatonTicks.begin(),
+																ModificatonTicks.end());
+					});
+				}
+			}
+		}
+
+		GetFolderContentStatistics&															   m_Stats;
+		std::atomic<bool>&																	   m_AbortFlag;
+		RwLock																				   m_Lock;
+		FolderContent&																		   m_FoundContent;
+		std::function<bool(const std::string_view& RelativePath)>							   m_AcceptDirectory;
+		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)> m_AcceptFile;
+	} Visitor(Stats, AbortFlag, Content, std::move(AcceptDirectory), std::move(AcceptFile));
+
+	Latch PendingWork(1);
+	GetDirectoryContent(RootPath,
+						DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes |
+							DirectoryContentFlags::IncludeAttributes | DirectoryContentFlags::IncludeModificationTick,
+						Visitor,
+						WorkerPool,
+						PendingWork);
+	PendingWork.CountDown();
+	while (!PendingWork.Wait(UpdateIntervalMS))
+	{
+		UpdateCallback(AbortFlag.load(), PendingWork.Remaining());
+	}
+	std::vector<size_t> Order;
+	size_t				PathCount = Content.Paths.size();
+	Order.resize(Content.Paths.size());
+	std::vector<std::string> Parents;
+	Parents.reserve(PathCount);
+	std::vector<std::string> Filenames;
+	Filenames.reserve(PathCount);
+	for (size_t OrderIndex = 0; OrderIndex < PathCount; OrderIndex++)
+	{
+		Order[OrderIndex] = OrderIndex;
+		Parents.emplace_back(Content.Paths[OrderIndex].parent_path().generic_string());
+		Filenames.emplace_back(Content.Paths[OrderIndex].filename().generic_string());
+	}
+	std::sort(Order.begin(), Order.end(), [&Parents, &Filenames](size_t Lhs, size_t Rhs) {
+		const std::string& LhsParent = Parents[Lhs];
+		const std::string& RhsParent = Parents[Rhs];
+		if (LhsParent < RhsParent)
+		{
+			return true;
+		}
+		else if (LhsParent > RhsParent)
+		{
+			return false;
+		}
+		return Filenames[Lhs] < Filenames[Rhs];
+	});
+	FolderContent OrderedContent;
+	OrderedContent.Paths.reserve(PathCount);
+	OrderedContent.RawSizes.reserve(PathCount);
+	OrderedContent.Attributes.reserve(PathCount);
+	OrderedContent.ModificationTicks.reserve(PathCount);
+	for (size_t OrderIndex : Order)
+	{
+		OrderedContent.Paths.emplace_back(std::move(Content.Paths[OrderIndex]));
+		OrderedContent.RawSizes.emplace_back(Content.RawSizes[OrderIndex]);
+		OrderedContent.Attributes.emplace_back(Content.Attributes[OrderIndex]);
+		OrderedContent.ModificationTicks.emplace_back(Content.ModificationTicks[OrderIndex]);
+	}
+	return OrderedContent;
+}
+
+void
+SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output)
+{
+	ZEN_TRACE_CPU("SaveChunkedFolderContentToCompactBinary");
+	Output.AddString("platform"sv, ToString(Content.Platform));
+	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
+	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.Attributes, "attributes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.RawHashes, "rawHashes"sv, Output);
+
+	Output.BeginObject("chunkedContent");
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.SequenceRawHashes, "sequenceRawHashes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkCounts, "chunkCounts"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkOrders, "chunkOrders"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkHashes, "chunkHashes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkRawSizes, "chunkRawSizes"sv, Output);
+	Output.EndObject();	 // chunkedContent
+}
+
+ChunkedFolderContent
+LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
+{
+	ZEN_TRACE_CPU("LoadChunkedFolderContentToCompactBinary");
+	ChunkedFolderContent Content;
+	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
+	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
+	compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes);
+	compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes);
+	compactbinary_helpers::ReadArray("rawHashes"sv, Input, Content.RawHashes);
+
+	CbObjectView ChunkedContentView = Input["chunkedContent"sv].AsObjectView();
+	compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkedContentView, Content.ChunkedContent.SequenceRawHashes);
+	compactbinary_helpers::ReadArray("chunkCounts"sv, ChunkedContentView, Content.ChunkedContent.ChunkCounts);
+	compactbinary_helpers::ReadArray("chunkOrders"sv, ChunkedContentView, Content.ChunkedContent.ChunkOrders);
+	compactbinary_helpers::ReadArray("chunkHashes"sv, ChunkedContentView, Content.ChunkedContent.ChunkHashes);
+	compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkedContentView, Content.ChunkedContent.ChunkRawSizes);
+	return Content;
+}
+
+ChunkedFolderContent
+MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays)
+{
+	ZEN_TRACE_CPU("MergeChunkedFolderContents");
+
+	ZEN_ASSERT(!Overlays.empty());
+
+	ChunkedFolderContent Result;
+	const size_t		 BasePathCount = Base.Paths.size();
+	Result.Paths.reserve(BasePathCount);
+	Result.RawSizes.reserve(BasePathCount);
+	Result.Attributes.reserve(BasePathCount);
+	Result.RawHashes.reserve(BasePathCount);
+
+	const size_t BaseChunkCount = Base.ChunkedContent.ChunkHashes.size();
+	Result.ChunkedContent.SequenceRawHashes.reserve(Base.ChunkedContent.SequenceRawHashes.size());
+	Result.ChunkedContent.ChunkCounts.reserve(BaseChunkCount);
+	Result.ChunkedContent.ChunkHashes.reserve(BaseChunkCount);
+	Result.ChunkedContent.ChunkRawSizes.reserve(BaseChunkCount);
+	Result.ChunkedContent.ChunkOrders.reserve(Base.ChunkedContent.ChunkOrders.size());
+
+	tsl::robin_map<std::string, std::filesystem::path> GenericPathToActualPath;
+	for (const std::filesystem::path& Path : Base.Paths)
+	{
+		GenericPathToActualPath.insert({PathCompareString(Path), Path});
+	}
+	for (const ChunkedFolderContent& Overlay : Overlays)
+	{
+		for (const std::filesystem::path& Path : Overlay.Paths)
+		{
+			GenericPathToActualPath.insert({PathCompareString(Path), Path});
+		}
+	}
+
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
+
+	auto BuildOverlayPaths = [](std::span<const ChunkedFolderContent> Overlays) -> tsl::robin_set<std::string> {
+		tsl::robin_set<std::string> Result;
+		for (const ChunkedFolderContent& OverlayContent : Overlays)
+		{
+			for (const std::filesystem::path& Path : OverlayContent.Paths)
+			{
+				Result.insert(PathCompareString(Path));
+			}
+		}
+		return Result;
+	};
+
+	auto AddContent = [&BuildOverlayPaths](ChunkedFolderContent&									 Result,
+										   const ChunkedFolderContent&								 OverlayContent,
+										   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>&		 ChunkHashToChunkIndex,
+										   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>&		 RawHashToSequenceRawHashIndex,
+										   const tsl::robin_map<std::string, std::filesystem::path>& GenericPathToActualPath,
+										   std::span<const ChunkedFolderContent>					 Overlays) {
+		const ChunkedContentLookup	OverlayLookup	 = BuildChunkedContentLookup(OverlayContent);
+		tsl::robin_set<std::string> BaseOverlayPaths = BuildOverlayPaths(Overlays);
+		for (uint32_t PathIndex = 0; PathIndex < OverlayContent.Paths.size(); PathIndex++)
+		{
+			std::string GenericPath = PathCompareString(OverlayContent.Paths[PathIndex]);
+			if (!BaseOverlayPaths.contains(GenericPath))
+			{
+				// This asset will not be overridden by a later layer - add it
+
+				const std::filesystem::path OriginalPath = GenericPathToActualPath.at(GenericPath);
+				Result.Paths.push_back(OriginalPath);
+				const IoHash& RawHash = OverlayContent.RawHashes[PathIndex];
+				Result.RawSizes.push_back(OverlayContent.RawSizes[PathIndex]);
+				Result.Attributes.push_back(OverlayContent.Attributes[PathIndex]);
+				Result.RawHashes.push_back(RawHash);
+
+				if (OverlayContent.RawSizes[PathIndex] > 0)
+				{
+					if (!RawHashToSequenceRawHashIndex.contains(RawHash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{RawHash, gsl::narrow<uint32_t>(Result.ChunkedContent.SequenceRawHashes.size())});
+						const uint32_t			  SequenceRawHashIndex = OverlayLookup.RawHashToSequenceIndex.at(RawHash);
+						const uint32_t			  OrderIndexOffset	   = OverlayLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex];
+						const uint32_t			  ChunkCount		   = OverlayContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
+						ChunkingStatistics		  Stats;
+						std::span<const uint32_t> OriginalChunkOrder =
+							std::span<const uint32_t>(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 OverlayContent.ChunkedContent.ChunkHashes,
+										 OverlayContent.ChunkedContent.ChunkRawSizes);
+						Stats.UniqueSequencesFound++;
+					}
+				}
+			}
+		}
+	};
+
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> MergedChunkHashToChunkIndex;
+	AddContent(Result, Base, MergedChunkHashToChunkIndex, RawHashToSequenceRawHashIndex, GenericPathToActualPath, Overlays);
+	for (uint32_t OverlayIndex = 0; OverlayIndex < Overlays.size(); OverlayIndex++)
+	{
+		AddContent(Result,
+				   Overlays[OverlayIndex],
+				   MergedChunkHashToChunkIndex,
+				   RawHashToSequenceRawHashIndex,
+				   GenericPathToActualPath,
+				   Overlays.subspan(OverlayIndex + 1));
+	}
+	return Result;
+}
+
+ChunkedFolderContent
+DeletePathsFromChunkedContent(const ChunkedFolderContent&			 BaseContent,
+							  const ChunkedContentLookup&			 BaseContentLookup,
+							  std::span<const std::filesystem::path> DeletedPaths)
+{
+	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
+
+	ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size());
+	ChunkedFolderContent Result = {.Platform = BaseContent.Platform};
+	if (DeletedPaths.size() < BaseContent.Paths.size())
+	{
+		tsl::robin_set<std::string> DeletedPathSet;
+		DeletedPathSet.reserve(DeletedPaths.size());
+		for (const std::filesystem::path& DeletedPath : DeletedPaths)
+		{
+			DeletedPathSet.insert(PathCompareString(DeletedPath));
+		}
+
+		const size_t		  BaseChunkCount = BaseContent.ChunkedContent.ChunkHashes.size();
+		std::vector<uint32_t> NewChunkIndexes(BaseChunkCount, (uint32_t)-1);
+
+		const size_t ExpectedPathCount = BaseContent.Paths.size() - DeletedPaths.size();
+		Result.Paths.reserve(ExpectedPathCount);
+		Result.RawSizes.reserve(ExpectedPathCount);
+		Result.Attributes.reserve(ExpectedPathCount);
+		Result.RawHashes.reserve(ExpectedPathCount);
+
+		Result.ChunkedContent.ChunkHashes.reserve(BaseChunkCount);
+		Result.ChunkedContent.ChunkRawSizes.reserve(BaseChunkCount);
+
+		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
+		for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++)
+		{
+			const std::filesystem::path& Path = BaseContent.Paths[PathIndex];
+			if (!DeletedPathSet.contains(PathCompareString(Path)))
+			{
+				const IoHash&  RawHash = BaseContent.RawHashes[PathIndex];
+				const uint64_t RawSize = BaseContent.RawSizes[PathIndex];
+				Result.Paths.push_back(Path);
+				Result.RawSizes.push_back(RawSize);
+				Result.Attributes.push_back(BaseContent.Attributes[PathIndex]);
+				Result.RawHashes.push_back(RawHash);
+				if (RawSize > 0)
+				{
+					if (!RawHashToSequenceRawHashIndex.contains(RawHash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{RawHash, gsl::narrow<uint32_t>(Result.ChunkedContent.SequenceRawHashes.size())});
+						const uint32_t SequenceRawHashIndex = BaseContentLookup.RawHashToSequenceIndex.at(RawHash);
+						const uint32_t OrderIndexOffset		= BaseContentLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex];
+						const uint32_t ChunkCount			= BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
+
+						std::span<const uint32_t> OriginalChunkOrder =
+							std::span<const uint32_t>(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
+
+						Result.ChunkedContent.ChunkCounts.push_back(gsl::narrow<uint32_t>(OriginalChunkOrder.size()));
+
+						for (uint32_t OldChunkIndex : OriginalChunkOrder)
+						{
+							if (uint32_t FoundChunkIndex = NewChunkIndexes[OldChunkIndex]; FoundChunkIndex != (uint32_t)-1)
+							{
+								Result.ChunkedContent.ChunkOrders.push_back(FoundChunkIndex);
+							}
+							else
+							{
+								const uint32_t NewChunkIndex   = gsl::narrow<uint32_t>(Result.ChunkedContent.ChunkHashes.size());
+								NewChunkIndexes[OldChunkIndex] = NewChunkIndex;
+								const IoHash&  ChunkHash	   = BaseContent.ChunkedContent.ChunkHashes[OldChunkIndex];
+								const uint64_t OldChunkSize	   = BaseContent.ChunkedContent.ChunkRawSizes[OldChunkIndex];
+								Result.ChunkedContent.ChunkHashes.push_back(ChunkHash);
+								Result.ChunkedContent.ChunkRawSizes.push_back(OldChunkSize);
+								Result.ChunkedContent.ChunkOrders.push_back(NewChunkIndex);
+							}
+						}
+						Result.ChunkedContent.SequenceRawHashes.push_back(RawHash);
+					}
+				}
+			}
+		}
+	}
+	return Result;
+}
+
+ChunkedFolderContent
+DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span<const std::filesystem::path> DeletedPaths)
+{
+	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
+	ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size());
+	if (DeletedPaths.size() == BaseContent.Paths.size())
+	{
+		return {};
+	}
+	const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent);
+	return DeletePathsFromChunkedContent(BaseContent, BaseLookup, DeletedPaths);
+}
+
+ChunkedFolderContent
+ChunkFolderContent(ChunkingStatistics&																Stats,
+				   WorkerThreadPool&																WorkerPool,
+				   const std::filesystem::path&														RootPath,
+				   const FolderContent&																Content,
+				   const ChunkingController&														InChunkingController,
+				   int32_t																			UpdateIntervalMS,
+				   std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+				   std::atomic<bool>&																AbortFlag,
+				   std::atomic<bool>&																PauseFlag)
+{
+	ZEN_TRACE_CPU("ChunkFolderContent");
+
+	Stopwatch Timer;
+	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
+
+	ChunkedFolderContent Result	   = {.Platform	  = Content.Platform,
+									  .Paths	  = Content.Paths,
+									  .RawSizes	  = Content.RawSizes,
+									  .Attributes = Content.Attributes};
+	const size_t		 ItemCount = Result.Paths.size();
+	Result.RawHashes.resize(ItemCount, IoHash::Zero);
+	Result.ChunkedContent.SequenceRawHashes.reserve(ItemCount);	 // Up to 1 per file, maybe less
+	Result.ChunkedContent.ChunkCounts.reserve(ItemCount);		 // Up to one per file
+	Result.ChunkedContent.ChunkOrders.reserve(ItemCount);		 // At least 1 per file, maybe more
+	Result.ChunkedContent.ChunkHashes.reserve(ItemCount);		 // At least 1 per file, maybe more
+	Result.ChunkedContent.ChunkRawSizes.reserve(ItemCount);		 // At least 1 per file, maybe more
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToChunkSequenceIndex;
+	RawHashToChunkSequenceIndex.reserve(ItemCount);
+	ChunkHashToChunkIndex.reserve(ItemCount);
+	{
+		std::vector<uint32_t> Order;
+		Order.resize(ItemCount);
+		for (uint32_t I = 0; I < ItemCount; I++)
+		{
+			Order[I] = I;
+		}
+
+		// Handle the biggest files first so we don't end up with one straggling large file at the end
+		//		std::sort(Order.begin(), Order.end(), [&](uint32_t Lhs, uint32_t Rhs) { return Result.RawSizes[Lhs] > Result.RawSizes[Rhs];
+		//});
+
+		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
+		RawHashToSequenceRawHashIndex.reserve(ItemCount);
+
+		RwLock Lock;
+
+		ParallelWork Work(AbortFlag, PauseFlag);
+
+		for (uint32_t PathIndex : Order)
+		{
+			if (Work.IsAborted())
+			{
+				break;
+			}
+			Work.ScheduleWork(WorkerPool,  // GetSyncWorkerPool()
+							  [&, PathIndex](std::atomic<bool>& AbortFlag) {
+								  if (!AbortFlag)
+								  {
+									  IoHash RawHash = HashOneFile(Stats,
+																   InChunkingController,
+																   Result,
+																   ChunkHashToChunkIndex,
+																   RawHashToSequenceRawHashIndex,
+																   Lock,
+																   RootPath,
+																   PathIndex,
+																   AbortFlag);
+									  Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; });
+									  Stats.FilesProcessed++;
+								  }
+							  });
+		}
+
+		Work.Wait(UpdateIntervalMS, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+			ZEN_UNUSED(PendingWork);
+			UpdateCallback(IsAborted, IsPaused, Work.PendingWork().Remaining());
+		});
+	}
+	return Result;
+}
+
+ChunkedContentLookup
+BuildChunkedContentLookup(const ChunkedFolderContent& Content)
+{
+	ZEN_TRACE_CPU("BuildChunkedContentLookup");
+
+	struct ChunkLocationReference
+	{
+		uint32_t ChunkIndex	   = (uint32_t)-1;
+		uint32_t SequenceIndex = (uint32_t)-1;
+		uint64_t Offset		   = (uint64_t)-1;
+	};
+
+	ChunkedContentLookup Result;
+	{
+		const uint32_t SequenceRawHashesCount = gsl::narrow<uint32_t>(Content.ChunkedContent.SequenceRawHashes.size());
+		Result.RawHashToSequenceIndex.reserve(SequenceRawHashesCount);
+		Result.SequenceIndexChunkOrderOffset.reserve(SequenceRawHashesCount);
+		uint32_t OrderOffset = 0;
+		for (uint32_t SequenceRawHashIndex = 0; SequenceRawHashIndex < Content.ChunkedContent.SequenceRawHashes.size();
+			 SequenceRawHashIndex++)
+		{
+			Result.RawHashToSequenceIndex.insert({Content.ChunkedContent.SequenceRawHashes[SequenceRawHashIndex], SequenceRawHashIndex});
+			Result.SequenceIndexChunkOrderOffset.push_back(OrderOffset);
+			OrderOffset += Content.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
+		}
+	}
+
+	std::vector<ChunkLocationReference> Locations;
+	Locations.reserve(Content.ChunkedContent.ChunkOrders.size());
+	for (uint32_t SequenceIndex = 0; SequenceIndex < Content.ChunkedContent.SequenceRawHashes.size(); SequenceIndex++)
+	{
+		const uint32_t OrderOffset	  = Result.SequenceIndexChunkOrderOffset[SequenceIndex];
+		const uint32_t ChunkCount	  = Content.ChunkedContent.ChunkCounts[SequenceIndex];
+		uint64_t	   LocationOffset = 0;
+		for (size_t OrderIndex = OrderOffset; OrderIndex < OrderOffset + ChunkCount; OrderIndex++)
+		{
+			uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex];
+
+			Locations.push_back(ChunkLocationReference{.ChunkIndex = ChunkIndex, .SequenceIndex = SequenceIndex, .Offset = LocationOffset});
+
+			LocationOffset += Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
+		}
+	}
+
+	std::sort(Locations.begin(), Locations.end(), [](const ChunkLocationReference& Lhs, const ChunkLocationReference& Rhs) {
+		if (Lhs.ChunkIndex < Rhs.ChunkIndex)
+		{
+			return true;
+		}
+		if (Lhs.ChunkIndex > Rhs.ChunkIndex)
+		{
+			return false;
+		}
+		if (Lhs.SequenceIndex < Rhs.SequenceIndex)
+		{
+			return true;
+		}
+		if (Lhs.SequenceIndex > Rhs.SequenceIndex)
+		{
+			return false;
+		}
+		return Lhs.Offset < Rhs.Offset;
+	});
+
+	Result.ChunkSequenceLocations.reserve(Locations.size());
+	const uint32_t ChunkCount = gsl::narrow<uint32_t>(Content.ChunkedContent.ChunkHashes.size());
+	Result.ChunkHashToChunkIndex.reserve(ChunkCount);
+	size_t RangeOffset = 0;
+	for (uint32_t ChunkIndex = 0; ChunkIndex < ChunkCount; ChunkIndex++)
+	{
+		Result.ChunkHashToChunkIndex.insert({Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkIndex});
+		uint32_t Count = 0;
+		while ((RangeOffset + Count < Locations.size()) && (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex))
+		{
+			const ChunkLocationReference& LocationReference = Locations[RangeOffset + Count];
+			Result.ChunkSequenceLocations.push_back(
+				ChunkedContentLookup::ChunkSequenceLocation{.SequenceIndex = LocationReference.SequenceIndex,
+															.Offset		   = LocationReference.Offset});
+			Count++;
+		}
+		Result.ChunkSequenceLocationOffset.push_back(RangeOffset);
+		Result.ChunkSequenceLocationCounts.push_back(Count);
+		RangeOffset += Count;
+	}
+
+	Result.SequenceIndexFirstPathIndex.resize(Content.ChunkedContent.SequenceRawHashes.size(), (uint32_t)-1);
+	Result.PathExtensionHash.resize(Content.Paths.size());
+	for (uint32_t PathIndex = 0; PathIndex < Content.Paths.size(); PathIndex++)
+	{
+		std::string LowercaseExtension = Content.Paths[PathIndex].extension().string();
+		std::transform(LowercaseExtension.begin(), LowercaseExtension.end(), LowercaseExtension.begin(), ::tolower);
+		Result.PathExtensionHash[PathIndex] = HashStringDjb2(LowercaseExtension);
+		if (Content.RawSizes[PathIndex] > 0)
+		{
+			const IoHash& RawHash		  = Content.RawHashes[PathIndex];
+			auto		  SequenceIndexIt = Result.RawHashToSequenceIndex.find(RawHash);
+			ZEN_ASSERT(SequenceIndexIt != Result.RawHashToSequenceIndex.end());
+			const uint32_t SequenceIndex = SequenceIndexIt->second;
+			if (Result.SequenceIndexFirstPathIndex[SequenceIndex] == (uint32_t)-1)
+			{
+				Result.SequenceIndexFirstPathIndex[SequenceIndex] = PathIndex;
+			}
+		}
+	}
+
+	return Result;
+}
+
+}  // namespace zen
diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp
new file mode 100644
index 000000000..a2c041ffd
--- /dev/null
+++ b/src/zenutil/chunkedfile.cpp
@@ -0,0 +1,525 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/chunkedfile.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/trace.h>
+
+#include "chunking.h"
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+namespace {
+	struct ChunkedHeader
+	{
+		static constexpr uint32_t ExpectedMagic	 = 0x646b6863;	// chkd
+		static constexpr uint32_t CurrentVersion = 1;
+
+		uint32_t Magic	 = ExpectedMagic;
+		uint32_t Version = CurrentVersion;
+		uint32_t ChunkSequenceLength;
+		uint32_t ChunkHashCount;
+		uint64_t ChunkSequenceOffset;
+		uint64_t ChunkHashesOffset;
+		uint64_t RawSize = 0;
+		IoHash	 RawHash;
+	};
+}  // namespace
+
+IoBuffer
+SerializeChunkedInfo(const ChunkedInfo& Info)
+{
+	ZEN_TRACE_CPU("SerializeChunkedInfo");
+	size_t HeaderSize = RoundUp(sizeof(ChunkedHeader), 16) + RoundUp(sizeof(uint32_t) * Info.ChunkSequence.size(), 16) +
+						RoundUp(sizeof(IoHash) * Info.ChunkHashes.size(), 16);
+	IoBuffer HeaderData(HeaderSize);
+
+	ChunkedHeader Header;
+	Header.ChunkSequenceLength = gsl::narrow<uint32_t>(Info.ChunkSequence.size());
+	Header.ChunkHashCount	   = gsl::narrow<uint32_t>(Info.ChunkHashes.size());
+	Header.ChunkSequenceOffset = RoundUp(sizeof(ChunkedHeader), 16);
+	Header.ChunkHashesOffset   = RoundUp(Header.ChunkSequenceOffset + sizeof(uint32_t) * Header.ChunkSequenceLength, 16);
+	Header.RawSize			   = Info.RawSize;
+	Header.RawHash			   = Info.RawHash;
+
+	MutableMemoryView WriteView = HeaderData.GetMutableView();
+	{
+		MutableMemoryView HeaderWriteView = WriteView.Left(sizeof(Header));
+		HeaderWriteView.CopyFrom(MemoryView(&Header, sizeof(Header)));
+	}
+	{
+		MutableMemoryView ChunkSequenceWriteView = WriteView.Mid(Header.ChunkSequenceOffset, sizeof(uint32_t) * Header.ChunkSequenceLength);
+		ChunkSequenceWriteView.CopyFrom(MemoryView(Info.ChunkSequence.data(), ChunkSequenceWriteView.GetSize()));
+	}
+	{
+		MutableMemoryView ChunksWriteView = WriteView.Mid(Header.ChunkHashesOffset, sizeof(IoHash) * Header.ChunkHashCount);
+		ChunksWriteView.CopyFrom(MemoryView(Info.ChunkHashes.data(), ChunksWriteView.GetSize()));
+	}
+
+	return HeaderData;
+}
+
+ChunkedInfo
+DeserializeChunkedInfo(IoBuffer& Buffer)
+{
+	ZEN_TRACE_CPU("DeserializeChunkedInfo");
+	MemoryView	  View = Buffer.GetView();
+	ChunkedHeader Header;
+	{
+		MutableMemoryView HeaderWriteView(&Header, sizeof(Header));
+		HeaderWriteView.CopyFrom(View.Left(sizeof(Header)));
+	}
+	if (Header.Magic != ChunkedHeader::ExpectedMagic)
+	{
+		return {};
+	}
+	if (Header.Version != ChunkedHeader::CurrentVersion)
+	{
+		return {};
+	}
+	ChunkedInfo Info;
+	Info.RawSize = Header.RawSize;
+	Info.RawHash = Header.RawHash;
+	Info.ChunkSequence.resize(Header.ChunkSequenceLength);
+	Info.ChunkHashes.resize(Header.ChunkHashCount);
+	{
+		MutableMemoryView ChunkSequenceWriteView(Info.ChunkSequence.data(), sizeof(uint32_t) * Header.ChunkSequenceLength);
+		ChunkSequenceWriteView.CopyFrom(View.Mid(Header.ChunkSequenceOffset, ChunkSequenceWriteView.GetSize()));
+	}
+	{
+		MutableMemoryView ChunksWriteView(Info.ChunkHashes.data(), sizeof(IoHash) * Header.ChunkHashCount);
+		ChunksWriteView.CopyFrom(View.Mid(Header.ChunkHashesOffset, ChunksWriteView.GetSize()));
+	}
+
+	return Info;
+}
+
+void
+Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk)
+{
+	ZEN_TRACE_CPU("Reconstruct");
+	BasicFile Reconstructed;
+	Reconstructed.Open(TargetPath, BasicFile::Mode::kTruncate);
+	BasicFileWriter ReconstructedWriter(Reconstructed, 64 * 1024);
+	uint64_t		Offset = 0;
+	for (uint32_t SequenceIndex : Info.ChunkSequence)
+	{
+		IoBuffer Chunk = GetChunk(Info.ChunkHashes[SequenceIndex]);
+		ReconstructedWriter.Write(Chunk.GetData(), Chunk.GetSize(), Offset);
+		Offset += Chunk.GetSize();
+	}
+}
+
+ChunkedInfoWithSource
+ChunkData(BasicFile&			 RawData,
+		  uint64_t				 Offset,
+		  uint64_t				 Size,
+		  ChunkedParams			 Params,
+		  std::atomic<uint64_t>* BytesProcessed,
+		  std::atomic<bool>*	 AbortFlag)
+{
+	ZEN_TRACE_CPU("ChunkData");
+
+	ChunkedInfoWithSource							 Result;
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> FoundChunks;
+
+	ZenChunkHelper Chunker;
+	Chunker.SetUseThreshold(Params.UseThreshold);
+	Chunker.SetChunkSize(Params.MinSize, Params.MaxSize, Params.AvgSize);
+	size_t			End			   = Offset + Size;
+	const size_t	ScanBufferSize = Max(1u * 1024 * 1024, Params.MaxSize);
+	BasicFileBuffer RawBuffer(RawData, ScanBufferSize);
+	MemoryView		SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset);
+	ZEN_ASSERT(!SliceView.IsEmpty());
+	size_t		 SliceSize = SliceView.GetSize();
+	IoHashStream RawHashStream;
+	while (Offset < End)
+	{
+		if (AbortFlag != nullptr && AbortFlag->load())
+		{
+			return {};
+		}
+		size_t ScanLength = Chunker.ScanChunk(SliceView.GetData(), SliceSize);
+		if (ScanLength == ZenChunkHelper::kNoBoundaryFound)
+		{
+			if (Offset + SliceSize == End)
+			{
+				ScanLength = SliceSize;
+			}
+			else
+			{
+				SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset);
+				SliceSize = SliceView.GetSize();
+				Chunker.Reset();
+				continue;
+			}
+		}
+		uint32_t   ChunkLength = gsl::narrow<uint32_t>(ScanLength);	 // +HashedLength);
+		MemoryView ChunkView   = SliceView.Left(ScanLength);
+		RawHashStream.Append(ChunkView);
+		IoHash ChunkHash = IoHash::HashBuffer(ChunkView);
+		SliceView.RightChopInline(ScanLength);
+		if (auto It = FoundChunks.find(ChunkHash); It != FoundChunks.end())
+		{
+			Result.Info.ChunkSequence.push_back(It->second);
+		}
+		else
+		{
+			uint32_t ChunkIndex = gsl::narrow<uint32_t>(Result.Info.ChunkHashes.size());
+			FoundChunks.insert_or_assign(ChunkHash, ChunkIndex);
+			Result.Info.ChunkHashes.push_back(ChunkHash);
+			Result.ChunkSources.push_back(ChunkSource{.Offset = Offset, .Size = ChunkLength});
+			Result.Info.ChunkSequence.push_back(ChunkIndex);
+		}
+
+		SliceSize = SliceView.GetSize();
+		Offset += ChunkLength;
+		if (BytesProcessed != nullptr)
+		{
+			BytesProcessed->fetch_add(ChunkLength);
+		}
+	}
+	Result.Info.RawSize = Size;
+	Result.Info.RawHash = RawHashStream.GetHash();
+	return Result;
+}
+
+}  // namespace zen
+
+#if ZEN_WITH_TESTS
+#	include <zencore/filesystem.h>
+#	include <zencore/fmtutils.h>
+#	include <zencore/iohash.h>
+#	include <zencore/logging.h>
+#	include <zencore/scopeguard.h>
+#	include <zencore/timer.h>
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#	include <zencore/workthreadpool.h>
+
+#	include "chunking.h"
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#	include <tsl/robin_map.h>
+#	include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+#	if 0
+TEST_CASE("chunkedfile.findparams")
+{
+#		if 1
+	DirectoryContent SourceContent1;
+	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContentFlags::IncludeFiles, SourceContent1);
+	const std::vector<std::filesystem::path>& SourceFiles1 = SourceContent1.Files;
+	DirectoryContent						  SourceContent2;
+	GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContentFlags::IncludeFiles, SourceContent2);
+	const std::vector<std::filesystem::path>& SourceFiles2 = SourceContent2.Files;
+#		else
+	std::filesystem::path SourcePath1 =
+		"E:\\Temp\\ChunkingTestData\\31375996\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode";
+	std::filesystem::path SourcePath2 =
+		"E:\\Temp\\ChunkingTestData\\31379208\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode";
+	const std::vector<std::filesystem::path>& SourceFiles1 = {SourcePath1};
+	const std::vector<std::filesystem::path>& SourceFiles2 = {SourcePath2};
+#		endif
+	ChunkedParams Params[] = {ChunkedParams{.UseThreshold = false, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15456, .MaxSize = 122880, .AvgSize = 35598},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 16848, .MaxSize = 135168, .AvgSize = 39030},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14256, .MaxSize = 114688, .AvgSize = 36222},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15744, .MaxSize = 126976, .AvgSize = 36600},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15264, .MaxSize = 122880, .AvgSize = 35442},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 16464, .MaxSize = 131072, .AvgSize = 37950},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15408, .MaxSize = 122880, .AvgSize = 38914},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15408, .MaxSize = 122880, .AvgSize = 35556},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15360, .MaxSize = 122880, .AvgSize = 35520},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15312, .MaxSize = 122880, .AvgSize = 35478},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 16896, .MaxSize = 135168, .AvgSize = 39072},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15360, .MaxSize = 122880, .AvgSize = 38880},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15840, .MaxSize = 126976, .AvgSize = 36678},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 16800, .MaxSize = 135168, .AvgSize = 38994},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15888, .MaxSize = 126976, .AvgSize = 36714},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15792, .MaxSize = 126976, .AvgSize = 36636},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14880, .MaxSize = 118784, .AvgSize = 37609},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15936, .MaxSize = 126976, .AvgSize = 36756},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15456, .MaxSize = 122880, .AvgSize = 38955},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15984, .MaxSize = 126976, .AvgSize = 36792},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14400, .MaxSize = 114688, .AvgSize = 36338},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14832, .MaxSize = 118784, .AvgSize = 37568},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 16944, .MaxSize = 135168, .AvgSize = 39108},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14352, .MaxSize = 114688, .AvgSize = 36297},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14208, .MaxSize = 114688, .AvgSize = 36188},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14448, .MaxSize = 114688, .AvgSize = 36372},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 13296, .MaxSize = 106496, .AvgSize = 36592},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15264, .MaxSize = 122880, .AvgSize = 38805},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14304, .MaxSize = 114688, .AvgSize = 36263},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14784, .MaxSize = 118784, .AvgSize = 37534},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 15312, .MaxSize = 122880, .AvgSize = 38839},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14256, .MaxSize = 114688, .AvgSize = 39360},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 13776, .MaxSize = 110592, .AvgSize = 37976},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14736, .MaxSize = 118784, .AvgSize = 37493},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14928, .MaxSize = 118784, .AvgSize = 37643},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14448, .MaxSize = 114688, .AvgSize = 39504},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 13392, .MaxSize = 106496, .AvgSize = 36664},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 13872, .MaxSize = 110592, .AvgSize = 38048},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 14352, .MaxSize = 114688, .AvgSize = 39432},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 13200, .MaxSize = 106496, .AvgSize = 36520},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17328, .MaxSize = 139264, .AvgSize = 36378},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17376, .MaxSize = 139264, .AvgSize = 36421},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17424, .MaxSize = 139264, .AvgSize = 36459},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17472, .MaxSize = 139264, .AvgSize = 36502},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17520, .MaxSize = 139264, .AvgSize = 36540},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17808, .MaxSize = 143360, .AvgSize = 37423},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17856, .MaxSize = 143360, .AvgSize = 37466},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 25834},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 21917},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 29751},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 33668},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17952, .MaxSize = 143360, .AvgSize = 37547},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 17904, .MaxSize = 143360, .AvgSize = 37504},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 22371},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 37585},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 26406},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 26450},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 30615},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 30441},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 22417},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 22557},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 30528},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 27112},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 34644},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 34476},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 35408},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 38592},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 30483},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 26586},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 26496},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 31302},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 34516},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 22964},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 35448},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 38630},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 23010},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 31260},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 34600},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 27156},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 30570},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 38549},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 22510},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 38673},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 34560},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 22464},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 26540},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 38511},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 23057},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 27202},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 31347},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 35492},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 31389},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 27246},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 23103},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 35532},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 23150},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 27292},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 31434},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 35576},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 27336},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 23196},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 31476},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 35616},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 27862},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 32121},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 23603},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 36380},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 27908},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 23650},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 32166},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 36424},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 23696},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 32253},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 32208},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 23743},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 36548},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 28042},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 23789},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 32295},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 36508},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 27952},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 27998},
+							  ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 36464}};
+
+	static const size_t				   ParamsCount = sizeof(Params) / sizeof(ChunkedParams);
+	std::vector<ChunkedInfoWithSource> Infos1(SourceFiles1.size());
+	std::vector<ChunkedInfoWithSource> Infos2(SourceFiles2.size());
+
+	WorkerThreadPool WorkerPool(32);
+
+	for (size_t I = 0; I < ParamsCount; I++)
+	{
+		for (int UseThreshold = 0; UseThreshold < 2; UseThreshold++)
+		{
+			Latch		  WorkLatch(1);
+			ChunkedParams Param = Params[I];
+			Param.UseThreshold	= UseThreshold == 1;
+			Stopwatch Timer;
+			for (size_t F = 0; F < SourceFiles1.size(); F++)
+			{
+				WorkLatch.AddCount(1);
+				WorkerPool.ScheduleWork([&WorkLatch, F, Param, &SourceFiles1, &Infos1]() {
+					auto	  _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
+					BasicFile SourceData1;
+					SourceData1.Open(SourceFiles1[F], BasicFile::Mode::kRead);
+					Infos1[F] = ChunkData(SourceData1, 0, SourceData1.FileSize(), Param);
+				});
+			}
+			for (size_t F = 0; F < SourceFiles2.size(); F++)
+			{
+				WorkLatch.AddCount(1);
+				WorkerPool.ScheduleWork([&WorkLatch, F, Param, &SourceFiles2, &Infos2]() {
+					auto	  _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
+					BasicFile SourceData2;
+					SourceData2.Open(SourceFiles2[F], BasicFile::Mode::kRead);
+					Infos2[F] = ChunkData(SourceData2, 0, SourceData2.FileSize(), Param);
+				});
+			}
+			WorkLatch.CountDown();
+			WorkLatch.Wait();
+			uint64_t ChunkTimeMS = Timer.GetElapsedTimeMs();
+
+			uint64_t			   Raw1Size = 0;
+			tsl::robin_set<IoHash> Chunks1;
+			size_t				   ChunkedSize1 = 0;
+			for (size_t F = 0; F < SourceFiles1.size(); F++)
+			{
+				const ChunkedInfoWithSource& Info = Infos1[F];
+				Raw1Size += Info.Info.RawSize;
+				for (uint32_t Chunk1Index = 0; Chunk1Index < Info.Info.ChunkHashes.size(); ++Chunk1Index)
+				{
+					const IoHash ChunkHash = Info.Info.ChunkHashes[Chunk1Index];
+					if (Chunks1.insert(ChunkHash).second)
+					{
+						ChunkedSize1 += Info.ChunkSources[Chunk1Index].Size;
+					}
+				}
+			}
+
+			uint64_t			   Raw2Size = 0;
+			tsl::robin_set<IoHash> Chunks2;
+			size_t				   ChunkedSize2 = 0;
+			size_t				   DiffSize		= 0;
+			for (size_t F = 0; F < SourceFiles2.size(); F++)
+			{
+				const ChunkedInfoWithSource& Info = Infos2[F];
+				Raw2Size += Info.Info.RawSize;
+				for (uint32_t Chunk2Index = 0; Chunk2Index < Info.Info.ChunkHashes.size(); ++Chunk2Index)
+				{
+					const IoHash ChunkHash = Info.Info.ChunkHashes[Chunk2Index];
+					if (Chunks2.insert(ChunkHash).second)
+					{
+						ChunkedSize2 += Info.ChunkSources[Chunk2Index].Size;
+						if (!Chunks1.contains(ChunkHash))
+						{
+							DiffSize += Info.ChunkSources[Chunk2Index].Size;
+						}
+					}
+				}
+			}
+
+			ZEN_INFO(
+				"Diff = {}, Chunks1 = {}, Chunks2 = {}, .UseThreshold = {}, .MinSize = {}, .MaxSize = {}, .AvgSize = {}, RawSize(1) = {}, "
+				"RawSize(2) = {}, "
+				"Saved(1) = {}, Saved(2) = {} in {}",
+				NiceBytes(DiffSize),
+				Chunks1.size(),
+				Chunks2.size(),
+				Param.UseThreshold,
+				Param.MinSize,
+				Param.MaxSize,
+				Param.AvgSize,
+				NiceBytes(Raw1Size),
+				NiceBytes(Raw2Size),
+				NiceBytes(Raw1Size - ChunkedSize1),
+				NiceBytes(Raw2Size - ChunkedSize2),
+				NiceTimeSpanMs(ChunkTimeMS));
+		}
+	}
+
+#		if 0
+	for (int64_t MinSizeBase = (12u * 1024u); MinSizeBase <= (32u * 1024u); MinSizeBase += 512)
+		{
+			for (int64_t Wiggle = -132; Wiggle < 126; Wiggle += 2)
+			{
+				//			size_t MinSize = 7 * 1024 - 61;	   // (size_t)(MinSizeBase + Wiggle);
+			//			size_t MaxSize = 16 * (7 * 1024);  // 8 * 7 * 1024;// MinSizeBase * 6;
+			//			size_t AvgSize = MaxSize / 2;	   // 4 * 7 * 1024;// MinSizeBase * 3;
+			size_t MinSize = (size_t)(MinSizeBase + Wiggle);
+			//for (size_t MaxSize = (MinSize * 4) - 768; MaxSize < (MinSize * 5) + 768; MaxSize += 64)
+			size_t MaxSize = 8u * MinSizeBase;
+			{
+				for (size_t AvgSize = (MaxSize - MinSize) / 32 + MinSize; AvgSize < (MaxSize - MinSize) / 4 + MinSize; AvgSize += (MaxSize - MinSize) / 32)
+//				size_t AvgSize = (MaxSize - MinSize) / 4 + MinSize;
+				{
+					WorkLatch.AddCount(1);
+					WorkerPool.ScheduleWork([&WorkLatch, MinSize, MaxSize, AvgSize, SourcePath1, SourcePath2]()
+					{
+						auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
+						ChunkedParams		   Params{ .UseThreshold = true, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize };
+						BasicFile SourceData1;
+						SourceData1.Open(SourcePath1, BasicFile::Mode::kRead);
+						BasicFile SourceData2;
+						SourceData2.Open(SourcePath2, BasicFile::Mode::kRead);
+						ChunkedInfoWithSource  Info1 = ChunkData(SourceData1, Params);
+						ChunkedInfoWithSource  Info2 = ChunkData(SourceData2, Params);
+
+						tsl::robin_set<IoHash> Chunks1;
+						Chunks1.reserve(Info1.Info.ChunkHashes.size());
+						Chunks1.insert(Info1.Info.ChunkHashes.begin(), Info1.Info.ChunkHashes.end());
+						size_t ChunkedSize1 = 0;
+						for (uint32_t Chunk1Index = 0; Chunk1Index < Info1.Info.ChunkHashes.size(); ++Chunk1Index)
+						{
+							ChunkedSize1 += Info1.ChunkSources[Chunk1Index].Size;
+						}
+						size_t DiffSavedSize = 0;
+						size_t ChunkedSize2 = 0;
+						for (uint32_t Chunk2Index = 0; Chunk2Index < Info2.Info.ChunkHashes.size(); ++Chunk2Index)
+						{
+							ChunkedSize2 += Info2.ChunkSources[Chunk2Index].Size;
+							if (Chunks1.find(Info2.Info.ChunkHashes[Chunk2Index]) == Chunks1.end())
+							{
+								DiffSavedSize += Info2.ChunkSources[Chunk2Index].Size;
+							}
+						}
+						ZEN_INFO("Diff {}, Chunks1: {}, Chunks2: {}, Min: {}, Max: {}, Avg: {}, Saved(1) {}, Saved(2) {}",
+								 NiceBytes(DiffSavedSize),
+								 Info1.Info.ChunkHashes.size(),
+								 Info2.Info.ChunkHashes.size(),
+								 MinSize,
+								 MaxSize,
+								 AvgSize,
+								 NiceBytes(Info1.Info.RawSize - ChunkedSize1),
+								 NiceBytes(Info2.Info.RawSize - ChunkedSize2));
+					});
+				}
+			}
+		}
+	}
+#		endif	// 0
+
+	//	WorkLatch.CountDown();
+	//	WorkLatch.Wait();
+}
+#	endif		// 0
+
+void
+chunkedfile_forcelink()
+{
+}
+
+}  // namespace zen
+
+#endif
diff --git a/src/zenutil/chunking.cpp b/src/zenutil/chunking.cpp
new file mode 100644
index 000000000..30edd322a
--- /dev/null
+++ b/src/zenutil/chunking.cpp
@@ -0,0 +1,382 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "chunking.h"
+
+#include <gsl/gsl-lite.hpp>
+
+#include <cmath>
+
+namespace zen::detail {
+
+static const uint32_t BuzhashTable[] = {
+	0x458be752, 0xc10748cc, 0xfbbcdbb8, 0x6ded5b68, 0xb10a82b5, 0x20d75648, 0xdfc5665f, 0xa8428801, 0x7ebf5191, 0x841135c7, 0x65cc53b3,
+	0x280a597c, 0x16f60255, 0xc78cbc3e, 0x294415f5, 0xb938d494, 0xec85c4e6, 0xb7d33edc, 0xe549b544, 0xfdeda5aa, 0x882bf287, 0x3116737c,
+	0x05569956, 0xe8cc1f68, 0x0806ac5e, 0x22a14443, 0x15297e10, 0x50d090e7, 0x4ba60f6f, 0xefd9f1a7, 0x5c5c885c, 0x82482f93, 0x9bfd7c64,
+	0x0b3e7276, 0xf2688e77, 0x8fad8abc, 0xb0509568, 0xf1ada29f, 0xa53efdfe, 0xcb2b1d00, 0xf2a9e986, 0x6463432b, 0x95094051, 0x5a223ad2,
+	0x9be8401b, 0x61e579cb, 0x1a556a14, 0x5840fdc2, 0x9261ddf6, 0xcde002bb, 0x52432bb0, 0xbf17373e, 0x7b7c222f, 0x2955ed16, 0x9f10ca59,
+	0xe840c4c9, 0xccabd806, 0x14543f34, 0x1462417a, 0x0d4a1f9c, 0x087ed925, 0xd7f8f24c, 0x7338c425, 0xcf86c8f5, 0xb19165cd, 0x9891c393,
+	0x325384ac, 0x0308459d, 0x86141d7e, 0xc922116a, 0xe2ffa6b6, 0x53f52aed, 0x2cd86197, 0xf5b9f498, 0xbf319c8f, 0xe0411fae, 0x977eb18c,
+	0xd8770976, 0x9833466a, 0xc674df7f, 0x8c297d45, 0x8ca48d26, 0xc49ed8e2, 0x7344f874, 0x556f79c7, 0x6b25eaed, 0xa03e2b42, 0xf68f66a4,
+	0x8e8b09a2, 0xf2e0e62a, 0x0d3a9806, 0x9729e493, 0x8c72b0fc, 0x160b94f6, 0x450e4d3d, 0x7a320e85, 0xbef8f0e1, 0x21d73653, 0x4e3d977a,
+	0x1e7b3929, 0x1cc6c719, 0xbe478d53, 0x8d752809, 0xe6d8c2c6, 0x275f0892, 0xc8acc273, 0x4cc21580, 0xecc4a617, 0xf5f7be70, 0xe795248a,
+	0x375a2fe9, 0x425570b6, 0x8898dcf8, 0xdc2d97c4, 0x0106114b, 0x364dc22f, 0x1e0cad1f, 0xbe63803c, 0x5f69fac2, 0x4d5afa6f, 0x1bc0dfb5,
+	0xfb273589, 0x0ea47f7b, 0x3c1c2b50, 0x21b2a932, 0x6b1223fd, 0x2fe706a8, 0xf9bd6ce2, 0xa268e64e, 0xe987f486, 0x3eacf563, 0x1ca2018c,
+	0x65e18228, 0x2207360a, 0x57cf1715, 0x34c37d2b, 0x1f8f3cde, 0x93b657cf, 0x31a019fd, 0xe69eb729, 0x8bca7b9b, 0x4c9d5bed, 0x277ebeaf,
+	0xe0d8f8ae, 0xd150821c, 0x31381871, 0xafc3f1b0, 0x927db328, 0xe95effac, 0x305a47bd, 0x426ba35b, 0x1233af3f, 0x686a5b83, 0x50e072e5,
+	0xd9d3bb2a, 0x8befc475, 0x487f0de6, 0xc88dff89, 0xbd664d5e, 0x971b5d18, 0x63b14847, 0xd7d3c1ce, 0x7f583cf3, 0x72cbcb09, 0xc0d0a81c,
+	0x7fa3429b, 0xe9158a1b, 0x225ea19a, 0xd8ca9ea3, 0xc763b282, 0xbb0c6341, 0x020b8293, 0xd4cd299d, 0x58cfa7f8, 0x91b4ee53, 0x37e4d140,
+	0x95ec764c, 0x30f76b06, 0x5ee68d24, 0x679c8661, 0xa41979c2, 0xf2b61284, 0x4fac1475, 0x0adb49f9, 0x19727a23, 0x15a7e374, 0xc43a18d5,
+	0x3fb1aa73, 0x342fc615, 0x924c0793, 0xbee2d7f0, 0x8a279de9, 0x4aa2d70c, 0xe24dd37f, 0xbe862c0b, 0x177c22c2, 0x5388e5ee, 0xcd8a7510,
+	0xf901b4fd, 0xdbc13dbc, 0x6c0bae5b, 0x64efe8c7, 0x48b02079, 0x80331a49, 0xca3d8ae6, 0xf3546190, 0xfed7108b, 0xc49b941b, 0x32baf4a9,
+	0xeb833a4a, 0x88a3f1a5, 0x3a91ce0a, 0x3cc27da1, 0x7112e684, 0x4a3096b1, 0x3794574c, 0xa3c8b6f3, 0x1d213941, 0x6e0a2e00, 0x233479f1,
+	0x0f4cd82f, 0x6093edd2, 0x5d7d209e, 0x464fe319, 0xd4dcac9e, 0x0db845cb, 0xfb5e4bc3, 0xe0256ce1, 0x09fb4ed1, 0x0914be1e, 0xa5bdb2c3,
+	0xc6eb57bb, 0x30320350, 0x3f397e91, 0xa67791bc, 0x86bc0e2c, 0xefa0a7e2, 0xe9ff7543, 0xe733612c, 0xd185897b, 0x329e5388, 0x91dd236b,
+	0x2ecb0d93, 0xf4d82a3d, 0x35b5c03f, 0xe4e606f0, 0x05b21843, 0x37b45964, 0x5eff22f4, 0x6027f4cc, 0x77178b3c, 0xae507131, 0x7bf7cabc,
+	0xf9c18d66, 0x593ade65, 0xd95ddf11,
+};
+
+// ROL operation (compiler turns this into a ROL when optimizing)
+ZEN_FORCEINLINE static uint32_t
+Rotate32(uint32_t Value, size_t RotateCount)
+{
+	RotateCount &= 31;
+
+	return ((Value) << (RotateCount)) | ((Value) >> (32 - RotateCount));
+}
+
+}  // namespace zen::detail
+
+namespace zen {
+
+void
+ZenChunkHelper::Reset()
+{
+	InternalReset();
+
+	m_BytesScanned = 0;
+}
+
+void
+ZenChunkHelper::InternalReset()
+{
+	m_CurrentHash	   = 0;
+	m_CurrentChunkSize = 0;
+	m_WindowSize	   = 0;
+}
+
+void
+ZenChunkHelper::SetChunkSize(size_t MinSize, size_t MaxSize, size_t AvgSize)
+{
+	if (m_WindowSize)
+		return;	 // Already started
+
+	static_assert(kChunkSizeLimitMin > kWindowSize);
+
+	if (AvgSize)
+	{
+		// TODO: Validate AvgSize range
+	}
+	else
+	{
+		if (MinSize && MaxSize)
+		{
+			AvgSize = std::lrint(std::pow(2, (std::log2(MinSize) + std::log2(MaxSize)) / 2));
+		}
+		else if (MinSize)
+		{
+			AvgSize = MinSize * 4;
+		}
+		else if (MaxSize)
+		{
+			AvgSize = MaxSize / 4;
+		}
+		else
+		{
+			AvgSize = kDefaultAverageChunkSize;
+		}
+	}
+
+	if (MinSize)
+	{
+		// TODO: Validate MinSize range
+	}
+	else
+	{
+		MinSize = std::max(AvgSize / 4, kChunkSizeLimitMin);
+	}
+
+	if (MaxSize)
+	{
+		// TODO: Validate MaxSize range
+	}
+	else
+	{
+		MaxSize = std::min(AvgSize * 4, kChunkSizeLimitMax);
+	}
+
+	m_Discriminator = gsl::narrow<uint32_t>(AvgSize - MinSize);
+
+	if (m_Discriminator < MinSize)
+	{
+		m_Discriminator = gsl::narrow<uint32_t>(MinSize);
+	}
+
+	if (m_Discriminator > MaxSize)
+	{
+		m_Discriminator = gsl::narrow<uint32_t>(MaxSize);
+	}
+
+	m_Threshold = gsl::narrow<uint32_t>((uint64_t(std::numeric_limits<uint32_t>::max()) + 1) / m_Discriminator);
+
+	m_ChunkSizeMin = MinSize;
+	m_ChunkSizeMax = MaxSize;
+	m_ChunkSizeAvg = AvgSize;
+}
+
+size_t
+ZenChunkHelper::ScanChunk(const void* DataBytesIn, size_t ByteCount)
+{
+	size_t Result = InternalScanChunk(DataBytesIn, ByteCount);
+
+	if (Result == kNoBoundaryFound)
+	{
+		m_BytesScanned += ByteCount;
+	}
+	else
+	{
+		m_BytesScanned += Result;
+	}
+
+	return Result;
+}
+
+size_t
+ZenChunkHelper::InternalScanChunk(const void* DataBytesIn, size_t ByteCount)
+{
+	size_t		   CurrentOffset = 0;
+	const uint8_t* CursorPtr	 = reinterpret_cast<const uint8_t*>(DataBytesIn);
+
+	// There's no point in updating the hash if we know we're not
+	// going to have a cut point, so just skip the data. This logic currently
+	// provides roughly a 20% speedup on my machine
+
+	const size_t NeedHashOffset = m_ChunkSizeMin - kWindowSize;
+
+	if (m_CurrentChunkSize < NeedHashOffset)
+	{
+		const uint32_t SkipBytes = gsl::narrow<uint32_t>(std::min<uint64_t>(ByteCount, NeedHashOffset - m_CurrentChunkSize));
+
+		ByteCount -= SkipBytes;
+		m_CurrentChunkSize += SkipBytes;
+		CurrentOffset += SkipBytes;
+		CursorPtr += SkipBytes;
+
+		m_WindowSize = 0;
+
+		if (ByteCount == 0)
+		{
+			return kNoBoundaryFound;
+		}
+	}
+
+	// Fill window first
+
+	if (m_WindowSize < kWindowSize)
+	{
+		const uint32_t FillBytes = uint32_t(std::min<size_t>(ByteCount, kWindowSize - m_WindowSize));
+
+		memcpy(&m_Window[m_WindowSize], CursorPtr, FillBytes);
+
+		CursorPtr += FillBytes;
+
+		m_WindowSize += FillBytes;
+		m_CurrentChunkSize += FillBytes;
+
+		CurrentOffset += FillBytes;
+		ByteCount -= FillBytes;
+
+		if (m_WindowSize < kWindowSize)
+		{
+			return kNoBoundaryFound;
+		}
+
+		// We have a full window, initialize hash
+
+		uint32_t CurrentHash = 0;
+
+		for (int i = 1; i < kWindowSize; ++i)
+		{
+			CurrentHash ^= detail::Rotate32(detail::BuzhashTable[m_Window[i - 1]], kWindowSize - i);
+		}
+
+		m_CurrentHash = CurrentHash ^ detail::BuzhashTable[m_Window[kWindowSize - 1]];
+	}
+
+	// Scan for boundaries (i.e points where the hash matches the value determined by
+	// the discriminator)
+
+	uint32_t CurrentHash	  = m_CurrentHash;
+	uint32_t CurrentChunkSize = m_CurrentChunkSize;
+
+	size_t Index = CurrentChunkSize % kWindowSize;
+
+	if (m_Threshold && m_UseThreshold)
+	{
+		// This is roughly 4x faster than the general modulo approach on my
+		// TR 3990X (~940MB/sec) and doesn't require any special parameters to
+		// achieve max performance
+
+		while (ByteCount)
+		{
+			const uint8_t NewByte = *CursorPtr;
+			const uint8_t OldByte = m_Window[Index];
+
+			CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^
+						  detail::BuzhashTable[NewByte];
+
+			CurrentChunkSize++;
+			CurrentOffset++;
+
+			if (CurrentChunkSize >= m_ChunkSizeMin)
+			{
+				bool FoundBoundary;
+
+				if (CurrentChunkSize >= m_ChunkSizeMax)
+				{
+					FoundBoundary = true;
+				}
+				else
+				{
+					FoundBoundary = CurrentHash <= m_Threshold;
+				}
+
+				if (FoundBoundary)
+				{
+					// Boundary found!
+					InternalReset();
+
+					return CurrentOffset;
+				}
+			}
+
+			m_Window[Index++] = *CursorPtr;
+
+			if (Index == kWindowSize)
+			{
+				Index = 0;
+			}
+
+			++CursorPtr;
+			--ByteCount;
+		}
+	}
+	else if ((m_Discriminator & (m_Discriminator - 1)) == 0)
+	{
+		// This is quite a bit faster than the generic modulo path, but
+		// requires a very specific average chunk size to be used. If you
+		// pass in an even power-of-two divided by 0.75 as the average
+		// chunk size you'll hit this path
+
+		const uint32_t Mask = m_Discriminator - 1;
+
+		while (ByteCount)
+		{
+			const uint8_t NewByte = *CursorPtr;
+			const uint8_t OldByte = m_Window[Index];
+
+			CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^
+						  detail::BuzhashTable[NewByte];
+
+			CurrentChunkSize++;
+			CurrentOffset++;
+
+			if (CurrentChunkSize >= m_ChunkSizeMin)
+			{
+				bool FoundBoundary;
+
+				if (CurrentChunkSize >= m_ChunkSizeMax)
+				{
+					FoundBoundary = true;
+				}
+				else
+				{
+					FoundBoundary = (CurrentHash & Mask) == Mask;
+				}
+
+				if (FoundBoundary)
+				{
+					// Boundary found!
+					InternalReset();
+
+					return CurrentOffset;
+				}
+			}
+
+			m_Window[Index++] = *CursorPtr;
+
+			if (Index == kWindowSize)
+			{
+				Index = 0;
+			}
+
+			++CursorPtr;
+			--ByteCount;
+		}
+	}
+	else
+	{
+		// This is the slowest path, which caps out around 250MB/sec for large sizes
+		// on my TR3900X
+
+		while (ByteCount)
+		{
+			const uint8_t NewByte = *CursorPtr;
+			const uint8_t OldByte = m_Window[Index];
+
+			CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^
+						  detail::BuzhashTable[NewByte];
+
+			CurrentChunkSize++;
+			CurrentOffset++;
+
+			if (CurrentChunkSize >= m_ChunkSizeMin)
+			{
+				bool FoundBoundary;
+
+				if (CurrentChunkSize >= m_ChunkSizeMax)
+				{
+					FoundBoundary = true;
+				}
+				else
+				{
+					FoundBoundary = (CurrentHash % m_Discriminator) == (m_Discriminator - 1);
+				}
+
+				if (FoundBoundary)
+				{
+					// Boundary found!
+					InternalReset();
+
+					return CurrentOffset;
+				}
+			}
+
+			m_Window[Index++] = *CursorPtr;
+
+			if (Index == kWindowSize)
+			{
+				Index = 0;
+			}
+
+			++CursorPtr;
+			--ByteCount;
+		}
+	}
+
+	m_CurrentChunkSize = CurrentChunkSize;
+	m_CurrentHash	   = CurrentHash;
+
+	return kNoBoundaryFound;
+}
+
+}  // namespace zen
diff --git a/src/zenutil/chunking.h b/src/zenutil/chunking.h
new file mode 100644
index 000000000..09c56454f
--- /dev/null
+++ b/src/zenutil/chunking.h
@@ -0,0 +1,56 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+#include <zencore/zencore.h>
+
+namespace zen {
+
+/** Content-defined chunking helper
+ */
+class ZenChunkHelper
+{
+public:
+	void   SetChunkSize(size_t MinSize, size_t MaxSize, size_t AvgSize);
+	size_t ScanChunk(const void* DataBytes, size_t ByteCount);
+	void   Reset();
+
+	// This controls which chunking approach is used - threshold or
+	// modulo based. Threshold is faster and generates similarly sized
+	// chunks
+	void SetUseThreshold(bool NewState) { m_UseThreshold = NewState; }
+
+	inline size_t	ChunkSizeMin() const { return m_ChunkSizeMin; }
+	inline size_t	ChunkSizeMax() const { return m_ChunkSizeMax; }
+	inline size_t	ChunkSizeAvg() const { return m_ChunkSizeAvg; }
+	inline uint64_t BytesScanned() const { return m_BytesScanned; }
+
+	static constexpr size_t kNoBoundaryFound = size_t(~0ull);
+
+private:
+	size_t m_ChunkSizeMin = 0;
+	size_t m_ChunkSizeMax = 0;
+	size_t m_ChunkSizeAvg = 0;
+
+	uint32_t m_Discriminator = 0;  // Computed in SetChunkSize()
+	uint32_t m_Threshold	 = 0;  // Computed in SetChunkSize()
+
+	bool m_UseThreshold = true;
+
+	static constexpr size_t kChunkSizeLimitMax		 = 64 * 1024 * 1024;
+	static constexpr size_t kChunkSizeLimitMin		 = 1024;
+	static constexpr size_t kDefaultAverageChunkSize = 64 * 1024;
+
+	static constexpr int kWindowSize = 48;
+	uint8_t				 m_Window[kWindowSize];
+	uint32_t			 m_WindowSize = 0;
+
+	uint32_t m_CurrentHash		= 0;
+	uint32_t m_CurrentChunkSize = 0;
+
+	uint64_t m_BytesScanned = 0;
+
+	size_t InternalScanChunk(const void* DataBytes, size_t ByteCount);
+	void   InternalReset();
+};
+
+}  // namespace zen
diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp
new file mode 100644
index 000000000..6fb4182c0
--- /dev/null
+++ b/src/zenutil/chunkingcontroller.cpp
@@ -0,0 +1,359 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/chunkingcontroller.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/filesystem.h>
+#include <zencore/trace.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+using namespace std::literals;
+
+namespace {
+	std::vector<std::string> ReadStringArray(CbArrayView StringArray)
+	{
+		std::vector<std::string> Result;
+		Result.reserve(StringArray.Num());
+		for (CbFieldView FieldView : StringArray)
+		{
+			Result.emplace_back(FieldView.AsString());
+		}
+		return Result;
+	}
+
+	ChunkedParams ReadChunkParams(CbObjectView Params)
+	{
+		bool   UseThreshold = Params["UseThreshold"sv].AsBool(true);
+		size_t MinSize		= Params["MinSize"sv].AsUInt64(DefaultChunkedParams.MinSize);
+		size_t MaxSize		= Params["MaxSize"sv].AsUInt64(DefaultChunkedParams.MaxSize);
+		size_t AvgSize		= Params["AvgSize"sv].AsUInt64(DefaultChunkedParams.AvgSize);
+
+		return ChunkedParams{.UseThreshold = UseThreshold, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize};
+	}
+
+	void WriteChunkParams(CbObjectWriter& Writer, const ChunkedParams& Params)
+	{
+		Writer.BeginObject("ChunkingParams"sv);
+		{
+			Writer.AddBool("UseThreshold"sv, Params.UseThreshold);
+
+			Writer.AddInteger("MinSize"sv, (uint64_t)Params.MinSize);
+			Writer.AddInteger("MaxSize"sv, (uint64_t)Params.MaxSize);
+			Writer.AddInteger("AvgSize"sv, (uint64_t)Params.AvgSize);
+		}
+		Writer.EndObject();	 // ChunkingParams
+	}
+
+	bool IsElfFile(BasicFile& Buffer)
+	{
+		if (Buffer.FileSize() > 4)
+		{
+			uint32_t ElfCheck = 0;
+			Buffer.Read(&ElfCheck, 4, 0);
+			if (ElfCheck == 0x464c457f)
+			{
+				return true;
+			}
+		}
+		return false;
+	}
+
+	bool IsMachOFile(BasicFile& Buffer)
+	{
+		if (Buffer.FileSize() > 4)
+		{
+			uint32_t MachOCheck = 0;
+			Buffer.Read(&MachOCheck, 4, 0);
+			if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe))
+			{
+				return true;
+			}
+		}
+		return false;
+	}
+}  // namespace
+
+class BasicChunkingController : public ChunkingController
+{
+public:
+	BasicChunkingController(const BasicChunkingControllerSettings& Settings) : m_Settings(Settings) {}
+
+	BasicChunkingController(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {}
+
+	virtual bool ProcessFile(const std::filesystem::path& InputPath,
+							 uint64_t					  RawSize,
+							 ChunkedInfoWithSource&		  OutChunked,
+							 std::atomic<uint64_t>&		  BytesProcessed,
+							 std::atomic<bool>&			  AbortFlag) const override
+	{
+		ZEN_TRACE_CPU("BasicChunkingController::ProcessFile");
+		const bool ExcludeFromChunking =
+			std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) !=
+			m_Settings.ExcludeExtensions.end();
+
+		if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit))
+		{
+			return false;
+		}
+
+		BasicFile Buffer(InputPath, BasicFile::Mode::kRead);
+		if (m_Settings.ExcludeElfFiles && IsElfFile(Buffer))
+		{
+			return false;
+		}
+		if (m_Settings.ExcludeMachOFiles && IsMachOFile(Buffer))
+		{
+			return false;
+		}
+
+		OutChunked = ChunkData(Buffer, 0, RawSize, m_Settings.ChunkingParams, &BytesProcessed, &AbortFlag);
+		return true;
+	}
+
+	virtual std::string_view GetName() const override { return Name; }
+
+	virtual CbObject GetParameters() const override
+	{
+		CbObjectWriter Writer;
+		Writer.BeginArray("ChunkExcludeExtensions"sv);
+		{
+			for (const std::string& Extension : m_Settings.ExcludeExtensions)
+			{
+				Writer.AddString(Extension);
+			}
+		}
+		Writer.EndArray();	// ChunkExcludeExtensions
+
+		Writer.AddBool("ExcludeElfFiles"sv, m_Settings.ExcludeElfFiles);
+		Writer.AddBool("ExcludeMachOFiles"sv, m_Settings.ExcludeMachOFiles);
+		Writer.AddInteger("ChunkFileSizeLimit"sv, m_Settings.ChunkFileSizeLimit);
+
+		WriteChunkParams(Writer, m_Settings.ChunkingParams);
+
+		return Writer.Save();
+	}
+	static constexpr std::string_view Name = "BasicChunkingController"sv;
+
+private:
+	static BasicChunkingControllerSettings ReadSettings(CbObjectView Parameters)
+	{
+		return BasicChunkingControllerSettings{
+			.ExcludeExtensions	= ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()),
+			.ExcludeElfFiles	= Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles),
+			.ExcludeMachOFiles	= Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles),
+			.ChunkFileSizeLimit = Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit),
+			.ChunkingParams		= ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())};
+	}
+
+	const BasicChunkingControllerSettings m_Settings;
+};
+
+class ChunkingControllerWithFixedChunking : public ChunkingController
+{
+public:
+	ChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Settings) : m_Settings(Settings) {}
+
+	ChunkingControllerWithFixedChunking(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {}
+
+	virtual bool ProcessFile(const std::filesystem::path& InputPath,
+							 uint64_t					  RawSize,
+							 ChunkedInfoWithSource&		  OutChunked,
+							 std::atomic<uint64_t>&		  BytesProcessed,
+							 std::atomic<bool>&			  AbortFlag) const override
+	{
+		ZEN_TRACE_CPU("ChunkingControllerWithFixedChunking::ProcessFile");
+		const bool ExcludeFromChunking =
+			std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) !=
+			m_Settings.ExcludeExtensions.end();
+
+		if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit))
+		{
+			return false;
+		}
+
+		const bool FixedChunkingExtension =
+			std::find(m_Settings.FixedChunkingExtensions.begin(), m_Settings.FixedChunkingExtensions.end(), InputPath.extension()) !=
+			m_Settings.FixedChunkingExtensions.end();
+
+		if (FixedChunkingExtension)
+		{
+			if (RawSize < m_Settings.MinSizeForFixedChunking)
+			{
+				return false;
+			}
+			ZEN_TRACE_CPU("FixedChunking");
+			IoHashStream									 FullHasher;
+			BasicFile										 Source(InputPath, BasicFile::Mode::kRead);
+			uint64_t										 Offset = 0;
+			tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+			const uint64_t									 ExpectedChunkCount = 1 + (RawSize / m_Settings.FixedChunkingChunkSize);
+			ChunkHashToChunkIndex.reserve(ExpectedChunkCount);
+			OutChunked.Info.ChunkHashes.reserve(ExpectedChunkCount);
+			OutChunked.Info.ChunkSequence.reserve(ExpectedChunkCount);
+			OutChunked.ChunkSources.reserve(ExpectedChunkCount);
+
+			static const uint64_t BufferingSize = 256u * 1024u;
+
+			IoHashStream ChunkHasher;
+
+			while (Offset < RawSize)
+			{
+				if (AbortFlag)
+				{
+					return false;
+				}
+
+				ChunkHasher.Reset();
+
+				uint64_t ChunkSize = std::min<uint64_t>(RawSize - Offset, m_Settings.FixedChunkingChunkSize);
+				if (ChunkSize >= (BufferingSize + BufferingSize / 2))
+				{
+					ScanFile(Source.Handle(),
+							 Offset,
+							 ChunkSize,
+							 BufferingSize,
+							 [&FullHasher, &ChunkHasher, &BytesProcessed](const void* Data, size_t Size) {
+								 FullHasher.Append(Data, Size);
+								 ChunkHasher.Append(Data, Size);
+								 BytesProcessed.fetch_add(Size);
+							 });
+				}
+				else
+				{
+					IoBuffer ChunkData = Source.ReadRange(Offset, ChunkSize);
+					FullHasher.Append(ChunkData);
+					ChunkHasher.Append(ChunkData);
+					BytesProcessed.fetch_add(ChunkSize);
+				}
+
+				const IoHash ChunkHash = ChunkHasher.GetHash();
+				if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end())
+				{
+					OutChunked.Info.ChunkSequence.push_back(It->second);
+				}
+				else
+				{
+					uint32_t ChunkIndex = gsl::narrow<uint32_t>(OutChunked.Info.ChunkHashes.size());
+					OutChunked.Info.ChunkHashes.push_back(ChunkHash);
+					OutChunked.Info.ChunkSequence.push_back(ChunkIndex);
+					OutChunked.ChunkSources.push_back({.Offset = Offset, .Size = gsl::narrow<uint32_t>(ChunkSize)});
+				}
+				Offset += ChunkSize;
+			}
+			OutChunked.Info.RawSize = RawSize;
+			OutChunked.Info.RawHash = FullHasher.GetHash();
+			return true;
+		}
+		else
+		{
+			BasicFile Buffer(InputPath, BasicFile::Mode::kRead);
+			if (m_Settings.ExcludeElfFiles && IsElfFile(Buffer))
+			{
+				return false;
+			}
+			if (m_Settings.ExcludeMachOFiles && IsMachOFile(Buffer))
+			{
+				return false;
+			}
+
+			OutChunked = ChunkData(Buffer, 0, RawSize, m_Settings.ChunkingParams, &BytesProcessed, &AbortFlag);
+			return true;
+		}
+	}
+
+	virtual std::string_view GetName() const override { return Name; }
+
+	virtual CbObject GetParameters() const override
+	{
+		CbObjectWriter Writer;
+		Writer.BeginArray("FixedChunkingExtensions");
+		{
+			for (const std::string& Extension : m_Settings.FixedChunkingExtensions)
+			{
+				Writer.AddString(Extension);
+			}
+		}
+		Writer.EndArray();	// ChunkExcludeExtensions
+
+		Writer.BeginArray("ChunkExcludeExtensions"sv);
+		{
+			for (const std::string& Extension : m_Settings.ExcludeExtensions)
+			{
+				Writer.AddString(Extension);
+			}
+		}
+		Writer.EndArray();	// ChunkExcludeExtensions
+
+		Writer.AddBool("ExcludeElfFiles"sv, m_Settings.ExcludeElfFiles);
+		Writer.AddBool("ExcludeMachOFiles"sv, m_Settings.ExcludeMachOFiles);
+
+		Writer.AddInteger("ChunkFileSizeLimit"sv, m_Settings.ChunkFileSizeLimit);
+
+		WriteChunkParams(Writer, m_Settings.ChunkingParams);
+
+		Writer.AddInteger("FixedChunkingChunkSize"sv, m_Settings.FixedChunkingChunkSize);
+		Writer.AddInteger("MinSizeForFixedChunking"sv, m_Settings.MinSizeForFixedChunking);
+		return Writer.Save();
+	}
+
+	static constexpr std::string_view Name = "ChunkingControllerWithFixedChunking"sv;
+
+private:
+	static ChunkingControllerWithFixedChunkingSettings ReadSettings(CbObjectView Parameters)
+	{
+		return ChunkingControllerWithFixedChunkingSettings{
+			.FixedChunkingExtensions = ReadStringArray(Parameters["FixedChunkingExtensions"sv].AsArrayView()),
+			.ExcludeExtensions		 = ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()),
+			.ExcludeElfFiles		 = Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles),
+			.ExcludeMachOFiles		 = Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles),
+			.ChunkFileSizeLimit		 = Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit),
+			.ChunkingParams			 = ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()),
+			.FixedChunkingChunkSize	 = Parameters["FixedChunkingChunkSize"sv].AsUInt64(DefaultFixedChunkingChunkSize),
+			.MinSizeForFixedChunking = Parameters["MinSizeForFixedChunking"sv].AsUInt64(DefaultFixedChunkingChunkSize)};
+	}
+
+	const ChunkingControllerWithFixedChunkingSettings m_Settings;
+};
+
+std::unique_ptr<ChunkingController>
+CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings)
+{
+	return std::make_unique<BasicChunkingController>(Settings);
+}
+std::unique_ptr<ChunkingController>
+CreateBasicChunkingController(CbObjectView Parameters)
+{
+	return std::make_unique<BasicChunkingController>(Parameters);
+}
+
+std::unique_ptr<ChunkingController>
+CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting)
+{
+	return std::make_unique<ChunkingControllerWithFixedChunking>(Setting);
+}
+std::unique_ptr<ChunkingController>
+CreateChunkingControllerWithFixedChunking(CbObjectView Parameters)
+{
+	return std::make_unique<ChunkingControllerWithFixedChunking>(Parameters);
+}
+
+std::unique_ptr<ChunkingController>
+CreateChunkingController(std::string_view Name, CbObjectView Parameters)
+{
+	if (Name == BasicChunkingController::Name)
+	{
+		return CreateBasicChunkingController(Parameters);
+	}
+	else if (Name == ChunkingControllerWithFixedChunking::Name)
+	{
+		return CreateChunkingControllerWithFixedChunking(Parameters);
+	}
+	return {};
+}
+
+}  // namespace zen
diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp
new file mode 100644
index 000000000..afef7f6f2
--- /dev/null
+++ b/src/zenutil/commandlineoptions.cpp
@@ -0,0 +1,221 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/commandlineoptions.h>
+
+#include <filesystem>
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#endif	// ZEN_WITH_TESTS
+
+void
+cxxopts::values::parse_value(const std::string& text, std::filesystem::path& value)
+{
+	value = zen::StringToPath(text);
+}
+
+namespace zen {
+
+std::vector<std::string>
+ParseCommandLine(std::string_view CommandLine)
+{
+	auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) {
+		if (Pos == CommandLine.length())
+		{
+			return true;
+		}
+		if (CommandLine[Pos] == ' ')
+		{
+			return true;
+		}
+		return false;
+	};
+
+	bool IsParsingArg = false;
+	bool IsInQuote	  = false;
+
+	std::string::size_type	 Pos	  = 0;
+	std::string::size_type	 ArgStart = 0;
+	std::vector<std::string> Args;
+	while (Pos < CommandLine.length())
+	{
+		if (IsInQuote)
+		{
+			if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1))
+			{
+				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1)));
+				Pos++;
+				IsInQuote	 = false;
+				IsParsingArg = false;
+			}
+			else
+			{
+				Pos++;
+			}
+		}
+		else if (IsParsingArg)
+		{
+			ZEN_ASSERT(Pos > ArgStart);
+			if (CommandLine[Pos] == ' ')
+			{
+				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart)));
+				Pos++;
+				IsParsingArg = false;
+			}
+			else if (CommandLine[Pos] == '"')
+			{
+				IsInQuote = true;
+				Pos++;
+			}
+			else
+			{
+				Pos++;
+			}
+		}
+		else if (CommandLine[Pos] == '"')
+		{
+			IsInQuote	 = true;
+			IsParsingArg = true;
+			ArgStart	 = Pos;
+			Pos++;
+		}
+		else if (CommandLine[Pos] != ' ')
+		{
+			IsParsingArg = true;
+			ArgStart	 = Pos;
+			Pos++;
+		}
+		else
+		{
+			Pos++;
+		}
+	}
+	if (IsParsingArg)
+	{
+		ZEN_ASSERT(Pos > ArgStart);
+		Args.push_back(std::string(CommandLine.substr(ArgStart)));
+	}
+
+	return Args;
+}
+
+std::vector<char*>
+StripCommandlineQuotes(std::vector<std::string>& InOutArgs)
+{
+	std::vector<char*> RawArgs;
+	RawArgs.reserve(InOutArgs.size());
+	for (std::string& Arg : InOutArgs)
+	{
+		std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1);
+		while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos)
+		{
+			Arg.erase(EscapedQuotePos, 1);
+			EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos);
+		}
+
+		if (Arg.starts_with("\""))
+		{
+			if (Arg.find('"', 1) == Arg.length() - 1)
+			{
+				if (Arg.find(' ', 1) == std::string::npos)
+				{
+					Arg = Arg.substr(1, Arg.length() - 2);
+				}
+			}
+		}
+		RawArgs.push_back(const_cast<char*>(Arg.c_str()));
+	}
+	return RawArgs;
+}
+
+void
+MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path)
+{
+	if (!Path.empty())
+	{
+		std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred();
+#if ZEN_PLATFORM_WINDOWS
+		const std::string_view Prefix = "\\\\?\\";
+		const std::u8string	   PrefixU8(Prefix.begin(), Prefix.end());
+		std::u8string		   PathString = AbsolutePath.u8string();
+		if (!PathString.empty() && !PathString.starts_with(PrefixU8))
+		{
+			PathString.insert(0, PrefixU8);
+			Path = PathString;
+		}
+#endif	// ZEN_PLATFORM_WINDOWS
+	}
+}
+
+std::filesystem::path
+MakeSafeAbsolutePath(const std::filesystem::path& Path)
+{
+	std::filesystem::path Tmp(Path);
+	MakeSafeAbsolutePathÍnPlace(Tmp);
+	return Tmp;
+}
+
+std::filesystem::path
+StringToPath(const std::string_view& Path)
+{
+	std::string_view UnquotedPath = RemoveQuotes(Path);
+
+	if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator))
+	{
+		UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1);
+	}
+
+	return std::filesystem::path(UnquotedPath).make_preferred();
+}
+
+std::string_view
+RemoveQuotes(const std::string_view& Arg)
+{
+	if (Arg.length() > 2)
+	{
+		if (Arg[0] == '"' && Arg[Arg.length() - 1] == '"')
+		{
+			return Arg.substr(1, Arg.length() - 2);
+		}
+	}
+	return Arg;
+}
+
+#if ZEN_WITH_TESTS
+
+void
+commandlineoptions_forcelink()
+{
+}
+
+TEST_CASE("CommandLine")
+{
+	std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\"");
+	CHECK_EQ(v1[0], "c:\\my\\exe.exe");
+	CHECK_EQ(v1[1], "\"quoted arg\"");
+	CHECK_EQ(v1[2], "\"one\",two,\"three\\\"");
+
+	std::vector<std::string> v2 = ParseCommandLine(
+		"--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog "
+		"--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" "
+		"--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" "
+		"--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5");
+
+	std::vector<char*> v2Stripped = StripCommandlineQuotes(v2);
+	CHECK_EQ(v2Stripped[0], std::string("--tracehost"));
+	CHECK_EQ(v2Stripped[1], std::string("127.0.0.1"));
+	CHECK_EQ(v2Stripped[2], std::string("builds"));
+	CHECK_EQ(v2Stripped[3], std::string("download"));
+	CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com"));
+	CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog"));
+	CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows"));
+	CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path"));
+	CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\""));
+	CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\""));
+	CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\""));
+	CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab"));
+	CHECK_EQ(v2Stripped[12], std::string("--build-part-name"));
+	CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5"));
+}
+
+#endif
+}  // namespace zen
diff --git a/src/zenutil/environmentoptions.cpp b/src/zenutil/environmentoptions.cpp
new file mode 100644
index 000000000..1b7ce8029
--- /dev/null
+++ b/src/zenutil/environmentoptions.cpp
@@ -0,0 +1,84 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/environmentoptions.h>
+
+#include <zencore/filesystem.h>
+
+namespace zen {
+
+EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value)
+{
+}
+void
+EnvironmentOptions::StringOption::Parse(std::string_view Value)
+{
+	RefValue = std::string(Value);
+}
+
+EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value)
+{
+}
+void
+EnvironmentOptions::FilePathOption::Parse(std::string_view Value)
+{
+	RefValue = MakeSafeAbsolutePath(Value);
+}
+
+EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value)
+{
+}
+void
+EnvironmentOptions::BoolOption::Parse(std::string_view Value)
+{
+	const std::string Lower = ToLower(Value);
+	if (Lower == "true" || Lower == "y" || Lower == "yes")
+	{
+		RefValue = true;
+	}
+	else if (Lower == "false" || Lower == "n" || Lower == "no")
+	{
+		RefValue = false;
+	}
+}
+
+std::shared_ptr<EnvironmentOptions::OptionValue>
+EnvironmentOptions::MakeOption(std::string& Value)
+{
+	return std::make_shared<StringOption>(Value);
+}
+
+std::shared_ptr<EnvironmentOptions::OptionValue>
+EnvironmentOptions::MakeOption(std::filesystem::path& Value)
+{
+	return std::make_shared<FilePathOption>(Value);
+}
+
+std::shared_ptr<EnvironmentOptions::OptionValue>
+EnvironmentOptions::MakeOption(bool& Value)
+{
+	return std::make_shared<BoolOption>(Value);
+}
+
+EnvironmentOptions::EnvironmentOptions()
+{
+}
+
+void
+EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult)
+{
+	for (auto& It : OptionMap)
+	{
+		std::string_view EnvName = It.first;
+		const Option&	 Opt	 = It.second;
+		if (CmdLineResult.count(Opt.CommandLineOptionName) == 0)
+		{
+			std::string EnvValue = GetEnvVariable(It.first);
+			if (!EnvValue.empty())
+			{
+				Opt.Value->Parse(EnvValue);
+			}
+		}
+	}
+}
+
+}  // namespace zen
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
new file mode 100644
index 000000000..c2cc5ab3c
--- /dev/null
+++ b/src/zenutil/filebuildstorage.cpp
@@ -0,0 +1,726 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/filebuildstorage.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+class FileBuildStorage : public BuildStorage
+{
+public:
+	explicit FileBuildStorage(const std::filesystem::path& StoragePath,
+							  BuildStorage::Statistics&	   Stats,
+							  bool						   EnableJsonOutput,
+							  double					   LatencySec,
+							  double					   DelayPerKBSec)
+	: m_StoragePath(StoragePath)
+	, m_Stats(Stats)
+	, m_EnableJsonOutput(EnableJsonOutput)
+	, m_LatencySec(LatencySec)
+	, m_DelayPerKBSec(DelayPerKBSec)
+	{
+		CreateDirectories(GetBuildsFolder());
+		CreateDirectories(GetBlobsFolder());
+		CreateDirectories(GetBlobsMetadataFolder());
+	}
+
+	virtual ~FileBuildStorage() {}
+
+	virtual CbObject ListNamespaces(bool bRecursive) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::ListNamespaces");
+		ZEN_UNUSED(bRecursive);
+
+		SimulateLatency(0, 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObjectWriter Writer;
+		Writer.BeginArray("results");
+		{
+		}
+		Writer.EndArray();	// results
+		Writer.Save();
+		SimulateLatency(Writer.GetSaveSize(), 0);
+		return Writer.Save();
+	}
+
+	virtual CbObject ListBuilds(CbObject Query) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::ListBuilds");
+		ZEN_UNUSED(Query);
+
+		SimulateLatency(Query.GetSize(), 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildFolder = GetBuildsFolder();
+		DirectoryContent			Content;
+		GetDirectoryContent(BuildFolder, DirectoryContentFlags::IncludeDirs, Content);
+		CbObjectWriter Writer;
+		Writer.BeginArray("results");
+		{
+			for (const std::filesystem::path& BuildPath : Content.Directories)
+			{
+				Oid BuildId = Oid::TryFromHexString(BuildPath.stem().string());
+				if (BuildId != Oid::Zero)
+				{
+					Writer.BeginObject();
+					{
+						Writer.AddObjectId("buildId", BuildId);
+						Writer.AddObject("metadata", ReadBuild(BuildId)["metadata"sv].AsObjectView());
+					}
+					Writer.EndObject();
+				}
+			}
+		}
+		Writer.EndArray();	// results
+		Writer.Save();
+		SimulateLatency(Writer.GetSaveSize(), 0);
+		return Writer.Save();
+	}
+
+	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuild");
+		SimulateLatency(MetaData.GetSize(), 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObjectWriter BuildObject;
+		BuildObject.AddObject("metadata", MetaData);
+		BuildObject.AddInteger("chunkSize"sv, 32u * 1024u * 1024u);
+		WriteBuild(BuildId, BuildObject.Save());
+
+		CbObjectWriter BuildResponse;
+		BuildResponse.AddInteger("chunkSize"sv, 32u * 1024u * 1024u);
+		BuildResponse.Save();
+
+		SimulateLatency(0, BuildResponse.GetSaveSize());
+		return BuildResponse.Save();
+	}
+
+	virtual CbObject GetBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuild");
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObject Build = ReadBuild(BuildId);
+		SimulateLatency(0, Build.GetSize());
+		return Build;
+	}
+
+	virtual void FinalizeBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuild");
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+	}
+
+	virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid&		 BuildId,
+																const Oid&		 BuildPartId,
+																std::string_view PartName,
+																const CbObject&	 MetaData) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildPart");
+		SimulateLatency(MetaData.GetSize(), 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
+		CreateDirectories(BuildPartDataPath.parent_path());
+
+		TemporaryFile::SafeWriteFile(BuildPartDataPath, MetaData.GetView());
+		m_WrittenBytes += MetaData.GetSize();
+		WriteAsJson(BuildPartDataPath, MetaData);
+
+		IoHash RawHash = IoHash::HashBuffer(MetaData.GetView());
+
+		CbObjectWriter Writer;
+		{
+			CbObject	 BuildObject  = ReadBuild(BuildId);
+			CbObjectView PartsObject  = BuildObject["parts"sv].AsObjectView();
+			CbObjectView MetaDataView = BuildObject["metadata"sv].AsObjectView();
+
+			Writer.AddObject("metadata"sv, MetaDataView);
+			Writer.BeginObject("parts"sv);
+			{
+				for (CbFieldView PartView : PartsObject)
+				{
+					if (PartView.GetName() != PartName)
+					{
+						Writer.AddObjectId(PartView.GetName(), PartView.AsObjectId());
+					}
+				}
+				Writer.AddObjectId(PartName, BuildPartId);
+			}
+			Writer.EndObject();	 // parts
+		}
+		WriteBuild(BuildId, Writer.Save());
+
+		std::vector<IoHash> NeededAttachments = GetNeededAttachments(MetaData);
+
+		SimulateLatency(0, sizeof(IoHash) * NeededAttachments.size());
+
+		return std::make_pair(RawHash, std::move(NeededAttachments));
+	}
+
+	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildPart");
+		SimulateLatency(0, 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
+
+		IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten();
+		m_Stats.TotalBytesRead += Payload.GetSize();
+
+		ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None);
+
+		CbObject BuildPartObject = CbObject(SharedBuffer(Payload));
+
+		SimulateLatency(0, BuildPartObject.GetSize());
+
+		return BuildPartObject;
+	}
+
+	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuildPart");
+		SimulateLatency(0, 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
+		IoBuffer					Payload			  = ReadFile(BuildPartDataPath).Flatten();
+		m_Stats.TotalBytesRead += Payload.GetSize();
+		IoHash RawHash = IoHash::HashBuffer(Payload.GetView());
+		if (RawHash != PartHash)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed finalizing build part {}: Expected hash {}, got {}", BuildPartId, PartHash, RawHash));
+		}
+
+		CbObject			BuildPartObject = CbObject(SharedBuffer(Payload));
+		std::vector<IoHash> NeededAttachments(GetNeededAttachments(BuildPartObject));
+
+		SimulateLatency(0, NeededAttachments.size() * sizeof(IoHash));
+
+		return NeededAttachments;
+	}
+
+	virtual void PutBuildBlob(const Oid&			 BuildId,
+							  const IoHash&			 RawHash,
+							  ZenContentType		 ContentType,
+							  const CompositeBuffer& Payload) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildBlob");
+		ZEN_UNUSED(BuildId);
+		ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
+		SimulateLatency(Payload.GetSize(), 0);
+
+		ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, Payload));
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (!IsFile(BlockPath))
+		{
+			CreateDirectories(BlockPath.parent_path());
+			TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView());
+		}
+		m_Stats.TotalBytesWritten += Payload.GetSize();
+		SimulateLatency(0, 0);
+	}
+
+	virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid&												   BuildId,
+																 const IoHash&											   RawHash,
+																 ZenContentType											   ContentType,
+																 uint64_t												   PayloadSize,
+																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
+																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob");
+		ZEN_UNUSED(BuildId);
+		ZEN_UNUSED(ContentType);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (!IsFile(BlockPath))
+		{
+			CreateDirectories(BlockPath.parent_path());
+
+			struct WorkloadData
+			{
+				std::function<IoBuffer(uint64_t Offset, uint64_t Size)> Transmitter;
+				std::function<void(uint64_t, bool)>						OnSentBytes;
+				TemporaryFile											TempFile;
+				std::atomic<size_t>										PartsLeft;
+			};
+
+			std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
+			Workload->Transmitter = std::move(Transmitter);
+			Workload->OnSentBytes = std::move(OnSentBytes);
+			std::error_code Ec;
+			Workload->TempFile.CreateTemporary(BlockPath.parent_path(), Ec);
+
+			if (Ec)
+			{
+				throw std::runtime_error(
+					fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value()));
+			}
+
+			std::vector<std::function<void()>> WorkItems;
+			uint64_t						   Offset = 0;
+			while (Offset < PayloadSize)
+			{
+				uint64_t Size = Min(32u * 1024u * 1024u, PayloadSize - Offset);
+
+				WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob_Work");
+					IoBuffer PartPayload = Workload->Transmitter(Offset, Size);
+					SimulateLatency(PartPayload.GetSize(), 0);
+
+					std::error_code Ec;
+					Workload->TempFile.Write(PartPayload, Offset, Ec);
+					if (Ec)
+					{
+						throw std::runtime_error(fmt::format("Failed writing to temporary file '{}': {} ({})",
+															 Workload->TempFile.GetPath(),
+															 Ec.message(),
+															 Ec.value()));
+					}
+					uint64_t BytesWritten = PartPayload.GetSize();
+					m_Stats.TotalBytesWritten += BytesWritten;
+					const bool IsLastPart = Workload->PartsLeft.fetch_sub(1) == 1;
+					if (IsLastPart)
+					{
+						Workload->TempFile.Flush();
+						ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(Workload->TempFile.ReadAll())));
+						Workload->TempFile.MoveTemporaryIntoPlace(BlockPath, Ec);
+						if (Ec)
+						{
+							throw std::runtime_error(fmt::format("Failed moving temporary file '{}' to '{}': {} ({})",
+																 Workload->TempFile.GetPath(),
+																 BlockPath,
+																 Ec.message(),
+																 Ec.value()));
+						}
+					}
+					Workload->OnSentBytes(BytesWritten, IsLastPart);
+					SimulateLatency(0, 0);
+				});
+
+				Offset += Size;
+			}
+			Workload->PartsLeft.store(WorkItems.size());
+
+			SimulateLatency(0, 0);
+			return WorkItems;
+		}
+		SimulateLatency(0, 0);
+		return {};
+	}
+
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlob");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (IsFile(BlockPath))
+		{
+			BasicFile File(BlockPath, BasicFile::Mode::kRead);
+			IoBuffer  Payload;
+			if (RangeOffset != 0 || RangeBytes != (uint64_t)-1)
+			{
+				Payload = IoBuffer(RangeBytes);
+				File.Read(Payload.GetMutableView().GetData(), RangeBytes, RangeOffset);
+			}
+			else
+			{
+				Payload = File.ReadAll();
+				ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload))));
+			}
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+			m_Stats.TotalBytesRead += Payload.GetSize();
+			SimulateLatency(0, Payload.GetSize());
+			return Payload;
+		}
+		SimulateLatency(0, 0);
+		return IoBuffer{};
+	}
+
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid&													   BuildId,
+																 const IoHash&												   RawHash,
+																 uint64_t													   ChunkSize,
+																 std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+																 std::function<void()>&& OnComplete) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (IsFile(BlockPath))
+		{
+			struct WorkloadData
+			{
+				std::atomic<uint64_t>										BytesRemaining;
+				BasicFile													BlobFile;
+				std::function<void(uint64_t Offset, const IoBuffer& Chunk)> OnReceive;
+				std::function<void()>										OnComplete;
+			};
+
+			std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
+			Workload->BlobFile.Open(BlockPath, BasicFile::Mode::kRead);
+			const uint64_t BlobSize = Workload->BlobFile.FileSize();
+
+			Workload->OnReceive		 = std::move(OnReceive);
+			Workload->OnComplete	 = std::move(OnComplete);
+			Workload->BytesRemaining = BlobSize;
+
+			std::vector<std::function<void()>> WorkItems;
+			uint64_t						   Offset = 0;
+			while (Offset < BlobSize)
+			{
+				uint64_t Size = Min(ChunkSize, BlobSize - Offset);
+				WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob_Work");
+					SimulateLatency(0, 0);
+					IoBuffer PartPayload(Size);
+					Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset);
+					m_Stats.TotalBytesRead += PartPayload.GetSize();
+					Workload->OnReceive(Offset, PartPayload);
+					uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size);
+					if (ByteRemaning == Size)
+					{
+						Workload->OnComplete();
+					}
+					SimulateLatency(Size, PartPayload.GetSize());
+				});
+
+				Offset += Size;
+			}
+			SimulateLatency(0, 0);
+			return WorkItems;
+		}
+		return {};
+	}
+
+	virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata");
+		ZEN_UNUSED(BuildId);
+
+		SimulateLatency(MetaData.GetSize(), 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockMetaDataPath = GetBlobMetadataPath(BlockRawHash);
+		CreateDirectories(BlockMetaDataPath.parent_path());
+		TemporaryFile::SafeWriteFile(BlockMetaDataPath, MetaData.GetView());
+		m_Stats.TotalBytesWritten += MetaData.GetSize();
+		WriteAsJson(BlockMetaDataPath, MetaData);
+		SimulateLatency(0, 0);
+		return true;
+	}
+
+	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::FindBlocks");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(sizeof(BuildId), 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		uint64_t FoundCount = 0;
+
+		DirectoryContent Content;
+		GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content);
+		CbObjectWriter Writer;
+		Writer.BeginArray("blocks");
+		for (const std::filesystem::path& MetaDataFile : Content.Files)
+		{
+			IoHash ChunkHash;
+			if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash))
+			{
+				std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash);
+				if (IsFile(BlockPath))
+				{
+					IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
+
+					m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
+
+					CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
+					Writer.AddObject(BlockObject);
+					FoundCount++;
+					if (FoundCount == MaxBlockCount)
+					{
+						break;
+					}
+				}
+			}
+		}
+		Writer.EndArray();	// blocks
+		CbObject Result = Writer.Save();
+		SimulateLatency(0, Result.GetSize());
+		return Result;
+	}
+
+	virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObjectWriter Writer;
+		Writer.BeginArray("blocks");
+
+		for (const IoHash& BlockHash : BlockHashes)
+		{
+			std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash);
+			if (IsFile(MetaDataFile))
+			{
+				IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
+
+				m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
+
+				CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
+				Writer.AddObject(BlockObject);
+			}
+		}
+		Writer.EndArray();	// blocks
+		CbObject Result = Writer.Save();
+		SimulateLatency(0, Result.GetSize());
+		return Result;
+	}
+
+	virtual void PutBuildPartStats(const Oid&								  BuildId,
+								   const Oid&								  BuildPartId,
+								   const tsl::robin_map<std::string, double>& FloatStats) override
+	{
+		CbObjectWriter Request;
+		Request.BeginObject("floatStats"sv);
+		for (auto It : FloatStats)
+		{
+			Request.AddFloat(It.first, It.second);
+		}
+		Request.EndObject();
+		CbObject Payload = Request.Save();
+
+		SimulateLatency(Payload.GetSize(), 0);
+
+		const std::filesystem::path BuildPartStatsDataPath = GetBuildPartStatsPath(BuildId, BuildPartId);
+		CreateDirectories(BuildPartStatsDataPath.parent_path());
+
+		TemporaryFile::SafeWriteFile(BuildPartStatsDataPath, Payload.GetView());
+		WriteAsJson(BuildPartStatsDataPath, Payload);
+
+		SimulateLatency(0, 0);
+	}
+
+protected:
+	std::filesystem::path GetBuildsFolder() const { return m_StoragePath / "builds"; }
+	std::filesystem::path GetBlobsFolder() const { return m_StoragePath / "blobs"; }
+	std::filesystem::path GetBlobsMetadataFolder() const { return m_StoragePath / "blocks"; }
+	std::filesystem::path GetBuildFolder(const Oid& BuildId) const { return GetBuildsFolder() / BuildId.ToString(); }
+
+	std::filesystem::path GetBuildPath(const Oid& BuildId) const { return GetBuildFolder(BuildId) / "metadata.cb"; }
+
+	std::filesystem::path GetBuildPartFolder(const Oid& BuildId, const Oid& BuildPartId) const
+	{
+		return GetBuildFolder(BuildId) / "parts" / BuildPartId.ToString();
+	}
+
+	std::filesystem::path GetBuildPartPath(const Oid& BuildId, const Oid& BuildPartId) const
+	{
+		return GetBuildPartFolder(BuildId, BuildPartId) / "metadata.cb";
+	}
+
+	std::filesystem::path GetBuildPartStatsPath(const Oid& BuildId, const Oid& BuildPartId) const
+	{
+		return GetBuildPartFolder(BuildId, BuildPartId) / fmt::format("stats_{}.cb", Oid::NewOid());
+	}
+
+	std::filesystem::path GetBlobPayloadPath(const IoHash& RawHash) const { return GetBlobsFolder() / fmt::format("{}.cbz", RawHash); }
+
+	std::filesystem::path GetBlobMetadataPath(const IoHash& RawHash) const
+	{
+		return GetBlobsMetadataFolder() / fmt::format("{}.cb", RawHash);
+	}
+
+	void SimulateLatency(uint64_t ReceiveSize, uint64_t SendSize)
+	{
+		double SleepSec = m_LatencySec;
+		if (m_DelayPerKBSec > 0.0)
+		{
+			SleepSec += m_DelayPerKBSec * (double(SendSize + ReceiveSize) / 1024u);
+		}
+		if (SleepSec > 0)
+		{
+			Sleep(int(SleepSec * 1000));
+		}
+	}
+
+	void WriteAsJson(const std::filesystem::path& OriginalPath, CbObjectView Data) const
+	{
+		if (m_EnableJsonOutput)
+		{
+			ExtendableStringBuilder<128> SB;
+			CompactBinaryToJson(Data, SB);
+			std::filesystem::path JsonPath = OriginalPath;
+			JsonPath.replace_extension(".json");
+			std::string_view JsonMetaData = SB.ToView();
+			TemporaryFile::SafeWriteFile(JsonPath, MemoryView(JsonMetaData.data(), JsonMetaData.length()));
+		}
+	}
+
+	void WriteBuild(const Oid& BuildId, CbObjectView Data)
+	{
+		const std::filesystem::path BuildDataPath = GetBuildPath(BuildId);
+		CreateDirectories(BuildDataPath.parent_path());
+		TemporaryFile::SafeWriteFile(BuildDataPath, Data.GetView());
+		m_Stats.TotalBytesWritten += Data.GetSize();
+		WriteAsJson(BuildDataPath, Data);
+	}
+
+	CbObject ReadBuild(const Oid& BuildId)
+	{
+		const std::filesystem::path BuildDataPath = GetBuildPath(BuildId);
+		FileContents				Content		  = ReadFile(BuildDataPath);
+		if (Content.ErrorCode)
+		{
+			throw std::runtime_error(fmt::format("Failed reading build '{}' from '{}': {} ({})",
+												 BuildId,
+												 BuildDataPath,
+												 Content.ErrorCode.message(),
+												 Content.ErrorCode.value()));
+		}
+		IoBuffer Payload = Content.Flatten();
+		m_Stats.TotalBytesRead += Payload.GetSize();
+		ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None);
+		CbObject BuildObject = CbObject(SharedBuffer(Payload));
+		return BuildObject;
+	}
+
+	std::vector<IoHash> GetNeededAttachments(CbObjectView BuildPartObject)
+	{
+		std::vector<IoHash> NeededAttachments;
+		BuildPartObject.IterateAttachments([&](CbFieldView FieldView) {
+			const IoHash				AttachmentHash = FieldView.AsBinaryAttachment();
+			const std::filesystem::path BlockPath	   = GetBlobPayloadPath(AttachmentHash);
+			if (!IsFile(BlockPath))
+			{
+				NeededAttachments.push_back(AttachmentHash);
+			}
+		});
+		return NeededAttachments;
+	}
+
+	bool ValidateCompressedBuffer(const IoHash& RawHash, const CompositeBuffer& Payload)
+	{
+		IoHash			 VerifyHash;
+		uint64_t		 VerifySize;
+		CompressedBuffer ValidateBuffer = CompressedBuffer::FromCompressed(Payload, VerifyHash, VerifySize);
+		if (!ValidateBuffer)
+		{
+			return false;
+		}
+		if (VerifyHash != RawHash)
+		{
+			return false;
+		}
+
+		IoHashStream Hash;
+		bool		 CouldDecompress = ValidateBuffer.DecompressToStream(
+			0,
+			(uint64_t)-1,
+			[&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+				ZEN_UNUSED(SourceOffset, SourceSize, Offset);
+				for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+				{
+					Hash.Append(Segment.GetView());
+				}
+				return true;
+			});
+		if (!CouldDecompress)
+		{
+			return false;
+		}
+		if (Hash.GetHash() != VerifyHash)
+		{
+			return false;
+		}
+		return true;
+	}
+
+private:
+	const std::filesystem::path m_StoragePath;
+	BuildStorage::Statistics&	m_Stats;
+	const bool					m_EnableJsonOutput = false;
+	std::atomic<uint64_t>		m_WrittenBytes;
+
+	const double m_LatencySec	 = 0.0;
+	const double m_DelayPerKBSec = 0.0;
+};
+
+std::unique_ptr<BuildStorage>
+CreateFileBuildStorage(const std::filesystem::path& StoragePath,
+					   BuildStorage::Statistics&	Stats,
+					   bool							EnableJsonOutput,
+					   double						LatencySec,
+					   double						DelayPerKBSec)
+{
+	return std::make_unique<FileBuildStorage>(StoragePath, Stats, EnableJsonOutput, LatencySec, DelayPerKBSec);
+}
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/bufferedwritefilecache.h b/src/zenutil/include/zenutil/bufferedwritefilecache.h
new file mode 100644
index 000000000..68d6c375e
--- /dev/null
+++ b/src/zenutil/include/zenutil/bufferedwritefilecache.h
@@ -0,0 +1,106 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/basicfile.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+class CompositeBuffer;
+
+class BufferedWriteFileCache
+{
+public:
+	BufferedWriteFileCache(const BufferedWriteFileCache&) = delete;
+	BufferedWriteFileCache& operator=(const BufferedWriteFileCache&) = delete;
+
+	BufferedWriteFileCache();
+
+	~BufferedWriteFileCache();
+
+	std::unique_ptr<BasicFile> Get(uint32_t FileIndex);
+
+	void Put(uint32_t FileIndex, std::unique_ptr<BasicFile>&& Writer);
+
+	void Close(std::span<uint32_t> FileIndexes);
+
+	class Local
+	{
+	public:
+		struct Writer
+		{
+			std::unique_ptr<BasicFile>		 File;
+			std::unique_ptr<BasicFileWriter> Writer;
+
+			inline void Write(const CompositeBuffer& Chunk, uint64_t FileOffset)
+			{
+				if (Writer)
+				{
+					Writer->Write(Chunk, FileOffset);
+				}
+				else
+				{
+					File->Write(Chunk, FileOffset);
+				}
+			}
+		};
+
+		Local(const Local&) = delete;
+		Local& operator=(const Local&) = delete;
+
+		explicit Local(BufferedWriteFileCache& Cache);
+		~Local();
+
+		Writer* GetWriter(uint32_t FileIndex);
+		Writer* PutWriter(uint32_t FileIndex, std::unique_ptr<Writer> Writer);
+
+	private:
+		tsl::robin_map<uint32_t, uint32_t>	 m_FileIndexToWriterIndex;
+		std::vector<std::unique_ptr<Writer>> m_ChunkWriters;
+		BufferedWriteFileCache&				 m_Cache;
+	};
+
+private:
+	static constexpr size_t MaxHandlesPerPath = 7;
+	static constexpr size_t MaxBufferedCount  = 1024;
+	struct TOpenHandles
+	{
+		BasicFile*		  Files[MaxHandlesPerPath];
+		uint64_t		  Size = 0;
+		inline BasicFile* Pop()
+		{
+			if (Size > 0)
+			{
+				return Files[--Size];
+			}
+			else
+			{
+				return nullptr;
+			}
+		}
+		inline bool Push(BasicFile* File)
+		{
+			if (Size < MaxHandlesPerPath)
+			{
+				Files[Size++] = File;
+				return true;
+			}
+			return false;
+		}
+	};
+	static_assert(sizeof(TOpenHandles) == 64);
+
+	RwLock							   m_WriterLock;
+	tsl::robin_map<uint32_t, uint32_t> m_ChunkWriters;
+	std::vector<TOpenHandles>		   m_OpenFiles;
+	std::atomic<uint32_t>			   m_CacheHitCount;
+	std::atomic<uint32_t>			   m_CacheMissCount;
+	std::atomic<uint32_t>			   m_OpenHandleCount;
+	std::atomic<uint32_t>			   m_DroppedHandleCount;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
new file mode 100644
index 000000000..f49d4b42a
--- /dev/null
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -0,0 +1,65 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zenutil/chunkblock.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+class BuildStorage
+{
+public:
+	struct Statistics
+	{
+		std::atomic<uint64_t> TotalBytesRead	   = 0;
+		std::atomic<uint64_t> TotalBytesWritten	   = 0;
+		std::atomic<uint64_t> TotalRequestCount	   = 0;
+		std::atomic<uint64_t> TotalRequestTimeUs   = 0;
+		std::atomic<uint64_t> TotalExecutionTimeUs = 0;
+	};
+
+	virtual ~BuildStorage() {}
+
+	virtual CbObject ListNamespaces(bool bRecursive = false)				= 0;
+	virtual CbObject ListBuilds(CbObject Query)								= 0;
+	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0;
+	virtual CbObject GetBuild(const Oid& BuildId)							= 0;
+	virtual void	 FinalizeBuild(const Oid& BuildId)						= 0;
+
+	virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid&		 BuildId,
+																const Oid&		 BuildPartId,
+																std::string_view PartName,
+																const CbObject&	 MetaData)											 = 0;
+	virtual CbObject							   GetBuildPart(const Oid& BuildId, const Oid& BuildPartId)							 = 0;
+	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash)				 = 0;
+	virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0;
+	virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid&												   BuildId,
+																 const IoHash&											   RawHash,
+																 ZenContentType											   ContentType,
+																 uint64_t												   PayloadSize,
+																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
+																 std::function<void(uint64_t, bool)>&&					   OnSentBytes)					 = 0;
+
+	virtual IoBuffer						   GetBuildBlob(const Oid&	  BuildId,
+															const IoHash& RawHash,
+															uint64_t	  RangeOffset = 0,
+															uint64_t	  RangeBytes  = (uint64_t)-1)		 = 0;
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid&													   BuildId,
+																 const IoHash&												   RawHash,
+																 uint64_t													   ChunkSize,
+																 std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+																 std::function<void()>&& OnComplete) = 0;
+
+	[[nodiscard]] virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0;
+	virtual CbObject		   FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount)									  = 0;
+	virtual CbObject		   GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes)				  = 0;
+
+	virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map<std::string, double>& FloatStats) = 0;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h
new file mode 100644
index 000000000..e1fb73fd4
--- /dev/null
+++ b/src/zenutil/include/zenutil/buildstoragecache.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/compositebuffer.h>
+#include <zenutil/chunkblock.h>
+
+namespace zen {
+
+class HttpClient;
+
+class BuildStorageCache
+{
+public:
+	struct Statistics
+	{
+		std::atomic<uint64_t> TotalBytesRead	   = 0;
+		std::atomic<uint64_t> TotalBytesWritten	   = 0;
+		std::atomic<uint64_t> TotalRequestCount	   = 0;
+		std::atomic<uint64_t> TotalRequestTimeUs   = 0;
+		std::atomic<uint64_t> TotalExecutionTimeUs = 0;
+	};
+
+	virtual ~BuildStorageCache() {}
+
+	virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0;
+	virtual IoBuffer GetBuildBlob(const Oid&	BuildId,
+								  const IoHash& RawHash,
+								  uint64_t		RangeOffset = 0,
+								  uint64_t		RangeBytes	= (uint64_t)-1)																 = 0;
+
+	virtual void PutBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes, std::span<const CbObject> MetaDatas) = 0;
+	virtual std::vector<CbObject> GetBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes)					   = 0;
+
+	struct BlobExistsResult
+	{
+		bool HasBody	 = 0;
+		bool HasMetadata = 0;
+	};
+
+	virtual std::vector<BlobExistsResult> BlobsExists(const Oid& BuildId, std::span<const IoHash> BlobHashes) = 0;
+
+	virtual void Flush(
+		int32_t									  UpdateIntervalMS,
+		std::function<bool(intptr_t Remaining)>&& UpdateCallback = [](intptr_t) { return true; }) = 0;
+};
+
+std::unique_ptr<BuildStorageCache> CreateZenBuildStorageCache(HttpClient&					 HttpClient,
+															  BuildStorageCache::Statistics& Stats,
+															  std::string_view				 Namespace,
+															  std::string_view				 Bucket,
+															  const std::filesystem::path&	 TempFolderPath,
+															  bool							 BoostBackgroundThreadCount);
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/cache/cachekey.h b/src/zenutil/include/zenutil/cache/cachekey.h
index 741375946..0ab05f4f1 100644
--- a/src/zenutil/include/zenutil/cache/cachekey.h
+++ b/src/zenutil/include/zenutil/cache/cachekey.h
@@ -17,6 +17,12 @@ struct CacheKey
 
 	static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; }
 
+	// This should be used whenever the bucket name has already been validated to avoid redundant ToLower calls
+	static CacheKey CreateValidated(std::string&& BucketValidated, const IoHash& Hash)
+	{
+		return {.Bucket = std::move(BucketValidated), .Hash = Hash};
+	}
+
 	auto operator<=>(const CacheKey& that) const
 	{
 		if (auto b = caseSensitiveCompareStrings(Bucket, that.Bucket); b != std::strong_ordering::equal)
diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h
new file mode 100644
index 000000000..277580c74
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkblock.h
@@ -0,0 +1,40 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/compress.h>
+
+#include <optional>
+#include <vector>
+
+namespace zen {
+
+struct ThinChunkBlockDescription
+{
+	IoHash				BlockHash;
+	std::vector<IoHash> ChunkRawHashes;
+};
+
+struct ChunkBlockDescription : public ThinChunkBlockDescription
+{
+	uint64_t			  HeaderSize;
+	std::vector<uint32_t> ChunkRawLengths;
+	std::vector<uint32_t> ChunkCompressedLengths;
+};
+
+std::vector<ChunkBlockDescription> ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject);
+ChunkBlockDescription			   ParseChunkBlockDescription(const CbObjectView& BlockObject);
+CbObject						   BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData);
+ChunkBlockDescription			   GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash);
+typedef std::function<std::pair<uint64_t, CompressedBuffer>(const IoHash& RawHash)> FetchChunkFunc;
+
+CompressedBuffer	  GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock);
+bool				  IterateChunkBlock(const SharedBuffer&															BlockPayload,
+										std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
+										uint64_t&																	OutHeaderSize);
+std::vector<uint32_t> ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize);
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
new file mode 100644
index 000000000..306a5d990
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkedcontent.h
@@ -0,0 +1,288 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/iohash.h>
+
+#include <filesystem>
+#include <vector>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+class CbWriter;
+class ChunkingController;
+class WorkerThreadPool;
+
+enum class SourcePlatform
+{
+	Windows = 0,
+	Linux	= 1,
+	MacOS	= 2,
+	_Count
+};
+
+std::string_view ToString(SourcePlatform Platform);
+SourcePlatform	 FromString(std::string_view Platform, SourcePlatform Default);
+SourcePlatform	 GetSourceCurrentPlatform();
+
+struct FolderContent
+{
+	SourcePlatform					   Platform = GetSourceCurrentPlatform();
+	std::vector<std::filesystem::path> Paths;
+	std::vector<uint64_t>			   RawSizes;
+	std::vector<uint32_t>			   Attributes;
+	std::vector<uint64_t>			   ModificationTicks;
+
+	bool operator==(const FolderContent& Rhs) const;
+
+	bool		AreKnownFilesEqual(const FolderContent& Rhs) const;
+	void		UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& PathIndexesOufOfDate);
+	static bool AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs);
+};
+
+FolderContent GetUpdatedContent(const FolderContent&				Old,
+								const FolderContent&				New,
+								std::vector<std::filesystem::path>& OutDeletedPathIndexes);
+
+void		  SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output);
+FolderContent LoadFolderContentToCompactBinary(CbObjectView Input);
+
+struct GetFolderContentStatistics
+{
+	std::atomic<uint64_t> FoundFileCount		= 0;
+	std::atomic<uint64_t> FoundFileByteCount	= 0;
+	std::atomic<uint64_t> AcceptedFileCount		= 0;
+	std::atomic<uint64_t> AcceptedFileByteCount = 0;
+	uint64_t			  ElapsedWallTimeUS		= 0;
+};
+
+FolderContent GetFolderContent(GetFolderContentStatistics&																Stats,
+							   const std::filesystem::path&																RootPath,
+							   std::function<bool(const std::string_view& RelativePath)>&&								AcceptDirectory,
+							   std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile,
+							   WorkerThreadPool&																		WorkerPool,
+							   int32_t																					UpdateIntervalMS,
+							   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						UpdateCallback,
+							   std::atomic<bool>&																		AbortFlag);
+
+struct ChunkedContentData
+{
+	// To describe one asset with a particular RawHash, find the index of the hash in SequenceRawHashes
+	// ChunkCounts for that index will be the number of indexes in ChunkOrders that describe
+	// the sequence of chunks required to reconstruct the asset.
+	// Offset into ChunkOrders is based on how many entries in ChunkOrders the previous [n - 1] SequenceRawHashes uses
+	std::vector<IoHash>	  SequenceRawHashes;  // Raw hash for Chunk sequence
+	std::vector<uint32_t> ChunkCounts;		  // Chunk count of ChunkOrder for SequenceRawHashes[n]
+	std::vector<uint32_t> ChunkOrders;		  // Chunk sequence indexed into ChunkHashes, ChunkCounts[n] indexes per SequenceRawHashes[n]
+	std::vector<IoHash>	  ChunkHashes;		  // Unique chunk hashes
+	std::vector<uint64_t> ChunkRawSizes;	  // Unique chunk raw size for ChunkHash[n]
+};
+
+struct ChunkedFolderContent
+{
+	SourcePlatform					   Platform = GetSourceCurrentPlatform();
+	std::vector<std::filesystem::path> Paths;
+	std::vector<uint64_t>			   RawSizes;
+	std::vector<uint32_t>			   Attributes;
+	std::vector<IoHash>				   RawHashes;
+	ChunkedContentData				   ChunkedContent;
+};
+
+struct ChunkedContentLookup
+{
+	struct ChunkSequenceLocation
+	{
+		uint32_t SequenceIndex = (uint32_t)-1;
+		uint64_t Offset		   = (uint64_t)-1;
+	};
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex;
+	std::vector<uint32_t>							 SequenceIndexChunkOrderOffset;
+	std::vector<ChunkSequenceLocation>				 ChunkSequenceLocations;
+	std::vector<size_t>
+		ChunkSequenceLocationOffset;  // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex
+	std::vector<uint32_t> ChunkSequenceLocationCounts;	// ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex
+	std::vector<uint32_t> SequenceIndexFirstPathIndex;	// SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash
+	std::vector<uint32_t> PathExtensionHash;
+};
+
+void				 SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output);
+ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input);
+
+ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays);
+ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent&			  Base,
+												   const ChunkedContentLookup&			  BaseContentLookup,
+												   std::span<const std::filesystem::path> DeletedPaths);
+ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span<const std::filesystem::path> DeletedPaths);
+
+struct ChunkingStatistics
+{
+	std::atomic<uint64_t> FilesProcessed	   = 0;
+	std::atomic<uint64_t> FilesChunked		   = 0;
+	std::atomic<uint64_t> BytesHashed		   = 0;
+	std::atomic<uint64_t> UniqueChunksFound	   = 0;
+	std::atomic<uint64_t> UniqueSequencesFound = 0;
+	std::atomic<uint64_t> UniqueBytesFound	   = 0;
+	uint64_t			  ElapsedWallTimeUS	   = 0;
+};
+
+ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&			 Stats,
+										WorkerThreadPool&			 WorkerPool,
+										const std::filesystem::path& RootPath,
+										const FolderContent&		 Content,
+										const ChunkingController&	 InChunkingController,
+										int32_t						 UpdateIntervalMS,
+										std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+										std::atomic<bool>&																 AbortFlag,
+										std::atomic<bool>&																 PauseFlag);
+
+ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content);
+
+inline std::pair<size_t, uint32_t>
+GetChunkSequenceLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
+{
+	return std::make_pair(Lookup.ChunkSequenceLocationOffset[ChunkIndex], Lookup.ChunkSequenceLocationCounts[ChunkIndex]);
+}
+
+inline std::span<const ChunkedContentLookup::ChunkSequenceLocation>
+GetChunkSequenceLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
+{
+	std::pair<size_t, uint32_t> Range = GetChunkSequenceLocationRange(Lookup, ChunkIndex);
+	return std::span<const ChunkedContentLookup::ChunkSequenceLocation>(Lookup.ChunkSequenceLocations).subspan(Range.first, Range.second);
+}
+
+inline uint32_t
+GetSequenceIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	return Lookup.RawHashToSequenceIndex.at(RawHash);
+}
+
+inline uint32_t
+GetChunkIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	return Lookup.RawHashToSequenceIndex.at(RawHash);
+}
+
+inline uint32_t
+GetFirstPathIndexForSeqeuenceIndex(const ChunkedContentLookup& Lookup, const uint32_t SequenceIndex)
+{
+	return Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
+}
+
+inline uint32_t
+GetFirstPathIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	const uint32_t SequenceIndex = GetSequenceIndexForRawHash(Lookup, RawHash);
+	return GetFirstPathIndexForSeqeuenceIndex(Lookup, SequenceIndex);
+}
+
+namespace compactbinary_helpers {
+	template<typename Type>
+	void WriteArray(std::span<const Type> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const Type Value : Values)
+		{
+			Output << Value;
+		}
+		Output.EndArray();
+	}
+
+	template<typename Type>
+	void WriteArray(const std::vector<Type>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const Type>(Values), ArrayName, Output);
+	}
+
+	template<>
+	inline void WriteArray(std::span<const std::filesystem::path> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const std::filesystem::path& Path : Values)
+		{
+			Output.AddString((const char*)Path.generic_u8string().c_str());
+		}
+		Output.EndArray();
+	}
+
+	template<>
+	inline void WriteArray(const std::vector<std::filesystem::path>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const std::filesystem::path>(Values), ArrayName, Output);
+	}
+
+	inline void WriteBinaryAttachmentArray(std::span<const IoHash> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const IoHash& Hash : Values)
+		{
+			Output.AddBinaryAttachment(Hash);
+		}
+		Output.EndArray();
+	}
+
+	inline void WriteBinaryAttachmentArray(const std::vector<IoHash>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const IoHash>(Values), ArrayName, Output);
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint32_t>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsUInt32());
+		}
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint64_t>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsUInt64());
+		}
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<std::filesystem::path>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			std::u8string_view U8Path = ItemView.AsU8String();
+			Result.push_back(std::filesystem::path(U8Path));
+		}
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsHash());
+		}
+	}
+
+	inline void ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsBinaryAttachment());
+		}
+	}
+
+}  // namespace compactbinary_helpers
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h
new file mode 100644
index 000000000..4cec80fdb
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkedfile.h
@@ -0,0 +1,59 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/zencore.h>
+
+#include <functional>
+#include <vector>
+
+namespace zen {
+
+class BasicFile;
+
+struct ChunkedInfo
+{
+	uint64_t			  RawSize = 0;
+	IoHash				  RawHash;
+	std::vector<uint32_t> ChunkSequence;
+	std::vector<IoHash>	  ChunkHashes;
+};
+
+struct ChunkSource
+{
+	uint64_t Offset;  // 8
+	uint32_t Size;	  // 4
+};
+
+struct ChunkedInfoWithSource
+{
+	ChunkedInfo				 Info;
+	std::vector<ChunkSource> ChunkSources;
+};
+
+struct ChunkedParams
+{
+	bool   UseThreshold = true;
+	size_t MinSize		= (2u * 1024u) - 128u;
+	size_t MaxSize		= (16u * 1024u);
+	size_t AvgSize		= (3u * 1024u);
+};
+
+static const ChunkedParams UShaderByteCodeParams = {.UseThreshold = true, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340};
+
+ChunkedInfoWithSource ChunkData(BasicFile&			   RawData,
+								uint64_t			   Offset,
+								uint64_t			   Size,
+								ChunkedParams		   Params		  = {},
+								std::atomic<uint64_t>* BytesProcessed = nullptr,
+								std::atomic<bool>*	   AbortFlag	  = nullptr);
+void				  Reconstruct(const ChunkedInfo&							   Info,
+								  const std::filesystem::path&					   TargetPath,
+								  std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk);
+IoBuffer			  SerializeChunkedInfo(const ChunkedInfo& Info);
+ChunkedInfo			  DeserializeChunkedInfo(IoBuffer& Buffer);
+
+void chunkedfile_forcelink();
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h
new file mode 100644
index 000000000..315502265
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkingcontroller.h
@@ -0,0 +1,75 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+
+#include <zenutil/chunkedfile.h>
+
+#include <atomic>
+#include <filesystem>
+
+namespace zen {
+
+const std::vector<std::string> DefaultChunkingExcludeExtensions =
+	{".exe", ".dll", ".pdb", ".self", ".mp4", ".zip", ".7z", ".bzip", ".rar", ".gzip"};
+const std::vector<std::string> DefaultFixedChunkingExtensions	= {".apk", ".nsp", ".xvc", ".pkg", ".dmg", ".ipa"};
+const bool					   DefaultChunkingExcludeElfFiles	= true;
+const bool					   DefaultChunkingExcludeMachOFiles = true;
+
+const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u,
+											.MaxSize = 128u * 1024u,
+											.AvgSize = ((8u * 4u) * 1024u) + 128u};
+
+const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize;
+
+const uint64_t DefaultFixedChunkingChunkSize  = 32u * 1024u * 1024u;
+const uint64_t DefaultMinSizeForFixedChunking = DefaultFixedChunkingChunkSize * 8u;
+
+struct ChunkedInfoWithSource;
+
+class ChunkingController
+{
+public:
+	virtual ~ChunkingController() {}
+
+	// Return true if the input file was processed. If true is returned OutChunked will contain the chunked info
+	virtual bool			 ProcessFile(const std::filesystem::path& InputPath,
+										 uint64_t					  RawSize,
+										 ChunkedInfoWithSource&		  OutChunked,
+										 std::atomic<uint64_t>&		  BytesProcessed,
+										 std::atomic<bool>&			  AbortFlag) const = 0;
+	virtual std::string_view GetName() const								 = 0;
+	virtual CbObject		 GetParameters() const							 = 0;
+};
+
+struct BasicChunkingControllerSettings
+{
+	std::vector<std::string> ExcludeExtensions	= DefaultChunkingExcludeExtensions;
+	bool					 ExcludeElfFiles	= DefaultChunkingExcludeElfFiles;
+	bool					 ExcludeMachOFiles	= DefaultChunkingExcludeMachOFiles;
+	uint64_t				 ChunkFileSizeLimit = DefaultChunkingFileSizeLimit;
+	ChunkedParams			 ChunkingParams		= DefaultChunkedParams;
+};
+
+std::unique_ptr<ChunkingController> CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings);
+std::unique_ptr<ChunkingController> CreateBasicChunkingController(CbObjectView Parameters);
+
+struct ChunkingControllerWithFixedChunkingSettings
+{
+	std::vector<std::string> FixedChunkingExtensions = DefaultFixedChunkingExtensions;
+	std::vector<std::string> ExcludeExtensions		 = DefaultChunkingExcludeExtensions;
+	bool					 ExcludeElfFiles		 = DefaultChunkingExcludeElfFiles;
+	bool					 ExcludeMachOFiles		 = DefaultChunkingExcludeMachOFiles;
+	uint64_t				 ChunkFileSizeLimit		 = DefaultChunkingFileSizeLimit;
+	ChunkedParams			 ChunkingParams			 = DefaultChunkedParams;
+	uint64_t				 FixedChunkingChunkSize	 = DefaultFixedChunkingChunkSize;
+	uint64_t				 MinSizeForFixedChunking = DefaultMinSizeForFixedChunking;
+};
+
+std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting);
+std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(CbObjectView Parameters);
+
+std::unique_ptr<ChunkingController> CreateChunkingController(std::string_view Name, CbObjectView Parameters);
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h
new file mode 100644
index 000000000..f927d41e5
--- /dev/null
+++ b/src/zenutil/include/zenutil/commandlineoptions.h
@@ -0,0 +1,29 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+#include <filesystem>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+
+namespace cxxopts::values {
+// We declare this specialization before including cxxopts to make it stick
+void parse_value(const std::string& text, std::filesystem::path& value);
+}  // namespace cxxopts::values
+
+#include <cxxopts.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+std::vector<std::string>			ParseCommandLine(std::string_view CommandLine);
+std::vector<char*>					StripCommandlineQuotes(std::vector<std::string>& InOutArgs);
+void								MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path);
+[[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path);
+std::filesystem::path				StringToPath(const std::string_view& Path);
+std::string_view					RemoveQuotes(const std::string_view& Arg);
+
+void commandlineoptions_forcelink();  // internal
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h
new file mode 100644
index 000000000..7418608e4
--- /dev/null
+++ b/src/zenutil/include/zenutil/environmentoptions.h
@@ -0,0 +1,92 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/string.h>
+#include <zenutil/commandlineoptions.h>
+
+namespace zen {
+
+class EnvironmentOptions
+{
+public:
+	class OptionValue
+	{
+	public:
+		virtual void Parse(std::string_view Value) = 0;
+
+		virtual ~OptionValue() {}
+	};
+
+	class StringOption : public OptionValue
+	{
+	public:
+		explicit StringOption(std::string& Value);
+		virtual void Parse(std::string_view Value) override;
+		std::string& RefValue;
+	};
+
+	class FilePathOption : public OptionValue
+	{
+	public:
+		explicit FilePathOption(std::filesystem::path& Value);
+		virtual void		   Parse(std::string_view Value) override;
+		std::filesystem::path& RefValue;
+	};
+
+	class BoolOption : public OptionValue
+	{
+	public:
+		explicit BoolOption(bool& Value);
+		virtual void Parse(std::string_view Value);
+		bool&		 RefValue;
+	};
+
+	template<Integral T>
+	class NumberOption : public OptionValue
+	{
+	public:
+		explicit NumberOption(T& Value) : RefValue(Value) {}
+		virtual void Parse(std::string_view Value) override
+		{
+			if (std::optional<T> OptionalValue = ParseInt<T>(Value); OptionalValue.has_value())
+			{
+				RefValue = OptionalValue.value();
+			}
+		}
+		T& RefValue;
+	};
+
+	struct Option
+	{
+		std::string					 CommandLineOptionName;
+		std::shared_ptr<OptionValue> Value;
+	};
+
+	std::shared_ptr<OptionValue> MakeOption(std::string& Value);
+	std::shared_ptr<OptionValue> MakeOption(std::filesystem::path& Value);
+
+	template<Integral T>
+	std::shared_ptr<OptionValue> MakeOption(T& Value)
+	{
+		return std::make_shared<NumberOption<T>>(Value);
+	};
+
+	std::shared_ptr<OptionValue> MakeOption(bool& Value);
+
+	template<typename T>
+	void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "")
+	{
+		OptionMap.insert_or_assign(std::string(EnvName),
+								   Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)});
+	};
+
+	EnvironmentOptions();
+
+	void Parse(const cxxopts::ParseResult& CmdLineResult);
+
+private:
+	std::unordered_map<std::string, Option> OptionMap;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/filebuildstorage.h b/src/zenutil/include/zenutil/filebuildstorage.h
new file mode 100644
index 000000000..c95fb32e6
--- /dev/null
+++ b/src/zenutil/include/zenutil/filebuildstorage.h
@@ -0,0 +1,16 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+#include <zenutil/buildstorage.h>
+
+namespace zen {
+class HttpClient;
+
+std::unique_ptr<BuildStorage> CreateFileBuildStorage(const std::filesystem::path& StoragePath,
+													 BuildStorage::Statistics&	  Stats,
+													 bool						  EnableJsonOutput,
+													 double						  LatencySec	= 0.0,
+													 double						  DelayPerKBSec = 0.0);
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h
new file mode 100644
index 000000000..bbf070993
--- /dev/null
+++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h
@@ -0,0 +1,18 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+#include <zenutil/buildstorage.h>
+
+namespace zen {
+class HttpClient;
+
+std::unique_ptr<BuildStorage> CreateJupiterBuildStorage(LoggerRef					 InLog,
+														HttpClient&					 InHttpClient,
+														BuildStorage::Statistics&	 Stats,
+														std::string_view			 Namespace,
+														std::string_view			 Bucket,
+														bool						 AllowRedirect,
+														const std::filesystem::path& TempFolderPath);
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupiterclient.h b/src/zenutil/include/zenutil/jupiter/jupiterclient.h
index defe50edc..8a51bd60a 100644
--- a/src/zenutil/include/zenutil/jupiter/jupiterclient.h
+++ b/src/zenutil/include/zenutil/jupiter/jupiterclient.h
@@ -44,12 +44,11 @@ public:
 	HttpClient& Client() { return m_HttpClient; }
 
 private:
-	LoggerRef							   m_Log;
-	const std::string					   m_DefaultDdcNamespace;
-	const std::string					   m_DefaultBlobStoreNamespace;
-	const std::string					   m_ComputeCluster;
-	std::function<HttpClientAccessToken()> m_TokenProvider;
-	HttpClient							   m_HttpClient;
+	LoggerRef		  m_Log;
+	const std::string m_DefaultDdcNamespace;
+	const std::string m_DefaultBlobStoreNamespace;
+	const std::string m_ComputeCluster;
+	HttpClient		  m_HttpClient;
 
 	friend class JupiterSession;
 };
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
index 6a80332f4..b79790f25 100644
--- a/src/zenutil/include/zenutil/jupiter/jupitersession.h
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -65,7 +65,7 @@ struct FinalizeBuildPartResult : JupiterResult
 class JupiterSession
 {
 public:
-	JupiterSession(LoggerRef InLog, HttpClient& InHttpClient);
+	JupiterSession(LoggerRef InLog, HttpClient& InHttpClient, bool AllowRedirect);
 	~JupiterSession();
 
 	JupiterResult Authenticate();
@@ -102,33 +102,52 @@ public:
 
 	std::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes);
 
-	JupiterResult			PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload);
-	JupiterResult			GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
-	JupiterResult			FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
-	PutBuildPartResult		PutBuildPart(std::string_view Namespace,
-										 std::string_view BucketId,
-										 const Oid&		  BuildId,
-										 const Oid&		  PartId,
-										 std::string_view PartName,
-										 const IoBuffer&  Payload);
-	JupiterResult			GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
-	JupiterResult			PutBuildBlob(std::string_view		Namespace,
-										 std::string_view		BucketId,
-										 const Oid&				BuildId,
-										 const Oid&				PartId,
-										 const IoHash&			Hash,
-										 ZenContentType			ContentType,
-										 const CompositeBuffer& Payload);
-	JupiterResult			GetBuildBlob(std::string_view	   Namespace,
-										 std::string_view	   BucketId,
-										 const Oid&			   BuildId,
-										 const Oid&			   PartId,
-										 const IoHash&		   Hash,
-										 std::filesystem::path TempFolderPath);
+	JupiterResult	   ListBuildNamespaces();
+	JupiterResult	   ListBuildBuckets(std::string_view Namespace);
+	JupiterResult	   ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload);
+	JupiterResult	   PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload);
+	JupiterResult	   GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
+	JupiterResult	   FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
+	PutBuildPartResult PutBuildPart(std::string_view Namespace,
+									std::string_view BucketId,
+									const Oid&		 BuildId,
+									const Oid&		 PartId,
+									std::string_view PartName,
+									const IoBuffer&	 Payload);
+	JupiterResult	   GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
+	JupiterResult	   PutBuildBlob(std::string_view	   Namespace,
+									std::string_view	   BucketId,
+									const Oid&			   BuildId,
+									const IoHash&		   Hash,
+									ZenContentType		   ContentType,
+									const CompositeBuffer& Payload);
+	JupiterResult	   GetBuildBlob(std::string_view	  Namespace,
+									std::string_view	  BucketId,
+									const Oid&			  BuildId,
+									const IoHash&		  Hash,
+									std::filesystem::path TempFolderPath,
+									uint64_t			  Offset = 0,
+									uint64_t			  Size	 = (uint64_t)-1);
+
+	JupiterResult			PutMultipartBuildBlob(std::string_view												  Namespace,
+												  std::string_view												  BucketId,
+												  const Oid&													  BuildId,
+												  const IoHash&													  Hash,
+												  ZenContentType												  ContentType,
+												  uint64_t														  PayloadSize,
+												  std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&&		  Transmitter,
+												  std::vector<std::function<JupiterResult(bool& OutIsComplete)>>& OutWorkItems);
+	JupiterResult			GetMultipartBuildBlob(std::string_view												Namespace,
+												  std::string_view												BucketId,
+												  const Oid&													BuildId,
+												  const IoHash&													Hash,
+												  uint64_t														ChunkSize,
+												  std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+												  std::function<void()>&&										OnComplete,
+												  std::vector<std::function<JupiterResult()>>&					OutWorkItems);
 	JupiterResult			PutBlockMetadata(std::string_view Namespace,
 											 std::string_view BucketId,
 											 const Oid&		  BuildId,
-											 const Oid&		  PartId,
 											 const IoHash&	  Hash,
 											 const IoBuffer&  Payload);
 	FinalizeBuildPartResult FinalizeBuildPart(std::string_view Namespace,
@@ -136,7 +155,14 @@ public:
 											  const Oid&	   BuildId,
 											  const Oid&	   PartId,
 											  const IoHash&	   RawHash);
-	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
+	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount);
+	JupiterResult			GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload);
+
+	JupiterResult PutBuildPartStats(std::string_view Namespace,
+									std::string_view BucketId,
+									const Oid&		 BuildId,
+									const Oid&		 BuildPartId,
+									IoBuffer		 Payload);
 
 private:
 	inline LoggerRef Log() { return m_Log; }
@@ -147,6 +173,7 @@ private:
 
 	LoggerRef	m_Log;
 	HttpClient& m_HttpClient;
+	const bool	m_AllowRedirect = false;
 };
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/logging.h b/src/zenutil/include/zenutil/logging.h
index ebf6372fc..d64eef207 100644
--- a/src/zenutil/include/zenutil/logging.h
+++ b/src/zenutil/include/zenutil/logging.h
@@ -32,6 +32,7 @@ struct LoggingOptions
 	bool				  IsDebug		  = false;
 	bool				  IsVerbose		  = false;
 	bool				  IsTest		  = false;
+	bool				  AllowAsync	  = true;
 	bool				  NoConsoleOutput = false;
 	std::filesystem::path AbsLogFile;  // Absolute path to main log file
 	std::string			  LogId;
diff --git a/src/zenutil/include/zenutil/logging/fullformatter.h b/src/zenutil/include/zenutil/logging/fullformatter.h
index 07ad408fa..0326870e5 100644
--- a/src/zenutil/include/zenutil/logging/fullformatter.h
+++ b/src/zenutil/include/zenutil/logging/fullformatter.h
@@ -45,6 +45,8 @@ public:
 
 		std::chrono::seconds TimestampSeconds;
 
+		std::chrono::milliseconds millis;
+
 		if (m_UseFullDate)
 		{
 			TimestampSeconds = std::chrono::duration_cast<std::chrono::seconds>(msg.time.time_since_epoch());
@@ -69,6 +71,8 @@ public:
 				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime);
 				m_CachedDatetime.push_back('.');
 			}
+
+			millis = spdlog::details::fmt_helper::time_fraction<std::chrono::milliseconds>(msg.time);
 		}
 		else
 		{
@@ -97,6 +101,8 @@ public:
 				spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime);
 				m_CachedDatetime.push_back('.');
 			}
+
+			millis = std::chrono::duration_cast<std::chrono::milliseconds>(ElapsedTime - TimestampSeconds);
 		}
 
 		{
@@ -104,7 +110,6 @@ public:
 			OutBuffer.append(m_CachedDatetime.begin(), m_CachedDatetime.end());
 		}
 
-		auto millis = spdlog::details::fmt_helper::time_fraction<std::chrono::milliseconds>(msg.time);
 		spdlog::details::fmt_helper::pad3(static_cast<uint32_t>(millis.count()), OutBuffer);
 		OutBuffer.push_back(']');
 		OutBuffer.push_back(' ');
diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h
index 758722156..cd28bdcb2 100644
--- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h
+++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h
@@ -27,7 +27,6 @@ public:
 	{
 		ZEN_MEMSCOPE(ELLMTag::Logging);
 
-		ZEN_MEMSCOPE(ELLMTag::Logging);
 		std::error_code Ec;
 		if (RotateOnOpen)
 		{
diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h
new file mode 100644
index 000000000..639c6968c
--- /dev/null
+++ b/src/zenutil/include/zenutil/parallelwork.h
@@ -0,0 +1,77 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/scopeguard.h>
+#include <zencore/thread.h>
+#include <zencore/workthreadpool.h>
+
+#include <atomic>
+
+namespace zen {
+
+class ParallelWork
+{
+public:
+	ParallelWork(std::atomic<bool>& AbortFlag, std::atomic<bool>& PauseFlag);
+
+	~ParallelWork();
+
+	typedef std::function<void(std::atomic<bool>& AbortFlag)>							   WorkCallback;
+	typedef std::function<void(std::exception_ptr Ex, std::atomic<bool>& AbortFlag)>	   ExceptionCallback;
+	typedef std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)> UpdateCallback;
+
+	void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {})
+	{
+		m_PendingWork.AddCount(1);
+		try
+		{
+			WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] {
+				auto _ = MakeGuard([this]() { m_PendingWork.CountDown(); });
+				try
+				{
+					while (m_PauseFlag && !m_AbortFlag)
+					{
+						Sleep(2000);
+					}
+					Work(m_AbortFlag);
+				}
+				catch (...)
+				{
+					OnError(std::current_exception(), m_AbortFlag);
+				}
+			});
+		}
+		catch (const std::exception&)
+		{
+			m_PendingWork.CountDown();
+			throw;
+		}
+	}
+
+	void Abort() { m_AbortFlag = true; }
+
+	bool IsAborted() const { return m_AbortFlag.load(); }
+
+	void Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback);
+
+	void Wait();
+
+	Latch& PendingWork() { return m_PendingWork; }
+
+private:
+	ExceptionCallback DefaultErrorFunction();
+	void			  RethrowErrors();
+
+	std::atomic<bool>& m_AbortFlag;
+	std::atomic<bool>& m_PauseFlag;
+	bool			   m_DispatchComplete = false;
+	Latch			   m_PendingWork;
+
+	RwLock							m_ErrorLock;
+	std::vector<std::exception_ptr> m_Errors;
+};
+
+void parallellwork_forcelink();
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h
index 9683ad720..df2033bca 100644
--- a/src/zenutil/include/zenutil/workerpools.h
+++ b/src/zenutil/include/zenutil/workerpools.h
@@ -21,6 +21,9 @@ WorkerThreadPool& GetMediumWorkerPool(EWorkloadType WorkloadType);
 // Worker pool with std::thread::hardware_concurrency() / 8 worker threads, but at least one thread
 WorkerThreadPool& GetSmallWorkerPool(EWorkloadType WorkloadType);
 
+// Worker pool with minimum number of worker threads, but at least one thread
+WorkerThreadPool& GetTinyWorkerPool(EWorkloadType WorkloadType);
+
 // Special worker pool that does not use worker thread but issues all scheduled work on the calling thread
 // This is useful for debugging when multiple async thread can make stepping in debugger complicated
 WorkerThreadPool& GetSyncWorkerPool();
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
new file mode 100644
index 000000000..9974725ff
--- /dev/null
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -0,0 +1,479 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/jupiter/jupiterbuildstorage.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zenutil/jupiter/jupitersession.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+class JupiterBuildStorage : public BuildStorage
+{
+public:
+	JupiterBuildStorage(LoggerRef					 InLog,
+						HttpClient&					 InHttpClient,
+						Statistics&					 Stats,
+						std::string_view			 Namespace,
+						std::string_view			 Bucket,
+						bool						 AllowRedirect,
+						const std::filesystem::path& TempFolderPath)
+	: m_Session(InLog, InHttpClient, AllowRedirect)
+	, m_Stats(Stats)
+	, m_Namespace(Namespace)
+	, m_Bucket(Bucket)
+	, m_TempFolderPath(TempFolderPath)
+	{
+	}
+	virtual ~JupiterBuildStorage() {}
+
+	virtual CbObject ListNamespaces(bool bRecursive) override
+	{
+		ZEN_TRACE_CPU("Jupiter::ListNamespaces");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult ListResult = m_Session.ListBuildNamespaces();
+		AddStatistic(ListResult);
+		if (!ListResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed listing namespaces: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+		}
+		CbObject NamespaceResponse = PayloadToCbObject("Failed listing namespaces"sv, ListResult.Response);
+
+		CbObjectWriter Response;
+		Response.BeginArray("results"sv);
+		for (CbFieldView NamespaceField : NamespaceResponse["namespaces"])
+		{
+			std::string_view Namespace = NamespaceField.AsString();
+			if (!Namespace.empty())
+			{
+				Response.BeginObject();
+				Response.AddString("name", Namespace);
+
+				if (bRecursive)
+				{
+					JupiterResult BucketsResult = m_Session.ListBuildBuckets(Namespace);
+					AddStatistic(BucketsResult);
+					if (!BucketsResult.Success)
+					{
+						throw std::runtime_error(
+							fmt::format("Failed listing namespaces: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode));
+					}
+					CbObject BucketResponse = PayloadToCbObject("Failed listing namespaces"sv, BucketsResult.Response);
+
+					Response.BeginArray("items");
+					for (CbFieldView BucketField : BucketResponse["buckets"])
+					{
+						std::string_view Bucket = BucketField.AsString();
+						if (!Bucket.empty())
+						{
+							Response.AddString(Bucket);
+						}
+					}
+					Response.EndArray();
+				}
+
+				Response.EndObject();
+			}
+		}
+		Response.EndArray();
+
+		return Response.Save();
+	}
+
+	virtual CbObject ListBuilds(CbObject Query) override
+	{
+		ZEN_TRACE_CPU("Jupiter::ListBuilds");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = Query.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult ListResult = m_Session.ListBuilds(m_Namespace, m_Bucket, Payload);
+		AddStatistic(ListResult);
+		if (!ListResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+		}
+		return PayloadToCbObject("Failed listing builds"sv, ListResult.Response);
+	}
+
+	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBuild");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult PutResult = m_Session.PutBuild(m_Namespace, m_Bucket, BuildId, Payload);
+		AddStatistic(PutResult);
+		if (!PutResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode));
+		}
+		return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
+	}
+
+	virtual CbObject GetBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBuild");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _				 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult GetBuildResult = m_Session.GetBuild(m_Namespace, m_Bucket, BuildId);
+		AddStatistic(GetBuildResult);
+		if (!GetBuildResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode));
+		}
+		return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
+	}
+
+	virtual void FinalizeBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuild");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _					  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult FinalizeBuildResult = m_Session.FinalizeBuild(m_Namespace, m_Bucket, BuildId);
+		AddStatistic(FinalizeBuildResult);
+		if (!FinalizeBuildResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed finalizing build part: {} ({})", FinalizeBuildResult.Reason, FinalizeBuildResult.ErrorCode));
+		}
+	}
+
+	virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid&		 BuildId,
+																const Oid&		 BuildPartId,
+																std::string_view PartName,
+																const CbObject&	 MetaData) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildPart");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		PutBuildPartResult PutPartResult = m_Session.PutBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartName, Payload);
+		AddStatistic(PutPartResult);
+		if (!PutPartResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed creating build part: {} ({})", PutPartResult.Reason, PutPartResult.ErrorCode));
+		}
+		return std::make_pair(PutPartResult.RawHash, std::move(PutPartResult.Needs));
+	}
+
+	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildPart");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult GetBuildPartResult = m_Session.GetBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId);
+		AddStatistic(GetBuildPartResult);
+		if (!GetBuildPartResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed fetching build part {}: {} ({})",
+												 BuildPartId,
+												 GetBuildPartResult.Reason,
+												 GetBuildPartResult.ErrorCode));
+		}
+		return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
+	}
+
+	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
+	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuildPart");
+
+		Stopwatch				ExecutionTimer;
+		auto					_ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		FinalizeBuildPartResult FinalizePartResult = m_Session.FinalizeBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartHash);
+		AddStatistic(FinalizePartResult);
+		if (!FinalizePartResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed finalizing build part: {} ({})", FinalizePartResult.Reason, FinalizePartResult.ErrorCode));
+		}
+		return std::move(FinalizePartResult.Needs);
+	}
+
+	virtual void PutBuildBlob(const Oid&			 BuildId,
+							  const IoHash&			 RawHash,
+							  ZenContentType		 ContentType,
+							  const CompositeBuffer& Payload) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildBlob");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _				= MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult PutBlobResult = m_Session.PutBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ContentType, Payload);
+		AddStatistic(PutBlobResult);
+		if (!PutBlobResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PutBlobResult.Reason, PutBlobResult.ErrorCode));
+		}
+	}
+
+	virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid&												   BuildId,
+																 const IoHash&											   RawHash,
+																 ZenContentType											   ContentType,
+																 uint64_t												   PayloadSize,
+																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
+																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutLargeBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		std::vector<std::function<JupiterResult(bool&)>> WorkItems;
+		JupiterResult									 PutMultipartBlobResult = m_Session.PutMultipartBuildBlob(m_Namespace,
+																				  m_Bucket,
+																				  BuildId,
+																				  RawHash,
+																				  ContentType,
+																				  PayloadSize,
+																				  std::move(Transmitter),
+																				  WorkItems);
+		AddStatistic(PutMultipartBlobResult);
+		if (!PutMultipartBlobResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed putting build part: {} ({})", PutMultipartBlobResult.Reason, PutMultipartBlobResult.ErrorCode));
+		}
+		OnSentBytes(PutMultipartBlobResult.SentBytes, WorkItems.empty());
+
+		std::vector<std::function<void()>> WorkList;
+		for (auto& WorkItem : WorkItems)
+		{
+			WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes]() {
+				Stopwatch	  ExecutionTimer;
+				auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+				bool		  IsComplete = false;
+				JupiterResult PartResult = WorkItem(IsComplete);
+				AddStatistic(PartResult);
+				if (!PartResult.Success)
+				{
+					throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode));
+				}
+				OnSentBytes(PartResult.SentBytes, IsComplete);
+			});
+		}
+		return WorkList;
+	}
+
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		CreateDirectories(m_TempFolderPath);
+		JupiterResult GetBuildBlobResult =
+			m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, RangeOffset, RangeBytes);
+		AddStatistic(GetBuildBlobResult);
+		if (!GetBuildBlobResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed fetching build blob {}: {} ({})", RawHash, GetBuildBlobResult.Reason, GetBuildBlobResult.ErrorCode));
+		}
+		return std::move(GetBuildBlobResult.Response);
+	}
+
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid&													   BuildId,
+																 const IoHash&												   RawHash,
+																 uint64_t													   ChunkSize,
+																 std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+																 std::function<void()>&& OnComplete) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		std::vector<std::function<JupiterResult()>> WorkItems;
+		JupiterResult								GetMultipartBlobResult = m_Session.GetMultipartBuildBlob(m_Namespace,
+																				 m_Bucket,
+																				 BuildId,
+																				 RawHash,
+																				 ChunkSize,
+																				 std::move(OnReceive),
+																				 std::move(OnComplete),
+																				 WorkItems);
+
+		AddStatistic(GetMultipartBlobResult);
+		if (!GetMultipartBlobResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed getting build part: {} ({})", GetMultipartBlobResult.Reason, GetMultipartBlobResult.ErrorCode));
+		}
+		std::vector<std::function<void()>> WorkList;
+		for (auto& WorkItem : WorkItems)
+		{
+			WorkList.emplace_back([this, WorkItem = std::move(WorkItem)]() {
+				Stopwatch	  ExecutionTimer;
+				auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+				JupiterResult PartResult = WorkItem();
+				AddStatistic(PartResult);
+				if (!PartResult.Success)
+				{
+					throw std::runtime_error(fmt::format("Failed getting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode));
+				}
+			});
+		}
+		return WorkList;
+	}
+
+	virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBlockMetadata");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult PutMetaResult = m_Session.PutBlockMetadata(m_Namespace, m_Bucket, BuildId, BlockRawHash, Payload);
+		AddStatistic(PutMetaResult);
+		if (!PutMetaResult.Success)
+		{
+			if (PutMetaResult.ErrorCode == int32_t(HttpResponseCode::NotFound))
+			{
+				return false;
+			}
+			throw std::runtime_error(
+				fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode));
+		}
+		return true;
+	}
+
+	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override
+	{
+		ZEN_TRACE_CPU("Jupiter::FindBlocks");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId, MaxBlockCount);
+		AddStatistic(FindResult);
+		if (!FindResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode));
+		}
+		return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response);
+	}
+
+	virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBlockMetadata");
+
+		Stopwatch	   ExecutionTimer;
+		auto		   _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		CbObjectWriter Request;
+
+		Request.BeginArray("blocks"sv);
+		for (const IoHash& BlockHash : BlockHashes)
+		{
+			Request.AddHash(BlockHash);
+		}
+		Request.EndArray();
+
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult GetBlockMetadataResult = m_Session.GetBlockMetadata(m_Namespace, m_Bucket, BuildId, Payload);
+		AddStatistic(GetBlockMetadataResult);
+		if (!GetBlockMetadataResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode));
+		}
+		return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response);
+	}
+
+	virtual void PutBuildPartStats(const Oid&								  BuildId,
+								   const Oid&								  BuildPartId,
+								   const tsl::robin_map<std::string, double>& FloatStats) override
+	{
+		ZEN_UNUSED(BuildId, BuildPartId, FloatStats);
+		CbObjectWriter Request;
+		Request.BeginObject("floatStats"sv);
+		for (auto It : FloatStats)
+		{
+			Request.AddFloat(It.first, It.second);
+		}
+		Request.EndObject();
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult PutBuildPartStatsResult = m_Session.PutBuildPartStats(m_Namespace, m_Bucket, BuildId, BuildPartId, Payload);
+		AddStatistic(PutBuildPartStatsResult);
+		if (!PutBuildPartStatsResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed posting build part statistics: {} ({})",
+												 PutBuildPartStatsResult.Reason,
+												 PutBuildPartStatsResult.ErrorCode));
+		}
+	}
+
+private:
+	static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload)
+	{
+		if (Payload.GetContentType() == ZenContentType::kJSON)
+		{
+			std::string_view Json(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize());
+			return LoadCompactBinaryFromJson(Json).AsObject();
+		}
+		else if (Payload.GetContentType() == ZenContentType::kCbObject)
+		{
+			return LoadCompactBinaryObject(Payload);
+		}
+		else if (Payload.GetContentType() == ZenContentType::kCompressedBinary)
+		{
+			IoHash	 RawHash;
+			uint64_t RawSize;
+			return LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize));
+		}
+		else
+		{
+			throw std::runtime_error(
+				fmt::format("{}: {} ({})", "Unsupported response format", Context, ToString(Payload.GetContentType())));
+		}
+	}
+
+	void AddStatistic(const JupiterResult& Result)
+	{
+		m_Stats.TotalBytesWritten += Result.SentBytes;
+		m_Stats.TotalBytesRead += Result.ReceivedBytes;
+		m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0);
+		m_Stats.TotalRequestCount++;
+	}
+
+	JupiterSession				m_Session;
+	Statistics&					m_Stats;
+	const std::string			m_Namespace;
+	const std::string			m_Bucket;
+	const std::filesystem::path m_TempFolderPath;
+};
+
+std::unique_ptr<BuildStorage>
+CreateJupiterBuildStorage(LoggerRef					   InLog,
+						  HttpClient&				   InHttpClient,
+						  BuildStorage::Statistics&	   Stats,
+						  std::string_view			   Namespace,
+						  std::string_view			   Bucket,
+						  bool						   AllowRedirect,
+						  const std::filesystem::path& TempFolderPath)
+{
+	ZEN_TRACE_CPU("CreateJupiterBuildStorage");
+
+	return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, AllowRedirect, TempFolderPath);
+}
+
+}  // namespace zen
diff --git a/src/zenutil/jupiter/jupiterclient.cpp b/src/zenutil/jupiter/jupiterclient.cpp
index 5e5da3750..dbac218a4 100644
--- a/src/zenutil/jupiter/jupiterclient.cpp
+++ b/src/zenutil/jupiter/jupiterclient.cpp
@@ -11,7 +11,6 @@ JupiterClient::JupiterClient(const JupiterClientOptions& Options, std::function<
 , m_DefaultDdcNamespace(Options.DdcNamespace)
 , m_DefaultBlobStoreNamespace(Options.BlobStoreNamespace)
 , m_ComputeCluster(Options.ComputeCluster)
-, m_TokenProvider(std::move(TokenProvider))
 , m_HttpClient(Options.ServiceUrl,
 			   HttpClientSettings{.ConnectTimeout	   = Options.ConnectTimeout,
 								  .Timeout			   = Options.Timeout,
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index f706a7efc..1fd59acdf 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -3,6 +3,9 @@
 #include <zenutil/jupiter/jupitersession.h>
 
 #include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compositebuffer.h>
+#include <zencore/compress.h>
 #include <zencore/fmtutils.h>
 #include <zencore/trace.h>
 
@@ -46,7 +49,10 @@ namespace detail {
 	}
 }  // namespace detail
 
-JupiterSession::JupiterSession(LoggerRef InLog, HttpClient& InHttpClient) : m_Log(InLog), m_HttpClient(InHttpClient)
+JupiterSession::JupiterSession(LoggerRef InLog, HttpClient& InHttpClient, bool AllowRedirect)
+: m_Log(InLog)
+, m_HttpClient(InHttpClient)
+, m_AllowRedirect(AllowRedirect)
 {
 }
 
@@ -355,6 +361,32 @@ JupiterSession::CacheTypeExists(std::string_view Namespace, std::string_view Typ
 }
 
 JupiterResult
+JupiterSession::ListBuildNamespaces()
+{
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds"), {HttpClient::Accept(ZenContentType::kJSON)});
+	return detail::ConvertResponse(Response, "JupiterSession::ListBuildNamespaces"sv);
+}
+
+JupiterResult
+JupiterSession::ListBuildBuckets(std::string_view Namespace)
+{
+	HttpClient::Response Response =
+		m_HttpClient.Get(fmt::format("/api/v2/builds/{}", Namespace), {HttpClient::Accept(ZenContentType::kJSON)});
+	return detail::ConvertResponse(Response, "JupiterSession::ListBuildBuckets"sv);
+}
+
+JupiterResult
+JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+	std::string			 OptionalBucketPath = BucketId.empty() ? "" : fmt::format("/{}", BucketId);
+	HttpClient::Response Response			= m_HttpClient.Post(fmt::format("/api/v2/builds/{}{}/search", Namespace, OptionalBucketPath),
+														Payload,
+														{HttpClient::Accept(ZenContentType::kCbObject)});
+	return detail::ConvertResponse(Response, "JupiterSession::ListBuilds"sv);
+}
+
+JupiterResult
 JupiterSession::PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
@@ -424,29 +456,330 @@ JupiterResult
 JupiterSession::PutBuildBlob(std::string_view		Namespace,
 							 std::string_view		BucketId,
 							 const Oid&				BuildId,
-							 const Oid&				PartId,
 							 const IoHash&			Hash,
 							 ZenContentType			ContentType,
 							 const CompositeBuffer& Payload)
 {
-	HttpClient::Response Response = m_HttpClient.Upload(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		Payload,
-		ContentType);
+	HttpClient::Response Response =
+		m_HttpClient.Upload(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
+							Payload,
+							ContentType);
 	return detail::ConvertResponse(Response, "JupiterSession::PutBuildBlob"sv);
 }
 
 JupiterResult
+JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
+									  std::string_view												  BucketId,
+									  const Oid&													  BuildId,
+									  const IoHash&													  Hash,
+									  ZenContentType												  ContentType,
+									  uint64_t														  PayloadSize,
+									  std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&&		  Transmitter,
+									  std::vector<std::function<JupiterResult(bool& OutIsComplete)>>& OutWorkItems)
+{
+	struct MultipartUploadResponse
+	{
+		struct Part
+		{
+			uint64_t	FirstByte;
+			uint64_t	LastByte;
+			std::string PartId;
+			std::string QueryString;
+		};
+
+		std::string		  UploadId;
+		std::string		  BlobName;
+		std::vector<Part> Parts;
+
+		static MultipartUploadResponse Parse(CbObject& Payload)
+		{
+			MultipartUploadResponse Result;
+			Result.UploadId		   = Payload["uploadId"sv].AsString();
+			Result.BlobName		   = Payload["blobName"sv].AsString();
+			CbArrayView PartsArray = Payload["parts"sv].AsArrayView();
+			Result.Parts.reserve(PartsArray.Num());
+			for (CbFieldView PartView : PartsArray)
+			{
+				CbObjectView PartObject = PartView.AsObjectView();
+				Result.Parts.emplace_back(Part{
+					.FirstByte	 = PartObject["firstByte"sv].AsUInt64(),
+					.LastByte	 = PartObject["lastByte"sv].AsUInt64(),
+					.PartId		 = std::string(PartObject["partId"sv].AsString()),
+					.QueryString = std::string(PartObject["queryString"sv].AsString()),
+				});
+			}
+			return Result;
+		}
+	};
+
+	CbObjectWriter StartMultipartPayloadWriter;
+	StartMultipartPayloadWriter.AddInteger("blobLength"sv, PayloadSize);
+	CbObject StartMultipartPayload = StartMultipartPayloadWriter.Save();
+
+	std::string StartMultipartResponseRequestString =
+		fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/startMultipartUpload", Namespace, BucketId, BuildId, Hash.ToHexString());
+	// ZEN_INFO("POST: {}", StartMultipartResponseRequestString);
+	HttpClient::Response StartMultipartResponse =
+		m_HttpClient.Post(StartMultipartResponseRequestString, StartMultipartPayload, HttpClient::Accept(ZenContentType::kCbObject));
+	if (!StartMultipartResponse.IsSuccess())
+	{
+		ZEN_WARN("{}", StartMultipartResponse.ErrorMessage("startMultipartUpload: "));
+		return detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv);
+	}
+	CbObject ResponseObject = LoadCompactBinaryObject(StartMultipartResponse.ResponsePayload);
+
+	struct WorkloadData
+	{
+		MultipartUploadResponse									PartDescription;
+		std::function<IoBuffer(uint64_t Offset, uint64_t Size)> Transmitter;
+		std::atomic<size_t>										PartsLeft;
+	};
+
+	std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
+
+	Workload->PartDescription = MultipartUploadResponse::Parse(ResponseObject);
+	Workload->Transmitter	  = std::move(Transmitter);
+	Workload->PartsLeft		  = Workload->PartDescription.Parts.size();
+
+	for (size_t PartIndex = 0; PartIndex < Workload->PartDescription.Parts.size(); PartIndex++)
+	{
+		OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, Hash, ContentType, Workload, PartIndex](
+									  bool& OutIsComplete) -> JupiterResult {
+			const MultipartUploadResponse::Part& Part		 = Workload->PartDescription.Parts[PartIndex];
+			IoBuffer							 PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte);
+			std::string							 MultipartUploadResponseRequestString =
+				fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}&supportsRedirect={}",
+							Namespace,
+							BucketId,
+							BuildId,
+							Hash.ToHexString(),
+							Part.QueryString,
+							m_AllowRedirect ? "true"sv : "false"sv);
+			// ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString);
+			HttpClient::Response MultipartUploadResponse = m_HttpClient.Put(MultipartUploadResponseRequestString, PartPayload);
+			if (!MultipartUploadResponse.IsSuccess())
+			{
+				ZEN_WARN("{}", MultipartUploadResponse.ErrorMessage(MultipartUploadResponseRequestString));
+			}
+			OutIsComplete = Workload->PartsLeft.fetch_sub(1) == 1;
+			if (OutIsComplete)
+			{
+				int64_t				 TotalUploadedBytes	  = MultipartUploadResponse.UploadedBytes;
+				int64_t				 TotalDownloadedBytes = MultipartUploadResponse.DownloadedBytes;
+				double				 TotalElapsedSeconds  = MultipartUploadResponse.ElapsedSeconds;
+				HttpClient::Response MultipartEndResponse = MultipartUploadResponse;
+				while (MultipartEndResponse.IsSuccess())
+				{
+					CbObjectWriter CompletePayloadWriter;
+					CompletePayloadWriter.AddString("blobName"sv, Workload->PartDescription.BlobName);
+					CompletePayloadWriter.AddString("uploadId"sv, Workload->PartDescription.UploadId);
+					CompletePayloadWriter.AddBool("isCompressed"sv, ContentType == ZenContentType::kCompressedBinary);
+					CompletePayloadWriter.BeginArray("partIds"sv);
+					std::unordered_map<std::string, size_t> PartNameToIndex;
+					for (size_t UploadPartIndex = 0; UploadPartIndex < Workload->PartDescription.Parts.size(); UploadPartIndex++)
+					{
+						const MultipartUploadResponse::Part& PartDescription = Workload->PartDescription.Parts[UploadPartIndex];
+						PartNameToIndex.insert({PartDescription.PartId, UploadPartIndex});
+						CompletePayloadWriter.AddString(PartDescription.PartId);
+					}
+					CompletePayloadWriter.EndArray();  // "partIds"
+					CbObject CompletePayload = CompletePayloadWriter.Save();
+
+					std::string MultipartEndResponseRequestString =
+						fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/completeMultipart", Namespace, BucketId, BuildId, Hash.ToHexString());
+
+					MultipartEndResponse = m_HttpClient.Post(MultipartEndResponseRequestString,
+															 CompletePayload,
+															 HttpClient::Accept(ZenContentType::kCbObject));
+					TotalUploadedBytes += MultipartEndResponse.UploadedBytes;
+					TotalDownloadedBytes += MultipartEndResponse.DownloadedBytes;
+					TotalElapsedSeconds += MultipartEndResponse.ElapsedSeconds;
+					if (MultipartEndResponse.IsSuccess())
+					{
+						CbObject	ResponseObject		  = MultipartEndResponse.AsObject();
+						CbArrayView MissingPartsArrayView = ResponseObject["missingParts"sv].AsArrayView();
+						if (MissingPartsArrayView.Num() == 0)
+						{
+							break;
+						}
+						else
+						{
+							for (CbFieldView PartIdView : MissingPartsArrayView)
+							{
+								std::string							 RetryPartId(PartIdView.AsString());
+								size_t								 RetryPartIndex = PartNameToIndex.at(RetryPartId);
+								const MultipartUploadResponse::Part& RetryPart		= Workload->PartDescription.Parts[RetryPartIndex];
+								IoBuffer							 RetryPartPayload =
+									Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte - 1);
+								std::string RetryMultipartUploadResponseRequestString =
+									fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}&supportsRedirect={}",
+												Namespace,
+												BucketId,
+												BuildId,
+												Hash.ToHexString(),
+												RetryPart.QueryString,
+												m_AllowRedirect ? "true"sv : "false"sv);
+
+								MultipartUploadResponse = m_HttpClient.Put(RetryMultipartUploadResponseRequestString, RetryPartPayload);
+								TotalUploadedBytes		= MultipartUploadResponse.UploadedBytes;
+								TotalDownloadedBytes	= MultipartUploadResponse.DownloadedBytes;
+								TotalElapsedSeconds		= MultipartUploadResponse.ElapsedSeconds;
+								if (!MultipartUploadResponse.IsSuccess())
+								{
+									ZEN_WARN("{}", MultipartUploadResponse.ErrorMessage(RetryMultipartUploadResponseRequestString));
+									MultipartEndResponse = MultipartUploadResponse;
+								}
+							}
+						}
+					}
+					else
+					{
+						ZEN_WARN("{}", MultipartEndResponse.ErrorMessage(MultipartEndResponseRequestString));
+					}
+				}
+				MultipartEndResponse.UploadedBytes	 = TotalUploadedBytes;
+				MultipartEndResponse.DownloadedBytes = TotalDownloadedBytes;
+				MultipartEndResponse.ElapsedSeconds	 = TotalElapsedSeconds;
+				return detail::ConvertResponse(MultipartEndResponse, "JupiterSession::PutMultipartBuildBlob"sv);
+			}
+			return detail::ConvertResponse(MultipartUploadResponse, "JupiterSession::PutMultipartBuildBlob"sv);
+		});
+	}
+	return detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv);
+}
+
+JupiterResult
+JupiterSession::GetMultipartBuildBlob(std::string_view												Namespace,
+									  std::string_view												BucketId,
+									  const Oid&													BuildId,
+									  const IoHash&													Hash,
+									  uint64_t														ChunkSize,
+									  std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+									  std::function<void()>&&										OnComplete,
+									  std::vector<std::function<JupiterResult()>>&					OutWorkItems)
+{
+	std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}",
+										  Namespace,
+										  BucketId,
+										  BuildId,
+										  Hash.ToHexString(),
+										  m_AllowRedirect ? "true"sv : "false"sv);
+	HttpClient::Response Response =
+		m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}}));
+	if (Response.IsSuccess())
+	{
+		if (std::string_view ContentRange = Response.Header.Entries["Content-Range"]; !ContentRange.empty())
+		{
+			if (std::string_view::size_type SizeDelimiterPos = ContentRange.find('/'); SizeDelimiterPos != std::string_view::npos)
+			{
+				if (std::optional<uint64_t> TotalSizeMaybe = ParseInt<uint64_t>(ContentRange.substr(SizeDelimiterPos + 1));
+					TotalSizeMaybe.has_value())
+				{
+					uint64_t TotalSize	 = TotalSizeMaybe.value();
+					uint64_t PayloadSize = Response.ResponsePayload.GetSize();
+
+					OnReceive(0, Response.ResponsePayload);
+
+					if (TotalSize > PayloadSize)
+					{
+						struct WorkloadData
+						{
+							std::function<void(uint64_t Offset, const IoBuffer& Chunk)> OnReceive;
+							std::function<void()>										OnComplete;
+							std::atomic<uint64_t>										BytesRemaining;
+						};
+
+						std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
+						Workload->OnReceive		 = std::move(OnReceive);
+						Workload->OnComplete	 = std::move(OnComplete);
+						Workload->BytesRemaining = TotalSize - PayloadSize;
+
+						uint64_t Offset = PayloadSize;
+						while (Offset < TotalSize)
+						{
+							uint64_t PartSize = Min(ChunkSize, TotalSize - Offset);
+							OutWorkItems.emplace_back([this,
+													   Namespace = std::string(Namespace),
+													   BucketId	 = std::string(BucketId),
+													   BuildId	 = Oid(BuildId),
+													   Hash		 = IoHash(Hash),
+													   TotalSize,
+													   Workload,
+													   Offset,
+													   PartSize]() -> JupiterResult {
+								std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}",
+																	  Namespace,
+																	  BucketId,
+																	  BuildId,
+																	  Hash.ToHexString(),
+																	  m_AllowRedirect ? "true"sv : "false"sv);
+								HttpClient::Response Response	= m_HttpClient.Get(
+									  RequestUrl,
+									  HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
+								if (Response.IsSuccess())
+								{
+									Workload->OnReceive(Offset, Response.ResponsePayload);
+									uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize());
+									if (ByteRemaning == Response.ResponsePayload.GetSize())
+									{
+										Workload->OnComplete();
+									}
+								}
+								return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv);
+							});
+							Offset += PartSize;
+						}
+					}
+					else
+					{
+						OnComplete();
+					}
+					return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv);
+				}
+			}
+		}
+		OnReceive(0, Response.ResponsePayload);
+		OnComplete();
+	}
+	return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv);
+}
+
+JupiterResult
 JupiterSession::GetBuildBlob(std::string_view	   Namespace,
 							 std::string_view	   BucketId,
 							 const Oid&			   BuildId,
-							 const Oid&			   PartId,
 							 const IoHash&		   Hash,
-							 std::filesystem::path TempFolderPath)
+							 std::filesystem::path TempFolderPath,
+							 uint64_t			   Offset,
+							 uint64_t			   Size)
 {
-	HttpClient::Response Response = m_HttpClient.Download(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		TempFolderPath);
+	HttpClient::KeyValueMap Headers;
+	if (Offset != 0 || Size != (uint64_t)-1)
+	{
+		Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", Offset, Offset + Size - 1)});
+	}
+	HttpClient::Response Response = m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}",
+																	  Namespace,
+																	  BucketId,
+																	  BuildId,
+																	  Hash.ToHexString(),
+																	  m_AllowRedirect ? "true"sv : "false"sv),
+														  TempFolderPath,
+														  Headers);
+	if (Response.IsSuccess())
+	{
+		// If we get a redirect to S3 or a non-Jupiter endpoint the content type will not be correct, validate it and set it
+		if (m_AllowRedirect && (Response.ResponsePayload.GetContentType() == HttpContentType::kBinary))
+		{
+			IoHash	 ValidateRawHash;
+			uint64_t ValidateRawSize = 0;
+			ZEN_ASSERT_SLOW(CompressedBuffer::ValidateCompressedHeader(Response.ResponsePayload, ValidateRawHash, ValidateRawSize));
+			ZEN_ASSERT_SLOW(ValidateRawHash == Hash);
+			ZEN_ASSERT_SLOW(ValidateRawSize > 0);
+			ZEN_UNUSED(ValidateRawHash, ValidateRawSize);
+			Response.ResponsePayload.SetContentType(ZenContentType::kCompressedBinary);
+		}
+	}
 	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
 }
 
@@ -454,14 +787,13 @@ JupiterResult
 JupiterSession::PutBlockMetadata(std::string_view Namespace,
 								 std::string_view BucketId,
 								 const Oid&		  BuildId,
-								 const Oid&		  PartId,
 								 const IoHash&	  Hash,
 								 const IoBuffer&  Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
-	HttpClient::Response Response = m_HttpClient.Put(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		Payload);
+	HttpClient::Response Response =
+		m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, Hash.ToHexString()),
+						 Payload);
 	return detail::ConvertResponse(Response, "JupiterSession::PutBlockMetadata"sv);
 }
 
@@ -494,12 +826,39 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace,
 }
 
 JupiterResult
-JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId)
+JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount)
 {
+	const std::string	 Parameters = MaxBlockCount == (uint64_t)-1 ? "" : fmt::format("?count={}", MaxBlockCount);
 	HttpClient::Response Response =
-		m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/listBlocks", Namespace, BucketId, BuildId, PartId),
+		m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks{}", Namespace, BucketId, BuildId, Parameters),
 						 HttpClient::Accept(ZenContentType::kCbObject));
 	return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv);
 }
 
+JupiterResult
+JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+	HttpClient::Response Response =
+		m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId),
+						  Payload,
+						  HttpClient::Accept(ZenContentType::kCbObject));
+	return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv);
+}
+
+JupiterResult
+JupiterSession::PutBuildPartStats(std::string_view Namespace,
+								  std::string_view BucketId,
+								  const Oid&	   BuildId,
+								  const Oid&	   BuildPartId,
+								  IoBuffer		   Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+	HttpClient::Response Response =
+		m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/stats", Namespace, BucketId, BuildId, BuildPartId),
+						 Payload,
+						 HttpClient::Accept(ZenContentType::kCbObject));
+	return detail::ConvertResponse(Response, "JupiterSession::PutBuildPartStats"sv);
+}
+
 }  // namespace zen
diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp
index 6314c407f..cb0fd6679 100644
--- a/src/zenutil/logging.cpp
+++ b/src/zenutil/logging.cpp
@@ -10,11 +10,13 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <spdlog/spdlog.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#include <zencore/callstack.h>
 #include <zencore/compactbinary.h>
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
 #include <zencore/memory/llm.h>
 #include <zencore/string.h>
+#include <zencore/timer.h>
 #include <zenutil/logging/fullformatter.h>
 #include <zenutil/logging/jsonformatter.h>
 #include <zenutil/logging/rotatingfilesink.h>
@@ -48,7 +50,7 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
 	zen::logging::InitializeLogging();
 	zen::logging::EnableVTMode();
 
-	bool IsAsync = true;
+	bool IsAsync = LogOptions.AllowAsync;
 
 	if (LogOptions.IsDebug)
 	{
@@ -97,7 +99,13 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
 		}
 	}
 
-	std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); });
+	std::set_terminate([]() {
+		void*			 Frames[8];
+		uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+		CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+		ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, "  "));
+		FreeCallstack(Callstack);
+	});
 
 	// Default
 
@@ -174,7 +182,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 		LogLevel = logging::level::Debug;
 	}
 
-	if (LogOptions.IsTest)
+	if (LogOptions.IsTest || LogOptions.IsVerbose)
 	{
 		LogLevel = logging::level::Trace;
 	}
@@ -184,21 +192,25 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 	logging::RefreshLogLevels(LogLevel);
 	spdlog::flush_on(spdlog::level::err);
 	spdlog::flush_every(std::chrono::seconds{2});
-	spdlog::set_formatter(
-		std::make_unique<logging::full_formatter>(LogOptions.LogId, std::chrono::system_clock::now()));	 // default to duration prefix
+	spdlog::set_formatter(std::make_unique<logging::full_formatter>(
+		LogOptions.LogId,
+		std::chrono::system_clock::now() - std::chrono::milliseconds(GetTimeSinceProcessStart())));	 // default to duration prefix
 
-	if (LogOptions.AbsLogFile.extension() == ".json")
+	if (g_FileSink)
 	{
-		g_FileSink->set_formatter(std::make_unique<logging::json_formatter>(LogOptions.LogId));
-	}
-	else
-	{
-		g_FileSink->set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId));	 // this will have a date prefix
-	}
+		if (LogOptions.AbsLogFile.extension() == ".json")
+		{
+			g_FileSink->set_formatter(std::make_unique<logging::json_formatter>(LogOptions.LogId));
+		}
+		else
+		{
+			g_FileSink->set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId));	 // this will have a date prefix
+		}
 
-	const std::string StartLogTime = zen::DateTime::Now().ToIso8601();
+		const std::string StartLogTime = zen::DateTime::Now().ToIso8601();
 
-	spdlog::apply_all([&](auto Logger) { Logger->info("log starting at {}", StartLogTime); });
+		spdlog::apply_all([&](auto Logger) { Logger->info("log starting at {}", StartLogTime); });
+	}
 
 	g_IsLoggingInitialized = true;
 }
@@ -206,7 +218,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 void
 ShutdownLogging()
 {
-	if (g_IsLoggingInitialized)
+	if (g_IsLoggingInitialized && g_FileSink)
 	{
 		auto DefaultLogger = zen::logging::Default();
 		ZEN_LOG_INFO(DefaultLogger, "log ending at {}", zen::DateTime::Now().ToIso8601());
diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp
new file mode 100644
index 000000000..aa806438b
--- /dev/null
+++ b/src/zenutil/parallelwork.cpp
@@ -0,0 +1,225 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/parallelwork.h>
+
+#include <zencore/callstack.h>
+#include <zencore/except.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+
+#include <typeinfo>
+
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#endif	// ZEN_WITH_TESTS
+
+namespace zen {
+
+ParallelWork::ParallelWork(std::atomic<bool>& AbortFlag, std::atomic<bool>& PauseFlag)
+: m_AbortFlag(AbortFlag)
+, m_PauseFlag(PauseFlag)
+, m_PendingWork(1)
+{
+}
+
+ParallelWork::~ParallelWork()
+{
+	try
+	{
+		if (!m_DispatchComplete)
+		{
+			ZEN_ASSERT(m_PendingWork.Remaining() > 0);
+			ZEN_WARN(
+				"ParallelWork disposed without explicit wait for completion, likely caused by an exception, waiting for dispatched threads "
+				"to complete");
+			m_PendingWork.CountDown();
+		}
+		m_PendingWork.Wait();
+		ptrdiff_t RemainingWork = m_PendingWork.Remaining();
+		if (RemainingWork != 0)
+		{
+			void*			 Frames[8];
+			uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+			CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+			ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}",
+					  RemainingWork,
+					  CallstackToString(Callstack, "  "));
+			FreeCallstack(Callstack);
+
+			uint32_t WaitedMs = 0;
+			while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000)
+			{
+				Sleep(50);
+				WaitedMs += 50;
+			}
+			RemainingWork = m_PendingWork.Remaining();
+			if (RemainingWork != 0)
+			{
+				ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork)
+			}
+			else
+			{
+				ZEN_INFO("ParallelWork destructor safety wait succeeded");
+			}
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("Exception in ~ParallelWork: {}", Ex.what());
+	}
+}
+
+ParallelWork::ExceptionCallback
+ParallelWork::DefaultErrorFunction()
+{
+	return [&](std::exception_ptr Ex, std::atomic<bool>& AbortFlag) {
+		m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex); });
+		AbortFlag = true;
+	};
+}
+
+void
+ParallelWork::Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback)
+{
+	ZEN_ASSERT(!m_DispatchComplete);
+	m_DispatchComplete = true;
+
+	ZEN_ASSERT(m_PendingWork.Remaining() > 0);
+	m_PendingWork.CountDown();
+
+	while (!m_PendingWork.Wait(UpdateIntervalMS))
+	{
+		UpdateCallback(m_AbortFlag.load(), m_PauseFlag.load(), m_PendingWork.Remaining());
+	}
+
+	RethrowErrors();
+}
+
+void
+ParallelWork::Wait()
+{
+	ZEN_ASSERT(!m_DispatchComplete);
+	m_DispatchComplete = true;
+
+	ZEN_ASSERT(m_PendingWork.Remaining() > 0);
+	m_PendingWork.CountDown();
+	m_PendingWork.Wait();
+
+	RethrowErrors();
+}
+
+void
+ParallelWork::RethrowErrors()
+{
+	if (!m_Errors.empty())
+	{
+		if (m_Errors.size() > 1)
+		{
+			ZEN_INFO("Multiple exceptions thrown during ParallelWork execution, dropping the following exceptions:");
+			auto It = m_Errors.begin() + 1;
+			while (It != m_Errors.end())
+			{
+				try
+				{
+					std::rethrow_exception(*It);
+				}
+				catch (const std::exception& Ex)
+				{
+					ZEN_INFO("    {}", Ex.what());
+				}
+				It++;
+			}
+		}
+		std::exception_ptr Ex = m_Errors.front();
+		m_Errors.clear();
+		std::rethrow_exception(Ex);
+	}
+}
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("parallellwork.nowork")
+{
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	Work.Wait();
+}
+
+TEST_CASE("parallellwork.basic")
+{
+	WorkerThreadPool WorkerPool(2);
+
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	for (uint32_t I = 0; I < 5; I++)
+	{
+		Work.ScheduleWork(WorkerPool, [](std::atomic<bool>& AbortFlag) { CHECK(!AbortFlag); });
+	}
+	Work.Wait();
+}
+
+TEST_CASE("parallellwork.throws_in_work")
+{
+	WorkerThreadPool WorkerPool(2);
+
+	std::atomic<bool> AbortFlag;
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
+	for (uint32_t I = 0; I < 10; I++)
+	{
+		Work.ScheduleWork(WorkerPool, [I](std::atomic<bool>& AbortFlag) {
+			ZEN_UNUSED(AbortFlag);
+			if (I > 3)
+			{
+				throw std::runtime_error("We throw in async thread");
+			}
+			else
+			{
+				Sleep(10);
+			}
+		});
+	}
+	CHECK_THROWS_WITH(Work.Wait(), "We throw in async thread");
+}
+
+TEST_CASE("parallellwork.throws_in_dispatch")
+{
+	WorkerThreadPool	  WorkerPool(2);
+	std::atomic<uint32_t> ExecutedCount;
+	try
+	{
+		std::atomic<bool> AbortFlag;
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
+		for (uint32_t I = 0; I < 5; I++)
+		{
+			Work.ScheduleWork(WorkerPool, [I, &ExecutedCount](std::atomic<bool>& AbortFlag) {
+				if (AbortFlag.load())
+				{
+					return;
+				}
+				ExecutedCount++;
+			});
+			if (I == 3)
+			{
+				throw std::runtime_error("We throw in dispatcher thread");
+			}
+		}
+		CHECK(false);
+	}
+	catch (const std::runtime_error& Ex)
+	{
+		CHECK_EQ("We throw in dispatcher thread", std::string(Ex.what()));
+		CHECK_LE(ExecutedCount.load(), 4);
+	}
+}
+
+void
+parallellwork_forcelink()
+{
+}
+#endif	// ZEN_WITH_TESTS
+
+}  // namespace zen
diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp
index e3165e838..797034978 100644
--- a/src/zenutil/workerpools.cpp
+++ b/src/zenutil/workerpools.cpp
@@ -11,9 +11,10 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 namespace {
-	const int LargeWorkerThreadPoolTreadCount  = gsl::narrow<int>(std::thread::hardware_concurrency());
+	const int LargeWorkerThreadPoolTreadCount  = gsl::narrow<int>(Max(std::thread::hardware_concurrency() - 1u, 2u));
 	const int MediumWorkerThreadPoolTreadCount = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 4u), 2u));
 	const int SmallWorkerThreadPoolTreadCount  = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 8u), 1u));
+	const int TinyWorkerThreadPoolTreadCount   = 1;
 
 	static bool IsShutDown = false;
 
@@ -35,6 +36,9 @@ namespace {
 	WorkerPool BurstSmallWorkerPool		 = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(burst)"};
 	WorkerPool BackgroundSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(bkg)"};
 
+	WorkerPool BurstTinyWorkerPool		= {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(burst)"};
+	WorkerPool BackgroundTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(bkg)"};
+
 	WorkerPool SyncWorkerPool = {.TreadCount = 0, .Name = "SyncThreadPool"};
 
 	WorkerThreadPool& EnsurePoolPtr(WorkerPool& Pool)
@@ -75,6 +79,12 @@ GetSmallWorkerPool(EWorkloadType WorkloadType)
 }
 
 WorkerThreadPool&
+GetTinyWorkerPool(EWorkloadType WorkloadType)
+{
+	return EnsurePoolPtr(WorkloadType == EWorkloadType::Burst ? BurstTinyWorkerPool : BackgroundTinyWorkerPool);
+}
+
+WorkerThreadPool&
 GetSyncWorkerPool()
 {
 	return EnsurePoolPtr(SyncWorkerPool);
@@ -91,6 +101,8 @@ ShutdownWorkerPools()
 	BackgroundMediumWorkerPool.Pool.reset();
 	BurstSmallWorkerPool.Pool.reset();
 	BackgroundSmallWorkerPool.Pool.reset();
+	BurstTinyWorkerPool.Pool.reset();
+	BackgroundTinyWorkerPool.Pool.reset();
 	SyncWorkerPool.Pool.reset();
 }
 }  // namespace zen
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index b36f11741..a5b342cb0 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -534,7 +534,7 @@ ZenServerEnvironment::CreateNewTestDir()
 	TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1));
 
 	std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str();
-	ZEN_ASSERT(!std::filesystem::exists(TestPath));
+	ZEN_ASSERT(!IsDir(TestPath));
 
 	ZEN_INFO("Creating new test dir @ '{}'", TestPath);
 
@@ -568,7 +568,7 @@ ZenServerInstance::~ZenServerInstance()
 	{
 		Shutdown();
 		std::error_code DummyEc;
-		std::filesystem::remove(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc);
+		RemoveFile(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc);
 	}
 	catch (const std::exception& Err)
 	{
@@ -619,6 +619,7 @@ ZenServerInstance::Shutdown()
 				std::error_code Ec;
 				if (SignalShutdown(Ec))
 				{
+					Stopwatch Timer;
 					ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
 					while (!m_Process.Wait(1000))
 					{
@@ -632,7 +633,10 @@ ZenServerInstance::Shutdown()
 							ZEN_WARN("Wait abandoned by exited process");
 							return 0;
 						}
-						ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid());
+						ZEN_WARN("Waited for zenserver process {} ({}) to exit for {}",
+								 m_Name,
+								 m_Process.Pid(),
+								 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 					}
 					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 					int ExitCode = m_Process.GetExitCode();
@@ -1033,7 +1037,7 @@ std::string
 ZenServerInstance::GetLogOutput() const
 {
 	std::filesystem::path OutputPath = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
-	if (std::filesystem::is_regular_file(OutputPath))
+	if (IsFile(OutputPath))
 	{
 		FileContents Contents = ReadFile(OutputPath);
 		if (!Contents.ErrorCode)
@@ -1055,7 +1059,7 @@ ZenServerInstance::Terminate()
 	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
 	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
 	ProcessHandle				RunningProcess;
-	std::error_code				Ec = FindProcess(Executable, RunningProcess);
+	std::error_code				Ec = FindProcess(Executable, RunningProcess, /*IncludeSelf*/ false);
 	if (Ec)
 	{
 		throw std::system_error(Ec, fmt::format("failed to look up running server executable '{}'", Executable));
@@ -1123,7 +1127,7 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason)
 		OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort);
 		return false;
 	}
-	if (!std::filesystem::is_directory(Info.DataDir))
+	if (!IsDir(Info.DataDir))
 	{
 		OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir);
 		return false;
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
index c54144549..fe23b00c1 100644
--- a/src/zenutil/zenutil.cpp
+++ b/src/zenutil/zenutil.cpp
@@ -6,6 +6,9 @@
 
 #	include <zenutil/cache/cacherequests.h>
 #	include <zenutil/cache/rpcrecording.h>
+#	include <zenutil/chunkedfile.h>
+#	include <zenutil/commandlineoptions.h>
+#	include <zenutil/parallelwork.h>
 
 namespace zen {
 
@@ -15,6 +18,9 @@ zenutil_forcelinktests()
 	cachepolicy_forcelink();
 	cache::rpcrecord_forcelink();
 	cacherequests_forcelink();
+	chunkedfile_forcelink();
+	commandlineoptions_forcelink();
+	parallellwork_forcelink();
 }
 
 }  // namespace zen