Merge branch 'de/zen-service-command' of https://github.ol.epicgames.net/ue-foundation/zen into de/zen-service-command

27 files changed, 4154 insertions, 128 deletions

diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp
index 6dae5af11..f3c14edc4 100644
--- a/src/zenutil/chunkblock.cpp
+++ b/src/zenutil/chunkblock.cpp
@@ -3,6 +3,7 @@
 #include <zenutil/chunkblock.h>
 
 #include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 
 #include <vector>
@@ -18,20 +19,27 @@ ParseChunkBlockDescription(const CbObjectView& BlockObject)
 	Result.BlockHash = BlockObject["rawHash"sv].AsHash();
 	if (Result.BlockHash != IoHash::Zero)
 	{
+		Result.HeaderSize		= BlockObject["headerSize"sv].AsUInt64();
 		CbArrayView ChunksArray = BlockObject["rawHashes"sv].AsArrayView();
-		Result.ChunkHashes.reserve(ChunksArray.Num());
+		Result.ChunkRawHashes.reserve(ChunksArray.Num());
 		for (CbFieldView ChunkView : ChunksArray)
 		{
-			Result.ChunkHashes.push_back(ChunkView.AsHash());
+			Result.ChunkRawHashes.push_back(ChunkView.AsHash());
 		}
 
-		CbArrayView			  ChunkRawLengthsArray = BlockObject["chunkRawLengths"sv].AsArrayView();
-		std::vector<uint32_t> ChunkLengths;
+		CbArrayView ChunkRawLengthsArray = BlockObject["chunkRawLengths"sv].AsArrayView();
 		Result.ChunkRawLengths.reserve(ChunkRawLengthsArray.Num());
 		for (CbFieldView ChunkView : ChunkRawLengthsArray)
 		{
 			Result.ChunkRawLengths.push_back(ChunkView.AsUInt32());
 		}
+
+		CbArrayView ChunkCompressedLengthsArray = BlockObject["chunkCompressedLengths"sv].AsArrayView();
+		Result.ChunkCompressedLengths.reserve(ChunkCompressedLengthsArray.Num());
+		for (CbFieldView ChunkView : ChunkCompressedLengthsArray)
+		{
+			Result.ChunkCompressedLengths.push_back(ChunkView.AsUInt32());
+		}
 	}
 	return Result;
 }
@@ -57,18 +65,23 @@ ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject)
 CbObject
 BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData)
 {
-	ZEN_ASSERT(Block.ChunkRawLengths.size() == Block.ChunkHashes.size());
+	ZEN_ASSERT(Block.BlockHash != IoHash::Zero);
+	ZEN_ASSERT(Block.HeaderSize > 0);
+	ZEN_ASSERT(Block.ChunkRawLengths.size() == Block.ChunkRawHashes.size());
+	ZEN_ASSERT(Block.ChunkCompressedLengths.size() == Block.ChunkRawHashes.size());
 
 	CbObjectWriter Writer;
 	Writer.AddHash("rawHash"sv, Block.BlockHash);
+	Writer.AddInteger("headerSize"sv, Block.HeaderSize);
 	Writer.BeginArray("rawHashes"sv);
 	{
-		for (const IoHash& ChunkHash : Block.ChunkHashes)
+		for (const IoHash& ChunkHash : Block.ChunkRawHashes)
 		{
 			Writer.AddHash(ChunkHash);
 		}
 	}
 	Writer.EndArray();
+
 	Writer.BeginArray("chunkRawLengths");
 	{
 		for (uint32_t ChunkSize : Block.ChunkRawLengths)
@@ -78,11 +91,58 @@ BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView Meta
 	}
 	Writer.EndArray();
 
+	Writer.BeginArray("chunkCompressedLengths");
+	{
+		for (uint32_t ChunkSize : Block.ChunkCompressedLengths)
+		{
+			Writer.AddInteger(ChunkSize);
+		}
+	}
+	Writer.EndArray();
+
 	Writer.AddObject("metadata", MetaData);
 
 	return Writer.Save();
 }
 
+ChunkBlockDescription
+GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash)
+{
+	ChunkBlockDescription BlockDescription = {{.BlockHash = IoHash::HashBuffer(BlockPayload)}};
+	if (BlockDescription.BlockHash != RawHash)
+	{
+		throw std::runtime_error(fmt::format("Block {} content hash {} does not match block hash", RawHash, BlockDescription.BlockHash));
+	}
+	if (IterateChunkBlock(
+			BlockPayload,
+			[&BlockDescription, RawHash](CompressedBuffer&& Chunk, const IoHash& AttachmentHash) {
+				if (CompositeBuffer Decompressed = Chunk.DecompressToComposite(); Decompressed)
+				{
+					IoHash ChunkHash = IoHash::HashBuffer(Decompressed.Flatten());
+					if (ChunkHash != AttachmentHash)
+					{
+						throw std::runtime_error(
+							fmt::format("Chunk {} in block {} content hash {} does not match chunk", AttachmentHash, RawHash, ChunkHash));
+					}
+					BlockDescription.ChunkRawHashes.push_back(AttachmentHash);
+					BlockDescription.ChunkRawLengths.push_back(gsl::narrow<uint32_t>(Decompressed.GetSize()));
+					BlockDescription.ChunkCompressedLengths.push_back(gsl::narrow<uint32_t>(Chunk.GetCompressedSize()));
+				}
+				else
+				{
+					throw std::runtime_error(fmt::format("Chunk {} in block {} is not a compressed buffer", AttachmentHash, RawHash));
+				}
+			},
+			BlockDescription.HeaderSize))
+	{
+		return BlockDescription;
+	}
+	else
+	{
+		throw std::runtime_error(fmt::format("Block {} is malformed", RawHash));
+	}
+}
+
 CompressedBuffer
 GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock)
 {
@@ -91,8 +151,9 @@ GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks,
 	std::vector<SharedBuffer> ChunkSegments;
 	ChunkSegments.resize(1);
 	ChunkSegments.reserve(1 + ChunkCount);
-	OutBlock.ChunkHashes.reserve(ChunkCount);
+	OutBlock.ChunkRawHashes.reserve(ChunkCount);
 	OutBlock.ChunkRawLengths.reserve(ChunkCount);
+	OutBlock.ChunkCompressedLengths.reserve(ChunkCount);
 	{
 		IoBuffer		  TempBuffer(ChunkCount * 9);
 		MutableMemoryView View			 = TempBuffer.GetMutableView();
@@ -106,16 +167,19 @@ GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks,
 			std::span<const SharedBuffer>		  Segments	= Chunk.second.GetCompressed().GetSegments();
 			for (const SharedBuffer& Segment : Segments)
 			{
+				ZEN_ASSERT(Segment.IsOwned());
 				ChunkSize += Segment.GetSize();
 				ChunkSegments.push_back(Segment);
 			}
 			BufferEndPtr += WriteVarUInt(ChunkSize, BufferEndPtr);
-			OutBlock.ChunkHashes.push_back(It.first);
+			OutBlock.ChunkRawHashes.push_back(It.first);
 			OutBlock.ChunkRawLengths.push_back(gsl::narrow<uint32_t>(Chunk.first));
+			OutBlock.ChunkCompressedLengths.push_back(gsl::narrow<uint32_t>(ChunkSize));
 		}
 		ZEN_ASSERT(BufferEndPtr <= View.GetDataEnd());
 		ptrdiff_t TempBufferLength = std::distance(BufferStartPtr, BufferEndPtr);
 		ChunkSegments[0]		   = SharedBuffer(IoBuffer(TempBuffer, 0, gsl::narrow<size_t>(TempBufferLength)));
+		OutBlock.HeaderSize		   = TempBufferLength;
 	}
 	CompressedBuffer CompressedBlock =
 		CompressedBuffer::Compress(CompositeBuffer(std::move(ChunkSegments)), OodleCompressor::Mermaid, OodleCompressionLevel::None);
@@ -123,42 +187,69 @@ GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks,
 	return CompressedBlock;
 }
 
-bool
-IterateChunkBlock(const SharedBuffer& BlockPayload, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor)
+std::vector<uint32_t>
+ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize)
 {
-	ZEN_ASSERT(BlockPayload);
-	if (BlockPayload.GetSize() < 1)
-	{
-		return false;
-	}
-
-	MemoryView	   BlockView = BlockPayload.GetView();
-	const uint8_t* ReadPtr	 = reinterpret_cast<const uint8_t*>(BlockView.GetData());
+	const uint8_t* ReadPtr = reinterpret_cast<const uint8_t*>(BlockView.GetData());
 	uint32_t	   NumberSize;
 	uint64_t	   ChunkCount = ReadVarUInt(ReadPtr, NumberSize);
 	ReadPtr += NumberSize;
-	std::vector<uint64_t> ChunkSizes;
+	std::vector<uint32_t> ChunkSizes;
 	ChunkSizes.reserve(ChunkCount);
 	while (ChunkCount--)
 	{
-		ChunkSizes.push_back(ReadVarUInt(ReadPtr, NumberSize));
+		if (ReadPtr >= BlockView.GetDataEnd())
+		{
+			throw std::runtime_error("Invalid block header, block data ended unexpectedly");
+		}
+		uint64_t ChunkSize = ReadVarUInt(ReadPtr, NumberSize);
+		if (ChunkSize > std::numeric_limits<uint32_t>::max())
+		{
+			throw std::runtime_error("Invalid block header, header data is corrupt");
+		}
+		if (ChunkSize < 1)
+		{
+			throw std::runtime_error("Invalid block header, header data is corrupt");
+		}
+		ChunkSizes.push_back(gsl::narrow<uint32_t>(ChunkSize));
 		ReadPtr += NumberSize;
 	}
+	uint64_t Offset = std::distance((const uint8_t*)BlockView.GetData(), ReadPtr);
+	OutHeaderSize	= Offset;
+	return ChunkSizes;
+}
+
+bool
+IterateChunkBlock(const SharedBuffer&														  BlockPayload,
+				  std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
+				  uint64_t&																	  OutHeaderSize)
+{
+	ZEN_ASSERT(BlockPayload);
+	if (BlockPayload.GetSize() < 1)
+	{
+		return false;
+	}
+
+	MemoryView BlockView = BlockPayload.GetView();
+
+	std::vector<uint32_t> ChunkSizes = ReadChunkBlockHeader(BlockView, OutHeaderSize);
+	uint64_t			  Offset	 = OutHeaderSize;
+	OutHeaderSize					 = Offset;
 	for (uint64_t ChunkSize : ChunkSizes)
 	{
-		IoBuffer		 Chunk(IoBuffer::Wrap, ReadPtr, ChunkSize);
+		IoBuffer		 Chunk(BlockPayload.AsIoBuffer(), Offset, ChunkSize);
 		IoHash			 AttachmentRawHash;
 		uint64_t		 AttachmentRawSize;
 		CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), AttachmentRawHash, AttachmentRawSize);
-
+		ZEN_ASSERT_SLOW(IoHash::HashBuffer(CompressedChunk.DecompressToComposite()) == AttachmentRawHash);
 		if (!CompressedChunk)
 		{
 			ZEN_ERROR("Invalid chunk in block");
 			return false;
 		}
 		Visitor(std::move(CompressedChunk), AttachmentRawHash);
-		ReadPtr += ChunkSize;
-		ZEN_ASSERT(ReadPtr <= BlockView.GetDataEnd());
+		Offset += ChunkSize;
+		ZEN_ASSERT(Offset <= BlockView.GetSize());
 	}
 	return true;
 };
diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
new file mode 100644
index 000000000..bb1ee5183
--- /dev/null
+++ b/src/zenutil/chunkedcontent.cpp
@@ -0,0 +1,896 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/chunkedcontent.h>
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+
+#include <zenutil/chunkedfile.h>
+#include <zenutil/chunkingcontroller.h>
+#include <zenutil/parallellwork.h>
+#include <zenutil/workerpools.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+namespace {
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  std::span<const uint32_t>							ChunkSequence,
+						  std::span<const IoHash>							ChunkHashes,
+						  std::span<const uint64_t>							ChunkRawSizes)
+	{
+		ZEN_ASSERT(ChunkHashes.size() == ChunkRawSizes.size());
+		InOutChunkedContent.ChunkCounts.push_back(gsl::narrow<uint32_t>(ChunkSequence.size()));
+		InOutChunkedContent.ChunkOrders.reserve(InOutChunkedContent.ChunkOrders.size() + ChunkSequence.size());
+
+		for (uint32_t ChunkedSequenceIndex : ChunkSequence)
+		{
+			const IoHash& ChunkHash = ChunkHashes[ChunkedSequenceIndex];
+			if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end())
+			{
+				uint32_t ChunkIndex = gsl::narrow<uint32_t>(It->second);
+				InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+			}
+			else
+			{
+				uint32_t ChunkIndex = gsl::narrow<uint32_t>(InOutChunkedContent.ChunkHashes.size());
+				ChunkHashToChunkIndex.insert_or_assign(ChunkHash, ChunkIndex);
+				InOutChunkedContent.ChunkHashes.push_back(ChunkHash);
+				InOutChunkedContent.ChunkRawSizes.push_back(ChunkRawSizes[ChunkedSequenceIndex]);
+				InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+				Stats.UniqueChunksFound++;
+				Stats.UniqueBytesFound += ChunkRawSizes[ChunkedSequenceIndex];
+			}
+		}
+		InOutChunkedContent.SequenceRawHashes.push_back(RawHash);
+		Stats.UniqueSequencesFound++;
+	}
+
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  const uint64_t									RawSize)
+	{
+		InOutChunkedContent.ChunkCounts.push_back(1);
+
+		if (auto It = ChunkHashToChunkIndex.find(RawHash); It != ChunkHashToChunkIndex.end())
+		{
+			uint32_t ChunkIndex = gsl::narrow<uint32_t>(It->second);
+			InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+		}
+		else
+		{
+			uint32_t ChunkIndex = gsl::narrow<uint32_t>(InOutChunkedContent.ChunkHashes.size());
+			ChunkHashToChunkIndex.insert_or_assign(RawHash, ChunkIndex);
+			InOutChunkedContent.ChunkHashes.push_back(RawHash);
+			InOutChunkedContent.ChunkRawSizes.push_back(RawSize);
+			InOutChunkedContent.ChunkOrders.push_back(ChunkIndex);
+			Stats.UniqueChunksFound++;
+			Stats.UniqueBytesFound += RawSize;
+		}
+		InOutChunkedContent.SequenceRawHashes.push_back(RawHash);
+		Stats.UniqueSequencesFound++;
+	}
+
+	IoHash HashOneFile(ChunkingStatistics&								 Stats,
+					   const ChunkingController&						 InChunkingController,
+					   ChunkedFolderContent&							 OutChunkedContent,
+					   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+					   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& RawHashToSequenceRawHashIndex,
+					   RwLock&											 Lock,
+					   const std::filesystem::path&						 FolderPath,
+					   uint32_t											 PathIndex,
+					   std::atomic<bool>&								 AbortFlag)
+	{
+		ZEN_TRACE_CPU("ChunkFolderContent");
+
+		const uint64_t				 RawSize = OutChunkedContent.RawSizes[PathIndex];
+		const std::filesystem::path& Path	 = OutChunkedContent.Paths[PathIndex];
+
+		if (RawSize == 0)
+		{
+			return IoHash::Zero;
+		}
+		else
+		{
+			ChunkedInfoWithSource Chunked;
+			const bool			  DidChunking =
+				InChunkingController.ProcessFile((FolderPath / Path).make_preferred(), RawSize, Chunked, Stats.BytesHashed, AbortFlag);
+			if (DidChunking)
+			{
+				Lock.WithExclusiveLock([&]() {
+					if (!RawHashToSequenceRawHashIndex.contains(Chunked.Info.RawHash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{Chunked.Info.RawHash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())});
+						std::vector<uint64_t> ChunkSizes;
+						ChunkSizes.reserve(Chunked.ChunkSources.size());
+						for (const ChunkSource& Source : Chunked.ChunkSources)
+						{
+							ChunkSizes.push_back(Source.Size);
+						}
+						AddChunkSequence(Stats,
+										 OutChunkedContent.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 Chunked.Info.RawHash,
+										 Chunked.Info.ChunkSequence,
+										 Chunked.Info.ChunkHashes,
+										 ChunkSizes);
+						Stats.UniqueSequencesFound++;
+					}
+				});
+				Stats.FilesChunked++;
+				return Chunked.Info.RawHash;
+			}
+			else
+			{
+				ZEN_TRACE_CPU("HashOnly");
+
+				IoBuffer	 Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred());
+				const IoHash Hash	= IoHash::HashBuffer(Buffer, &Stats.BytesHashed);
+
+				Lock.WithExclusiveLock([&]() {
+					if (!RawHashToSequenceRawHashIndex.contains(Hash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{Hash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())});
+						AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
+						Stats.UniqueSequencesFound++;
+					}
+				});
+				return Hash;
+			}
+		}
+	}
+
+	std::string PathCompareString(const std::filesystem::path& Path) { return ToLower(Path.generic_string()); }
+
+}  // namespace
+
+std::string_view FolderContentSourcePlatformNames[(size_t)SourcePlatform::_Count] = {"Windows"sv, "Linux"sv, "MacOS"sv};
+
+std::string_view
+ToString(SourcePlatform Platform)
+{
+	return FolderContentSourcePlatformNames[(size_t)Platform];
+}
+
+SourcePlatform
+FromString(std::string_view Platform, SourcePlatform Default)
+{
+	for (size_t Index = 0; Index < (size_t)SourcePlatform::_Count; Index++)
+	{
+		if (Platform == FolderContentSourcePlatformNames[Index])
+		{
+			return (SourcePlatform)Index;
+		}
+	}
+	return Default;
+}
+
+SourcePlatform
+GetSourceCurrentPlatform()
+{
+#if ZEN_PLATFORM_WINDOWS
+	return SourcePlatform::Windows;
+#endif
+#if ZEN_PLATFORM_MAC
+	return SourcePlatform::MacOS;
+#endif
+#if ZEN_PLATFORM_LINUX
+	return SourcePlatform::Linux;
+#endif
+}
+
+bool
+FolderContent::AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return (Lhs & 0xff) == (Rhs & 0xff);
+#endif
+#if ZEN_PLATFORM_MAC
+	return Lhs == Rhs;
+#endif
+#if ZEN_PLATFORM_LINUX
+	return Lhs == Rhs;
+#endif
+}
+
+bool
+FolderContent::operator==(const FolderContent& Rhs) const
+{
+	if ((Platform == Rhs.Platform) && (RawSizes == Rhs.RawSizes) && (Attributes == Rhs.Attributes) &&
+		(ModificationTicks == Rhs.ModificationTicks) && (Paths.size() == Rhs.Paths.size()))
+	{
+		size_t PathCount = 0;
+		for (size_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
+		{
+			if (Paths[PathIndex].generic_string() != Rhs.Paths[PathIndex].generic_string())
+			{
+				return false;
+			}
+		}
+		return true;
+	}
+	return false;
+}
+
+bool
+FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const
+{
+	ZEN_TRACE_CPU("FolderContent::AreKnownFilesEqual");
+	tsl::robin_map<std::string, size_t> RhsPathToIndex;
+	const size_t						RhsPathCount = Rhs.Paths.size();
+	RhsPathToIndex.reserve(RhsPathCount);
+	for (size_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++)
+	{
+		RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex});
+	}
+	const size_t PathCount = Paths.size();
+	for (size_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
+	{
+		if (auto It = RhsPathToIndex.find(Paths[PathIndex].generic_string()); It != RhsPathToIndex.end())
+		{
+			const size_t RhsPathIndex = It->second;
+			if ((RawSizes[PathIndex] != Rhs.RawSizes[RhsPathIndex]) ||
+				(!AreFileAttributesEqual(Attributes[PathIndex], Rhs.Attributes[RhsPathIndex])) ||
+				(ModificationTicks[PathIndex] != Rhs.ModificationTicks[RhsPathIndex]))
+			{
+				return false;
+			}
+		}
+		else
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
+void
+FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutPathIndexesOufOfDate)
+{
+	ZEN_TRACE_CPU("FolderContent::UpdateState");
+	tsl::robin_map<std::string, uint32_t> RhsPathToIndex;
+	const uint32_t						  RhsPathCount = gsl::narrow<uint32_t>(Rhs.Paths.size());
+	RhsPathToIndex.reserve(RhsPathCount);
+	for (uint32_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++)
+	{
+		RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex});
+	}
+	uint32_t PathCount = gsl::narrow<uint32_t>(Paths.size());
+	for (uint32_t PathIndex = 0; PathIndex < PathCount;)
+	{
+		if (auto It = RhsPathToIndex.find(Paths[PathIndex].generic_string()); It != RhsPathToIndex.end())
+		{
+			const uint32_t RhsPathIndex = It->second;
+
+			if ((RawSizes[PathIndex] != Rhs.RawSizes[RhsPathIndex]) ||
+				(ModificationTicks[PathIndex] != Rhs.ModificationTicks[RhsPathIndex]))
+			{
+				RawSizes[PathIndex]			 = Rhs.RawSizes[RhsPathIndex];
+				ModificationTicks[PathIndex] = Rhs.ModificationTicks[RhsPathIndex];
+				OutPathIndexesOufOfDate.push_back(PathIndex);
+			}
+			Attributes[PathIndex] = Rhs.Attributes[RhsPathIndex];
+			PathIndex++;
+		}
+		else
+		{
+			Paths.erase(Paths.begin() + PathIndex);
+			RawSizes.erase(RawSizes.begin() + PathIndex);
+			Attributes.erase(Attributes.begin() + PathIndex);
+			ModificationTicks.erase(ModificationTicks.begin() + PathIndex);
+			PathCount--;
+		}
+	}
+}
+
+FolderContent
+GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPathIndexes)
+{
+	ZEN_TRACE_CPU("FolderContent::GetUpdatedContent");
+	FolderContent						  Result = {.Platform = Old.Platform};
+	tsl::robin_map<std::string, uint32_t> NewPathToIndex;
+	const uint32_t						  NewPathCount = gsl::narrow<uint32_t>(New.Paths.size());
+	NewPathToIndex.reserve(NewPathCount);
+	for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++)
+	{
+		NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex});
+	}
+	uint32_t OldPathCount = gsl::narrow<uint32_t>(Old.Paths.size());
+	for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++)
+	{
+		if (auto It = NewPathToIndex.find(Old.Paths[OldPathIndex].generic_string()); It != NewPathToIndex.end())
+		{
+			const uint32_t NewPathIndex = It->second;
+
+			if ((Old.RawSizes[OldPathIndex] != New.RawSizes[NewPathIndex]) ||
+				(Old.ModificationTicks[OldPathIndex] != New.ModificationTicks[NewPathIndex]))
+			{
+				Result.Paths.push_back(New.Paths[NewPathIndex]);
+				Result.RawSizes.push_back(New.RawSizes[NewPathIndex]);
+				Result.Attributes.push_back(New.Attributes[NewPathIndex]);
+				Result.ModificationTicks.push_back(New.ModificationTicks[NewPathIndex]);
+			}
+		}
+		else
+		{
+			OutDeletedPathIndexes.push_back(Old.Paths[OldPathIndex]);
+		}
+	}
+	return Result;
+}
+
+void
+SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output)
+{
+	ZEN_TRACE_CPU("SaveFolderContentToCompactBinary");
+	Output.AddString("platform"sv, ToString(Content.Platform));
+	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
+	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.Attributes, "attributes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ModificationTicks, "modificationTimes"sv, Output);
+}
+
+FolderContent
+LoadFolderContentToCompactBinary(CbObjectView Input)
+{
+	ZEN_TRACE_CPU("LoadFolderContentToCompactBinary");
+	FolderContent Content;
+	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
+	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
+	compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes);
+	compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes);
+	compactbinary_helpers::ReadArray("modificationTimes"sv, Input, Content.ModificationTicks);
+	return Content;
+}
+
+FolderContent
+GetFolderContent(GetFolderContentStatistics&															  Stats,
+				 const std::filesystem::path&															  RootPath,
+				 std::function<bool(const std::string_view& RelativePath)>&&							  AcceptDirectory,
+				 std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile,
+				 WorkerThreadPool&																		  WorkerPool,
+				 int32_t																				  UpdateInteralMS,
+				 std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						  UpdateCallback,
+				 std::atomic<bool>&																		  AbortFlag)
+{
+	ZEN_TRACE_CPU("GetFolderContent");
+
+	Stopwatch Timer;
+	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
+
+	FolderContent Content;
+	struct AsyncVisitor : public GetDirectoryContentVisitor
+	{
+		AsyncVisitor(GetFolderContentStatistics&															  Stats,
+					 std::atomic<bool>&																		  AbortFlag,
+					 FolderContent&																			  Content,
+					 std::function<bool(const std::string_view& RelativePath)>&&							  AcceptDirectory,
+					 std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile)
+		: m_Stats(Stats)
+		, m_AbortFlag(AbortFlag)
+		, m_FoundContent(Content)
+		, m_AcceptDirectory(std::move(AcceptDirectory))
+		, m_AcceptFile(std::move(AcceptFile))
+		{
+		}
+		virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+		{
+			if (!m_AbortFlag)
+			{
+				m_Stats.FoundFileCount += Content.FileNames.size();
+				for (uint64_t FileSize : Content.FileSizes)
+				{
+					m_Stats.FoundFileByteCount += FileSize;
+				}
+				std::string RelativeDirectoryPath = RelativeRoot.generic_string();
+				if (m_AcceptDirectory(RelativeDirectoryPath))
+				{
+					std::vector<std::filesystem::path> Paths;
+					std::vector<uint64_t>			   RawSizes;
+					std::vector<uint32_t>			   Attributes;
+					std::vector<uint64_t>			   ModificatonTicks;
+					Paths.reserve(Content.FileNames.size());
+					RawSizes.reserve(Content.FileNames.size());
+					Attributes.reserve(Content.FileNames.size());
+					ModificatonTicks.reserve(Content.FileModificationTicks.size());
+
+					for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+					{
+						const std::filesystem::path& FileName	  = Content.FileNames[FileIndex];
+						std::string					 RelativePath = (RelativeRoot / FileName).generic_string();
+						std::replace(RelativePath.begin(), RelativePath.end(), '\\', '/');
+						if (m_AcceptFile(RelativePath, Content.FileSizes[FileIndex], Content.FileAttributes[FileIndex]))
+						{
+							Paths.emplace_back(std::move(RelativePath));
+							RawSizes.emplace_back(Content.FileSizes[FileIndex]);
+							Attributes.emplace_back(Content.FileAttributes[FileIndex]);
+							ModificatonTicks.emplace_back(Content.FileModificationTicks[FileIndex]);
+
+							m_Stats.AcceptedFileCount++;
+							m_Stats.AcceptedFileByteCount += Content.FileSizes[FileIndex];
+						}
+					}
+					m_Lock.WithExclusiveLock([&]() {
+						m_FoundContent.Paths.insert(m_FoundContent.Paths.end(), Paths.begin(), Paths.end());
+						m_FoundContent.RawSizes.insert(m_FoundContent.RawSizes.end(), RawSizes.begin(), RawSizes.end());
+						m_FoundContent.Attributes.insert(m_FoundContent.Attributes.end(), Attributes.begin(), Attributes.end());
+						m_FoundContent.ModificationTicks.insert(m_FoundContent.ModificationTicks.end(),
+																ModificatonTicks.begin(),
+																ModificatonTicks.end());
+					});
+				}
+			}
+		}
+
+		GetFolderContentStatistics&															   m_Stats;
+		std::atomic<bool>&																	   m_AbortFlag;
+		RwLock																				   m_Lock;
+		FolderContent&																		   m_FoundContent;
+		std::function<bool(const std::string_view& RelativePath)>							   m_AcceptDirectory;
+		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)> m_AcceptFile;
+	} Visitor(Stats, AbortFlag, Content, std::move(AcceptDirectory), std::move(AcceptFile));
+
+	Latch PendingWork(1);
+	GetDirectoryContent(RootPath,
+						DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes |
+							DirectoryContentFlags::IncludeAttributes | DirectoryContentFlags::IncludeModificationTick,
+						Visitor,
+						WorkerPool,
+						PendingWork);
+	PendingWork.CountDown();
+	while (!PendingWork.Wait(UpdateInteralMS))
+	{
+		UpdateCallback(AbortFlag.load(), PendingWork.Remaining());
+	}
+	std::vector<size_t> Order;
+	size_t				PathCount = Content.Paths.size();
+	Order.resize(Content.Paths.size());
+	std::vector<std::string> Parents;
+	Parents.reserve(PathCount);
+	std::vector<std::string> Filenames;
+	Filenames.reserve(PathCount);
+	for (size_t OrderIndex = 0; OrderIndex < PathCount; OrderIndex++)
+	{
+		Order[OrderIndex] = OrderIndex;
+		Parents.emplace_back(Content.Paths[OrderIndex].parent_path().generic_string());
+		Filenames.emplace_back(Content.Paths[OrderIndex].filename().generic_string());
+	}
+	std::sort(Order.begin(), Order.end(), [&Parents, &Filenames](size_t Lhs, size_t Rhs) {
+		const std::string& LhsParent = Parents[Lhs];
+		const std::string& RhsParent = Parents[Rhs];
+		if (LhsParent < RhsParent)
+		{
+			return true;
+		}
+		else if (LhsParent > RhsParent)
+		{
+			return false;
+		}
+		return Filenames[Lhs] < Filenames[Rhs];
+	});
+	FolderContent OrderedContent;
+	OrderedContent.Paths.reserve(PathCount);
+	OrderedContent.RawSizes.reserve(PathCount);
+	OrderedContent.Attributes.reserve(PathCount);
+	OrderedContent.ModificationTicks.reserve(PathCount);
+	for (size_t OrderIndex : Order)
+	{
+		OrderedContent.Paths.emplace_back(std::move(Content.Paths[OrderIndex]));
+		OrderedContent.RawSizes.emplace_back(Content.RawSizes[OrderIndex]);
+		OrderedContent.Attributes.emplace_back(Content.Attributes[OrderIndex]);
+		OrderedContent.ModificationTicks.emplace_back(Content.ModificationTicks[OrderIndex]);
+	}
+	return OrderedContent;
+}
+
+void
+SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output)
+{
+	ZEN_TRACE_CPU("SaveChunkedFolderContentToCompactBinary");
+	Output.AddString("platform"sv, ToString(Content.Platform));
+	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
+	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.Attributes, "attributes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.RawHashes, "rawHashes"sv, Output);
+
+	Output.BeginObject("chunkedContent");
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.SequenceRawHashes, "sequenceRawHashes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkCounts, "chunkCounts"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkOrders, "chunkOrders"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkHashes, "chunkHashes"sv, Output);
+	compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkRawSizes, "chunkRawSizes"sv, Output);
+	Output.EndObject();	 // chunkedContent
+}
+
+ChunkedFolderContent
+LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
+{
+	ZEN_TRACE_CPU("LoadChunkedFolderContentToCompactBinary");
+	ChunkedFolderContent Content;
+	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
+	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
+	compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes);
+	compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes);
+	compactbinary_helpers::ReadArray("rawHashes"sv, Input, Content.RawHashes);
+
+	CbObjectView ChunkedContentView = Input["chunkedContent"sv].AsObjectView();
+	compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkedContentView, Content.ChunkedContent.SequenceRawHashes);
+	compactbinary_helpers::ReadArray("chunkCounts"sv, ChunkedContentView, Content.ChunkedContent.ChunkCounts);
+	compactbinary_helpers::ReadArray("chunkOrders"sv, ChunkedContentView, Content.ChunkedContent.ChunkOrders);
+	compactbinary_helpers::ReadArray("chunkHashes"sv, ChunkedContentView, Content.ChunkedContent.ChunkHashes);
+	compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkedContentView, Content.ChunkedContent.ChunkRawSizes);
+	return Content;
+}
+
+ChunkedFolderContent
+MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays)
+{
+	ZEN_TRACE_CPU("MergeChunkedFolderContents");
+
+	ZEN_ASSERT(!Overlays.empty());
+
+	ChunkedFolderContent Result;
+	const size_t		 BasePathCount = Base.Paths.size();
+	Result.Paths.reserve(BasePathCount);
+	Result.RawSizes.reserve(BasePathCount);
+	Result.Attributes.reserve(BasePathCount);
+	Result.RawHashes.reserve(BasePathCount);
+
+	const size_t BaseChunkCount = Base.ChunkedContent.ChunkHashes.size();
+	Result.ChunkedContent.SequenceRawHashes.reserve(Base.ChunkedContent.SequenceRawHashes.size());
+	Result.ChunkedContent.ChunkCounts.reserve(BaseChunkCount);
+	Result.ChunkedContent.ChunkHashes.reserve(BaseChunkCount);
+	Result.ChunkedContent.ChunkRawSizes.reserve(BaseChunkCount);
+	Result.ChunkedContent.ChunkOrders.reserve(Base.ChunkedContent.ChunkOrders.size());
+
+	tsl::robin_map<std::string, std::filesystem::path> GenericPathToActualPath;
+	for (const std::filesystem::path& Path : Base.Paths)
+	{
+		GenericPathToActualPath.insert({PathCompareString(Path), Path});
+	}
+	for (const ChunkedFolderContent& Overlay : Overlays)
+	{
+		for (const std::filesystem::path& Path : Overlay.Paths)
+		{
+			GenericPathToActualPath.insert({PathCompareString(Path), Path});
+		}
+	}
+
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
+
+	auto BuildOverlayPaths = [](std::span<const ChunkedFolderContent> Overlays) -> tsl::robin_set<std::string> {
+		tsl::robin_set<std::string> Result;
+		for (const ChunkedFolderContent& OverlayContent : Overlays)
+		{
+			for (const std::filesystem::path& Path : OverlayContent.Paths)
+			{
+				Result.insert(PathCompareString(Path));
+			}
+		}
+		return Result;
+	};
+
+	auto AddContent = [&BuildOverlayPaths](ChunkedFolderContent&									 Result,
+										   const ChunkedFolderContent&								 OverlayContent,
+										   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>&		 ChunkHashToChunkIndex,
+										   tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>&		 RawHashToSequenceRawHashIndex,
+										   const tsl::robin_map<std::string, std::filesystem::path>& GenericPathToActualPath,
+										   std::span<const ChunkedFolderContent>					 Overlays) {
+		const ChunkedContentLookup	OverlayLookup	 = BuildChunkedContentLookup(OverlayContent);
+		tsl::robin_set<std::string> BaseOverlayPaths = BuildOverlayPaths(Overlays);
+		for (uint32_t PathIndex = 0; PathIndex < OverlayContent.Paths.size(); PathIndex++)
+		{
+			std::string GenericPath = PathCompareString(OverlayContent.Paths[PathIndex]);
+			if (!BaseOverlayPaths.contains(GenericPath))
+			{
+				// This asset will not be overridden by a later layer - add it
+
+				const std::filesystem::path OriginalPath = GenericPathToActualPath.at(GenericPath);
+				Result.Paths.push_back(OriginalPath);
+				const IoHash& RawHash = OverlayContent.RawHashes[PathIndex];
+				Result.RawSizes.push_back(OverlayContent.RawSizes[PathIndex]);
+				Result.Attributes.push_back(OverlayContent.Attributes[PathIndex]);
+				Result.RawHashes.push_back(RawHash);
+
+				if (OverlayContent.RawSizes[PathIndex] > 0)
+				{
+					if (!RawHashToSequenceRawHashIndex.contains(RawHash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{RawHash, gsl::narrow<uint32_t>(Result.ChunkedContent.SequenceRawHashes.size())});
+						const uint32_t			  SequenceRawHashIndex = OverlayLookup.RawHashToSequenceIndex.at(RawHash);
+						const uint32_t			  OrderIndexOffset	   = OverlayLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex];
+						const uint32_t			  ChunkCount		   = OverlayContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
+						ChunkingStatistics		  Stats;
+						std::span<const uint32_t> OriginalChunkOrder =
+							std::span<const uint32_t>(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 OverlayContent.ChunkedContent.ChunkHashes,
+										 OverlayContent.ChunkedContent.ChunkRawSizes);
+						Stats.UniqueSequencesFound++;
+					}
+				}
+			}
+		}
+	};
+
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> MergedChunkHashToChunkIndex;
+	AddContent(Result, Base, MergedChunkHashToChunkIndex, RawHashToSequenceRawHashIndex, GenericPathToActualPath, Overlays);
+	for (uint32_t OverlayIndex = 0; OverlayIndex < Overlays.size(); OverlayIndex++)
+	{
+		AddContent(Result,
+				   Overlays[OverlayIndex],
+				   MergedChunkHashToChunkIndex,
+				   RawHashToSequenceRawHashIndex,
+				   GenericPathToActualPath,
+				   Overlays.subspan(OverlayIndex + 1));
+	}
+	return Result;
+}
+
+ChunkedFolderContent
+DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span<const std::filesystem::path> DeletedPaths)
+{
+	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
+
+	ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size());
+	ChunkedFolderContent Result = {.Platform = BaseContent.Platform};
+	if (DeletedPaths.size() < BaseContent.Paths.size())
+	{
+		tsl::robin_set<std::string> DeletedPathSet;
+		DeletedPathSet.reserve(DeletedPaths.size());
+		for (const std::filesystem::path& DeletedPath : DeletedPaths)
+		{
+			DeletedPathSet.insert(PathCompareString(DeletedPath));
+		}
+		const ChunkedContentLookup						 BaseLookup = BuildChunkedContentLookup(BaseContent);
+		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+
+		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
+		for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++)
+		{
+			const std::filesystem::path& Path = BaseContent.Paths[PathIndex];
+			if (!DeletedPathSet.contains(PathCompareString(Path)))
+			{
+				const IoHash&  RawHash = BaseContent.RawHashes[PathIndex];
+				const uint64_t RawSize = BaseContent.RawSizes[PathIndex];
+				Result.Paths.push_back(Path);
+				Result.RawSizes.push_back(RawSize);
+				Result.Attributes.push_back(BaseContent.Attributes[PathIndex]);
+				Result.RawHashes.push_back(RawHash);
+				if (RawSize > 0)
+				{
+					if (!RawHashToSequenceRawHashIndex.contains(RawHash))
+					{
+						RawHashToSequenceRawHashIndex.insert(
+							{RawHash, gsl::narrow<uint32_t>(Result.ChunkedContent.SequenceRawHashes.size())});
+						const uint32_t			  SequenceRawHashIndex = BaseLookup.RawHashToSequenceIndex.at(RawHash);
+						const uint32_t			  OrderIndexOffset	   = BaseLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex];
+						const uint32_t			  ChunkCount		   = BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
+						ChunkingStatistics		  Stats;
+						std::span<const uint32_t> OriginalChunkOrder =
+							std::span<const uint32_t>(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 BaseContent.ChunkedContent.ChunkHashes,
+										 BaseContent.ChunkedContent.ChunkRawSizes);
+						Stats.UniqueSequencesFound++;
+					}
+				}
+			}
+		}
+	}
+	return Result;
+}
+
+ChunkedFolderContent
+ChunkFolderContent(ChunkingStatistics&												 Stats,
+				   WorkerThreadPool&												 WorkerPool,
+				   const std::filesystem::path&										 RootPath,
+				   const FolderContent&												 Content,
+				   const ChunkingController&										 InChunkingController,
+				   int32_t															 UpdateInteralMS,
+				   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+				   std::atomic<bool>&												 AbortFlag)
+{
+	ZEN_TRACE_CPU("ChunkFolderContent");
+
+	Stopwatch Timer;
+	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
+
+	ChunkedFolderContent Result	   = {.Platform	  = Content.Platform,
+									  .Paths	  = Content.Paths,
+									  .RawSizes	  = Content.RawSizes,
+									  .Attributes = Content.Attributes};
+	const size_t		 ItemCount = Result.Paths.size();
+	Result.RawHashes.resize(ItemCount, IoHash::Zero);
+	Result.ChunkedContent.SequenceRawHashes.reserve(ItemCount);	 // Up to 1 per file, maybe less
+	Result.ChunkedContent.ChunkCounts.reserve(ItemCount);		 // Up to one per file
+	Result.ChunkedContent.ChunkOrders.reserve(ItemCount);		 // At least 1 per file, maybe more
+	Result.ChunkedContent.ChunkHashes.reserve(ItemCount);		 // At least 1 per file, maybe more
+	Result.ChunkedContent.ChunkRawSizes.reserve(ItemCount);		 // At least 1 per file, maybe more
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToChunkSequenceIndex;
+	RawHashToChunkSequenceIndex.reserve(ItemCount);
+	ChunkHashToChunkIndex.reserve(ItemCount);
+	{
+		std::vector<uint32_t> Order;
+		Order.resize(ItemCount);
+		for (uint32_t I = 0; I < ItemCount; I++)
+		{
+			Order[I] = I;
+		}
+
+		// Handle the biggest files first so we don't end up with one straggling large file at the end
+		//		std::sort(Order.begin(), Order.end(), [&](uint32_t Lhs, uint32_t Rhs) { return Result.RawSizes[Lhs] > Result.RawSizes[Rhs];
+		//});
+
+		tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
+		RawHashToSequenceRawHashIndex.reserve(ItemCount);
+
+		RwLock Lock;
+
+		ParallellWork Work(AbortFlag);
+
+		for (uint32_t PathIndex : Order)
+		{
+			if (Work.IsAborted())
+			{
+				break;
+			}
+			Work.ScheduleWork(
+				WorkerPool,	 // GetSyncWorkerPool()
+				[&, PathIndex](std::atomic<bool>& AbortFlag) {
+					if (!AbortFlag)
+					{
+						IoHash RawHash = HashOneFile(Stats,
+													 InChunkingController,
+													 Result,
+													 ChunkHashToChunkIndex,
+													 RawHashToSequenceRawHashIndex,
+													 Lock,
+													 RootPath,
+													 PathIndex,
+													 AbortFlag);
+						Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; });
+						Stats.FilesProcessed++;
+					}
+				},
+				Work.DefaultErrorFunction());
+		}
+
+		Work.Wait(UpdateInteralMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
+			ZEN_UNUSED(IsAborted);
+			ZEN_UNUSED(PendingWork);
+			UpdateCallback(Work.IsAborted(), Work.PendingWork().Remaining());
+		});
+	}
+	return Result;
+}
+
+ChunkedContentLookup
+BuildChunkedContentLookup(const ChunkedFolderContent& Content)
+{
+	ZEN_TRACE_CPU("BuildChunkedContentLookup");
+
+	struct ChunkLocationReference
+	{
+		uint32_t									ChunkIndex = (uint32_t)-1;
+		ChunkedContentLookup::ChunkSequenceLocation Location;
+	};
+
+	ChunkedContentLookup Result;
+	{
+		const uint32_t SequenceRawHashesCount = gsl::narrow<uint32_t>(Content.ChunkedContent.SequenceRawHashes.size());
+		Result.RawHashToSequenceIndex.reserve(SequenceRawHashesCount);
+		Result.SequenceIndexChunkOrderOffset.reserve(SequenceRawHashesCount);
+		uint32_t OrderOffset = 0;
+		for (uint32_t SequenceRawHashIndex = 0; SequenceRawHashIndex < Content.ChunkedContent.SequenceRawHashes.size();
+			 SequenceRawHashIndex++)
+		{
+			Result.RawHashToSequenceIndex.insert({Content.ChunkedContent.SequenceRawHashes[SequenceRawHashIndex], SequenceRawHashIndex});
+			Result.SequenceIndexChunkOrderOffset.push_back(OrderOffset);
+			OrderOffset += Content.ChunkedContent.ChunkCounts[SequenceRawHashIndex];
+		}
+	}
+
+	std::vector<ChunkLocationReference> Locations;
+	Locations.reserve(Content.ChunkedContent.ChunkOrders.size());
+	for (uint32_t SequenceIndex = 0; SequenceIndex < Content.ChunkedContent.SequenceRawHashes.size(); SequenceIndex++)
+	{
+		const uint32_t OrderOffset	  = Result.SequenceIndexChunkOrderOffset[SequenceIndex];
+		const uint32_t ChunkCount	  = Content.ChunkedContent.ChunkCounts[SequenceIndex];
+		uint64_t	   LocationOffset = 0;
+		for (size_t OrderIndex = OrderOffset; OrderIndex < OrderOffset + ChunkCount; OrderIndex++)
+		{
+			uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex];
+
+			Locations.push_back(
+				ChunkLocationReference{ChunkIndex, ChunkedContentLookup::ChunkSequenceLocation{SequenceIndex, LocationOffset}});
+
+			LocationOffset += Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
+		}
+	}
+
+	std::sort(Locations.begin(), Locations.end(), [](const ChunkLocationReference& Lhs, const ChunkLocationReference& Rhs) {
+		if (Lhs.ChunkIndex < Rhs.ChunkIndex)
+		{
+			return true;
+		}
+		if (Lhs.ChunkIndex > Rhs.ChunkIndex)
+		{
+			return false;
+		}
+		if (Lhs.Location.SequenceIndex < Rhs.Location.SequenceIndex)
+		{
+			return true;
+		}
+		if (Lhs.Location.SequenceIndex > Rhs.Location.SequenceIndex)
+		{
+			return false;
+		}
+		return Lhs.Location.Offset < Rhs.Location.Offset;
+	});
+
+	Result.ChunkSequenceLocations.reserve(Locations.size());
+	const uint32_t ChunkCount = gsl::narrow<uint32_t>(Content.ChunkedContent.ChunkHashes.size());
+	Result.ChunkHashToChunkIndex.reserve(ChunkCount);
+	size_t RangeOffset = 0;
+	for (uint32_t ChunkIndex = 0; ChunkIndex < ChunkCount; ChunkIndex++)
+	{
+		Result.ChunkHashToChunkIndex.insert({Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkIndex});
+		uint32_t Count = 0;
+		while ((RangeOffset + Count < Locations.size()) && (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex))
+		{
+			Result.ChunkSequenceLocations.push_back(Locations[RangeOffset + Count].Location);
+			Count++;
+		}
+		Result.ChunkSequenceLocationOffset.push_back(RangeOffset);
+		Result.ChunkSequenceLocationCounts.push_back(Count);
+		RangeOffset += Count;
+	}
+
+	Result.SequenceIndexFirstPathIndex.resize(Content.ChunkedContent.SequenceRawHashes.size(), (uint32_t)-1);
+	for (uint32_t PathIndex = 0; PathIndex < Content.Paths.size(); PathIndex++)
+	{
+		if (Content.RawSizes[PathIndex] > 0)
+		{
+			const IoHash& RawHash		  = Content.RawHashes[PathIndex];
+			auto		  SequenceIndexIt = Result.RawHashToSequenceIndex.find(RawHash);
+			ZEN_ASSERT(SequenceIndexIt != Result.RawHashToSequenceIndex.end());
+			const uint32_t SequenceIndex = SequenceIndexIt->second;
+			if (Result.SequenceIndexFirstPathIndex[SequenceIndex] == (uint32_t)-1)
+			{
+				Result.SequenceIndexFirstPathIndex[SequenceIndex] = PathIndex;
+			}
+		}
+	}
+
+	return Result;
+}
+
+}  // namespace zen
diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp
index c08492eb0..a2c041ffd 100644
--- a/src/zenutil/chunkedfile.cpp
+++ b/src/zenutil/chunkedfile.cpp
@@ -3,6 +3,7 @@
 #include <zenutil/chunkedfile.h>
 
 #include <zencore/basicfile.h>
+#include <zencore/trace.h>
 
 #include "chunking.h"
 
@@ -33,6 +34,7 @@ namespace {
 IoBuffer
 SerializeChunkedInfo(const ChunkedInfo& Info)
 {
+	ZEN_TRACE_CPU("SerializeChunkedInfo");
 	size_t HeaderSize = RoundUp(sizeof(ChunkedHeader), 16) + RoundUp(sizeof(uint32_t) * Info.ChunkSequence.size(), 16) +
 						RoundUp(sizeof(IoHash) * Info.ChunkHashes.size(), 16);
 	IoBuffer HeaderData(HeaderSize);
@@ -65,6 +67,7 @@ SerializeChunkedInfo(const ChunkedInfo& Info)
 ChunkedInfo
 DeserializeChunkedInfo(IoBuffer& Buffer)
 {
+	ZEN_TRACE_CPU("DeserializeChunkedInfo");
 	MemoryView	  View = Buffer.GetView();
 	ChunkedHeader Header;
 	{
@@ -99,6 +102,7 @@ DeserializeChunkedInfo(IoBuffer& Buffer)
 void
 Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk)
 {
+	ZEN_TRACE_CPU("Reconstruct");
 	BasicFile Reconstructed;
 	Reconstructed.Open(TargetPath, BasicFile::Mode::kTruncate);
 	BasicFileWriter ReconstructedWriter(Reconstructed, 64 * 1024);
@@ -112,8 +116,15 @@ Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, st
 }
 
 ChunkedInfoWithSource
-ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params, std::atomic<uint64_t>* BytesProcessed)
+ChunkData(BasicFile&			 RawData,
+		  uint64_t				 Offset,
+		  uint64_t				 Size,
+		  ChunkedParams			 Params,
+		  std::atomic<uint64_t>* BytesProcessed,
+		  std::atomic<bool>*	 AbortFlag)
 {
+	ZEN_TRACE_CPU("ChunkData");
+
 	ChunkedInfoWithSource							 Result;
 	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> FoundChunks;
 
@@ -121,7 +132,7 @@ ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Para
 	Chunker.SetUseThreshold(Params.UseThreshold);
 	Chunker.SetChunkSize(Params.MinSize, Params.MaxSize, Params.AvgSize);
 	size_t			End			   = Offset + Size;
-	const size_t	ScanBufferSize = 1u * 1024 * 1024;	// (Params.MaxSize * 9) / 3;//1 * 1024 * 1024;
+	const size_t	ScanBufferSize = Max(1u * 1024 * 1024, Params.MaxSize);
 	BasicFileBuffer RawBuffer(RawData, ScanBufferSize);
 	MemoryView		SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset);
 	ZEN_ASSERT(!SliceView.IsEmpty());
@@ -129,6 +140,10 @@ ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Para
 	IoHashStream RawHashStream;
 	while (Offset < End)
 	{
+		if (AbortFlag != nullptr && AbortFlag->load())
+		{
+			return {};
+		}
 		size_t ScanLength = Chunker.ScanChunk(SliceView.GetData(), SliceSize);
 		if (ScanLength == ZenChunkHelper::kNoBoundaryFound)
 		{
diff --git a/src/zenutil/chunking.cpp b/src/zenutil/chunking.cpp
index 30edd322a..71f0a06e4 100644
--- a/src/zenutil/chunking.cpp
+++ b/src/zenutil/chunking.cpp
@@ -5,6 +5,7 @@
 #include <gsl/gsl-lite.hpp>
 
 #include <cmath>
+#include <cstring>
 
 namespace zen::detail {
 
diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp
new file mode 100644
index 000000000..2a7057a46
--- /dev/null
+++ b/src/zenutil/chunkingcontroller.cpp
@@ -0,0 +1,275 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/chunkingcontroller.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/trace.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+using namespace std::literals;
+
+namespace {
+	std::vector<std::string> ReadStringArray(CbArrayView StringArray)
+	{
+		std::vector<std::string> Result;
+		Result.reserve(StringArray.Num());
+		for (CbFieldView FieldView : StringArray)
+		{
+			Result.emplace_back(FieldView.AsString());
+		}
+		return Result;
+	}
+
+	ChunkedParams ReadChunkParams(CbObjectView Params)
+	{
+		bool   UseThreshold = Params["UseThreshold"sv].AsBool(true);
+		size_t MinSize		= Params["MinSize"sv].AsUInt64(DefaultChunkedParams.MinSize);
+		size_t MaxSize		= Params["MaxSize"sv].AsUInt64(DefaultChunkedParams.MaxSize);
+		size_t AvgSize		= Params["AvgSize"sv].AsUInt64(DefaultChunkedParams.AvgSize);
+
+		return ChunkedParams{.UseThreshold = UseThreshold, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize};
+	}
+
+}  // namespace
+
+class BasicChunkingController : public ChunkingController
+{
+public:
+	BasicChunkingController(std::span<const std::string_view> ExcludeExtensions,
+							uint64_t						  ChunkFileSizeLimit,
+							const ChunkedParams&			  ChunkingParams)
+	: m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end())
+	, m_ChunkFileSizeLimit(ChunkFileSizeLimit)
+	, m_ChunkingParams(ChunkingParams)
+	{
+	}
+
+	BasicChunkingController(CbObjectView Parameters)
+	: m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()))
+	, m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit))
+	, m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()))
+	{
+	}
+
+	virtual bool ProcessFile(const std::filesystem::path& InputPath,
+							 uint64_t					  RawSize,
+							 ChunkedInfoWithSource&		  OutChunked,
+							 std::atomic<uint64_t>&		  BytesProcessed,
+							 std::atomic<bool>&			  AbortFlag) const override
+	{
+		ZEN_TRACE_CPU("BasicChunkingController::ProcessFile");
+		const bool ExcludeFromChunking =
+			std::find(m_ChunkExcludeExtensions.begin(), m_ChunkExcludeExtensions.end(), InputPath.extension()) !=
+			m_ChunkExcludeExtensions.end();
+
+		if (ExcludeFromChunking || (RawSize < m_ChunkFileSizeLimit))
+		{
+			return false;
+		}
+
+		BasicFile Buffer(InputPath, BasicFile::Mode::kRead);
+		OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag);
+		return true;
+	}
+
+	virtual std::string_view GetName() const override { return Name; }
+
+	virtual CbObject GetParameters() const override
+	{
+		CbObjectWriter Writer;
+		Writer.BeginArray("ChunkExcludeExtensions"sv);
+		{
+			for (const std::string& Extension : m_ChunkExcludeExtensions)
+			{
+				Writer.AddString(Extension);
+			}
+		}
+		Writer.EndArray();	// ChunkExcludeExtensions
+		Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit);
+		Writer.BeginObject("ChunkingParams"sv);
+		{
+			Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold);
+
+			Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize);
+			Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize);
+			Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize);
+		}
+		Writer.EndObject();	 // ChunkingParams
+		return Writer.Save();
+	}
+	static constexpr std::string_view Name = "BasicChunkingController"sv;
+
+protected:
+	const std::vector<std::string> m_ChunkExcludeExtensions;
+	const uint64_t				   m_ChunkFileSizeLimit;
+	const ChunkedParams			   m_ChunkingParams;
+};
+
+class ChunkingControllerWithFixedChunking : public ChunkingController
+{
+public:
+	ChunkingControllerWithFixedChunking(std::span<const std::string_view> FixedChunkingExtensions,
+										uint64_t						  ChunkFileSizeLimit,
+										const ChunkedParams&			  ChunkingParams,
+										uint32_t						  FixedChunkingChunkSize)
+	: m_FixedChunkingExtensions(FixedChunkingExtensions.begin(), FixedChunkingExtensions.end())
+	, m_ChunkFileSizeLimit(ChunkFileSizeLimit)
+	, m_ChunkingParams(ChunkingParams)
+	, m_FixedChunkingChunkSize(FixedChunkingChunkSize)
+	{
+	}
+
+	ChunkingControllerWithFixedChunking(CbObjectView Parameters)
+	: m_FixedChunkingExtensions(ReadStringArray(Parameters["FixedChunkingExtensions"sv].AsArrayView()))
+	, m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit))
+	, m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()))
+	, m_FixedChunkingChunkSize(Parameters["FixedChunkingChunkSize"sv].AsUInt32(16u * 1024u * 1024u))
+	{
+	}
+
+	virtual bool ProcessFile(const std::filesystem::path& InputPath,
+							 uint64_t					  RawSize,
+							 ChunkedInfoWithSource&		  OutChunked,
+							 std::atomic<uint64_t>&		  BytesProcessed,
+							 std::atomic<bool>&			  AbortFlag) const override
+	{
+		ZEN_TRACE_CPU("ChunkingControllerWithFixedChunking::ProcessFile");
+		if (RawSize < m_ChunkFileSizeLimit)
+		{
+			return false;
+		}
+		const bool FixedChunking = std::find(m_FixedChunkingExtensions.begin(), m_FixedChunkingExtensions.end(), InputPath.extension()) !=
+								   m_FixedChunkingExtensions.end();
+
+		if (FixedChunking)
+		{
+			ZEN_TRACE_CPU("FixedChunking");
+			IoHashStream									 FullHash;
+			IoBuffer										 Source = IoBufferBuilder::MakeFromFile(InputPath);
+			uint64_t										 Offset = 0;
+			tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+			ChunkHashToChunkIndex.reserve(1 + (RawSize / m_FixedChunkingChunkSize));
+			while (Offset < RawSize)
+			{
+				if (AbortFlag)
+				{
+					return false;
+				}
+				uint64_t   ChunkSize = std::min<uint64_t>(RawSize - Offset, m_FixedChunkingChunkSize);
+				IoBuffer   Chunk(Source, Offset, ChunkSize);
+				MemoryView ChunkData = Chunk.GetView();
+				FullHash.Append(ChunkData);
+
+				IoHash ChunkHash = IoHash::HashBuffer(ChunkData);
+				if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end())
+				{
+					OutChunked.Info.ChunkSequence.push_back(It->second);
+				}
+				else
+				{
+					uint32_t ChunkIndex = gsl::narrow<uint32_t>(OutChunked.Info.ChunkHashes.size());
+					OutChunked.Info.ChunkHashes.push_back(ChunkHash);
+					OutChunked.Info.ChunkSequence.push_back(ChunkIndex);
+					OutChunked.ChunkSources.push_back({.Offset = Offset, .Size = gsl::narrow<uint32_t>(ChunkSize)});
+				}
+				Offset += ChunkSize;
+				BytesProcessed.fetch_add(ChunkSize);
+			}
+			OutChunked.Info.RawSize = RawSize;
+			OutChunked.Info.RawHash = FullHash.GetHash();
+			return true;
+		}
+		else
+		{
+			BasicFile Buffer(InputPath, BasicFile::Mode::kRead);
+			OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed);
+			return true;
+		}
+	}
+
+	virtual std::string_view GetName() const override { return Name; }
+
+	virtual CbObject GetParameters() const override
+	{
+		CbObjectWriter Writer;
+		Writer.BeginArray("FixedChunkingExtensions");
+		{
+			for (const std::string& Extension : m_FixedChunkingExtensions)
+			{
+				Writer.AddString(Extension);
+			}
+		}
+		Writer.EndArray();	// ChunkExcludeExtensions
+		Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit);
+		Writer.BeginObject("ChunkingParams"sv);
+		{
+			Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold);
+
+			Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize);
+			Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize);
+			Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize);
+		}
+		Writer.EndObject();	 // ChunkingParams
+		Writer.AddInteger("FixedChunkingChunkSize"sv, m_FixedChunkingChunkSize);
+		return Writer.Save();
+	}
+
+	static constexpr std::string_view Name = "ChunkingControllerWithFixedChunking"sv;
+
+protected:
+	const std::vector<std::string> m_FixedChunkingExtensions;
+	const uint64_t				   m_ChunkFileSizeLimit;
+	const ChunkedParams			   m_ChunkingParams;
+	const uint32_t				   m_FixedChunkingChunkSize;
+};
+
+std::unique_ptr<ChunkingController>
+CreateBasicChunkingController(std::span<const std::string_view> ExcludeExtensions,
+							  uint64_t							ChunkFileSizeLimit,
+							  const ChunkedParams&				ChunkingParams)
+{
+	return std::make_unique<BasicChunkingController>(ExcludeExtensions, ChunkFileSizeLimit, ChunkingParams);
+}
+std::unique_ptr<ChunkingController>
+CreateBasicChunkingController(CbObjectView Parameters)
+{
+	return std::make_unique<BasicChunkingController>(Parameters);
+}
+
+std::unique_ptr<ChunkingController>
+CreateChunkingControllerWithFixedChunking(std::span<const std::string_view> FixedChunkingExtensions,
+										  uint64_t							ChunkFileSizeLimit,
+										  const ChunkedParams&				ChunkingParams,
+										  uint32_t							FixedChunkingChunkSize)
+{
+	return std::make_unique<ChunkingControllerWithFixedChunking>(FixedChunkingExtensions,
+																 ChunkFileSizeLimit,
+																 ChunkingParams,
+																 FixedChunkingChunkSize);
+}
+std::unique_ptr<ChunkingController>
+CreateChunkingControllerWithFixedChunking(CbObjectView Parameters)
+{
+	return std::make_unique<ChunkingControllerWithFixedChunking>(Parameters);
+}
+
+std::unique_ptr<ChunkingController>
+CreateChunkingController(std::string_view Name, CbObjectView Parameters)
+{
+	if (Name == BasicChunkingController::Name)
+	{
+		return CreateBasicChunkingController(Parameters);
+	}
+	else if (Name == ChunkingControllerWithFixedChunking::Name)
+	{
+		return CreateChunkingControllerWithFixedChunking(Parameters);
+	}
+	return {};
+}
+
+}  // namespace zen
diff --git a/src/zenutil/chunkrequests.cpp b/src/zenutil/chunkrequests.cpp
index 745363668..e28df02a8 100644
--- a/src/zenutil/chunkrequests.cpp
+++ b/src/zenutil/chunkrequests.cpp
@@ -1,5 +1,7 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
+#include <algorithm>
+
 #include <zenutil/chunkrequests.h>
 
 #include <zencore/blake3.h>
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
new file mode 100644
index 000000000..47a4e1cc4
--- /dev/null
+++ b/src/zenutil/filebuildstorage.cpp
@@ -0,0 +1,644 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/filebuildstorage.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+class FileBuildStorage : public BuildStorage
+{
+public:
+	explicit FileBuildStorage(const std::filesystem::path& StoragePath,
+							  BuildStorage::Statistics&	   Stats,
+							  bool						   EnableJsonOutput,
+							  double					   LatencySec,
+							  double					   DelayPerKBSec)
+	: m_StoragePath(StoragePath)
+	, m_Stats(Stats)
+	, m_EnableJsonOutput(EnableJsonOutput)
+	, m_LatencySec(LatencySec)
+	, m_DelayPerKBSec(DelayPerKBSec)
+	{
+		CreateDirectories(GetBuildsFolder());
+		CreateDirectories(GetBlobsFolder());
+		CreateDirectories(GetBlobsMetadataFolder());
+	}
+
+	virtual ~FileBuildStorage() {}
+
+	virtual CbObject ListBuilds(CbObject Query) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::ListBuilds");
+		ZEN_UNUSED(Query);
+
+		SimulateLatency(Query.GetSize(), 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildFolder = GetBuildsFolder();
+		DirectoryContent			Content;
+		GetDirectoryContent(BuildFolder, DirectoryContentFlags::IncludeDirs, Content);
+		CbObjectWriter Writer;
+		Writer.BeginArray("results");
+		{
+			for (const std::filesystem::path& BuildPath : Content.Directories)
+			{
+				Oid BuildId = Oid::TryFromHexString(BuildPath.stem().string());
+				if (BuildId != Oid::Zero)
+				{
+					Writer.BeginObject();
+					{
+						Writer.AddObjectId("buildId", BuildId);
+						Writer.AddObject("metadata", ReadBuild(BuildId)["metadata"sv].AsObjectView());
+					}
+					Writer.EndObject();
+				}
+			}
+		}
+		Writer.EndArray();	// builds
+		Writer.Save();
+		SimulateLatency(Writer.GetSaveSize(), 0);
+		return Writer.Save();
+	}
+
+	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuild");
+		SimulateLatency(MetaData.GetSize(), 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObjectWriter BuildObject;
+		BuildObject.AddObject("metadata", MetaData);
+		BuildObject.AddInteger("chunkSize"sv, 32u * 1024u * 1024u);
+		WriteBuild(BuildId, BuildObject.Save());
+
+		CbObjectWriter BuildResponse;
+		BuildResponse.AddInteger("chunkSize"sv, 32u * 1024u * 1024u);
+		BuildResponse.Save();
+
+		SimulateLatency(0, BuildResponse.GetSaveSize());
+		return BuildResponse.Save();
+	}
+
+	virtual CbObject GetBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuild");
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObject Build = ReadBuild(BuildId);
+		SimulateLatency(0, Build.GetSize());
+		return Build;
+	}
+
+	virtual void FinalizeBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuild");
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+	}
+
+	virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid&		 BuildId,
+																const Oid&		 BuildPartId,
+																std::string_view PartName,
+																const CbObject&	 MetaData) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildPart");
+		SimulateLatency(MetaData.GetSize(), 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
+		CreateDirectories(BuildPartDataPath.parent_path());
+
+		TemporaryFile::SafeWriteFile(BuildPartDataPath, MetaData.GetView());
+		m_WrittenBytes += MetaData.GetSize();
+		WriteAsJson(BuildPartDataPath, MetaData);
+
+		IoHash RawHash = IoHash::HashBuffer(MetaData.GetView());
+
+		CbObjectWriter Writer;
+		{
+			CbObject	 BuildObject  = ReadBuild(BuildId);
+			CbObjectView PartsObject  = BuildObject["parts"sv].AsObjectView();
+			CbObjectView MetaDataView = BuildObject["metadata"sv].AsObjectView();
+
+			Writer.AddObject("metadata"sv, MetaDataView);
+			Writer.BeginObject("parts"sv);
+			{
+				for (CbFieldView PartView : PartsObject)
+				{
+					if (PartView.GetName() != PartName)
+					{
+						Writer.AddObjectId(PartView.GetName(), PartView.AsObjectId());
+					}
+				}
+				Writer.AddObjectId(PartName, BuildPartId);
+			}
+			Writer.EndObject();	 // parts
+		}
+		WriteBuild(BuildId, Writer.Save());
+
+		std::vector<IoHash> NeededAttachments = GetNeededAttachments(MetaData);
+
+		SimulateLatency(0, sizeof(IoHash) * NeededAttachments.size());
+
+		return std::make_pair(RawHash, std::move(NeededAttachments));
+	}
+
+	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildPart");
+		SimulateLatency(0, 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
+
+		IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten();
+		m_Stats.TotalBytesRead += Payload.GetSize();
+
+		ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None);
+
+		CbObject BuildPartObject = CbObject(SharedBuffer(Payload));
+
+		SimulateLatency(0, BuildPartObject.GetSize());
+
+		return BuildPartObject;
+	}
+
+	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuildPart");
+		SimulateLatency(0, 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId);
+		IoBuffer					Payload			  = ReadFile(BuildPartDataPath).Flatten();
+		m_Stats.TotalBytesRead += Payload.GetSize();
+		IoHash RawHash = IoHash::HashBuffer(Payload.GetView());
+		if (RawHash != PartHash)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed finalizing build part {}: Expected hash {}, got {}", BuildPartId, PartHash, RawHash));
+		}
+
+		CbObject			BuildPartObject = CbObject(SharedBuffer(Payload));
+		std::vector<IoHash> NeededAttachments(GetNeededAttachments(BuildPartObject));
+
+		SimulateLatency(0, NeededAttachments.size() * sizeof(IoHash));
+
+		return NeededAttachments;
+	}
+
+	virtual void PutBuildBlob(const Oid&			 BuildId,
+							  const IoHash&			 RawHash,
+							  ZenContentType		 ContentType,
+							  const CompositeBuffer& Payload) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildBlob");
+		ZEN_UNUSED(BuildId);
+		ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
+		SimulateLatency(Payload.GetSize(), 0);
+
+		ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, Payload));
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (!std::filesystem::is_regular_file(BlockPath))
+		{
+			CreateDirectories(BlockPath.parent_path());
+			TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView());
+		}
+		m_Stats.TotalBytesWritten += Payload.GetSize();
+		SimulateLatency(0, 0);
+	}
+
+	virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid&												   BuildId,
+																 const IoHash&											   RawHash,
+																 ZenContentType											   ContentType,
+																 uint64_t												   PayloadSize,
+																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
+																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob");
+		ZEN_UNUSED(BuildId);
+		ZEN_UNUSED(ContentType);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (!std::filesystem::is_regular_file(BlockPath))
+		{
+			CreateDirectories(BlockPath.parent_path());
+
+			struct WorkloadData
+			{
+				std::function<IoBuffer(uint64_t Offset, uint64_t Size)> Transmitter;
+				std::function<void(uint64_t, bool)>						OnSentBytes;
+				TemporaryFile											TempFile;
+				std::atomic<size_t>										PartsLeft;
+			};
+
+			std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
+			Workload->Transmitter = std::move(Transmitter);
+			Workload->OnSentBytes = std::move(OnSentBytes);
+			std::error_code Ec;
+			Workload->TempFile.CreateTemporary(BlockPath.parent_path(), Ec);
+
+			if (Ec)
+			{
+				throw std::runtime_error(
+					fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value()));
+			}
+
+			std::vector<std::function<void()>> WorkItems;
+			uint64_t						   Offset = 0;
+			while (Offset < PayloadSize)
+			{
+				uint64_t Size = Min(32u * 1024u * 1024u, PayloadSize - Offset);
+
+				WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob_Work");
+					IoBuffer PartPayload = Workload->Transmitter(Offset, Size);
+					SimulateLatency(PartPayload.GetSize(), 0);
+
+					std::error_code Ec;
+					Workload->TempFile.Write(PartPayload, Offset, Ec);
+					if (Ec)
+					{
+						throw std::runtime_error(fmt::format("Failed writing to temporary file '{}': {} ({})",
+															 Workload->TempFile.GetPath(),
+															 Ec.message(),
+															 Ec.value()));
+					}
+					uint64_t BytesWritten = PartPayload.GetSize();
+					m_Stats.TotalBytesWritten += BytesWritten;
+					const bool IsLastPart = Workload->PartsLeft.fetch_sub(1) == 1;
+					if (IsLastPart)
+					{
+						Workload->TempFile.Flush();
+						ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(Workload->TempFile.ReadAll())));
+						Workload->TempFile.MoveTemporaryIntoPlace(BlockPath, Ec);
+						if (Ec)
+						{
+							throw std::runtime_error(fmt::format("Failed moving temporary file '{}' to '{}': {} ({})",
+																 Workload->TempFile.GetPath(),
+																 BlockPath,
+																 Ec.message(),
+																 Ec.value()));
+						}
+					}
+					Workload->OnSentBytes(BytesWritten, IsLastPart);
+					SimulateLatency(0, 0);
+				});
+
+				Offset += Size;
+			}
+			Workload->PartsLeft.store(WorkItems.size());
+
+			SimulateLatency(0, 0);
+			return WorkItems;
+		}
+		SimulateLatency(0, 0);
+		return {};
+	}
+
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlob");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (std::filesystem::is_regular_file(BlockPath))
+		{
+			BasicFile File(BlockPath, BasicFile::Mode::kRead);
+			IoBuffer  Payload;
+			if (RangeOffset != 0 || RangeBytes != (uint64_t)-1)
+			{
+				Payload = IoBuffer(RangeBytes);
+				File.Read(Payload.GetMutableView().GetData(), RangeBytes, RangeOffset);
+			}
+			else
+			{
+				Payload = File.ReadAll();
+				ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload))));
+			}
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+			m_Stats.TotalBytesRead += Payload.GetSize();
+			SimulateLatency(0, Payload.GetSize());
+			return Payload;
+		}
+		SimulateLatency(0, 0);
+		return IoBuffer{};
+	}
+
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(
+		const Oid&																			   BuildId,
+		const IoHash&																		   RawHash,
+		uint64_t																			   ChunkSize,
+		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+		if (std::filesystem::is_regular_file(BlockPath))
+		{
+			struct WorkloadData
+			{
+				std::atomic<uint64_t>																 BytesRemaining;
+				BasicFile																			 BlobFile;
+				std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)> Receiver;
+			};
+
+			std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
+			Workload->BlobFile.Open(BlockPath, BasicFile::Mode::kRead);
+			const uint64_t BlobSize = Workload->BlobFile.FileSize();
+
+			Workload->Receiver		 = std::move(Receiver);
+			Workload->BytesRemaining = BlobSize;
+
+			std::vector<std::function<void()>> WorkItems;
+			uint64_t						   Offset = 0;
+			while (Offset < BlobSize)
+			{
+				uint64_t Size = Min(ChunkSize, BlobSize - Offset);
+				WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob_Work");
+					SimulateLatency(0, 0);
+					IoBuffer PartPayload(Size);
+					Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset);
+					m_Stats.TotalBytesRead += PartPayload.GetSize();
+					uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size);
+					Workload->Receiver(Offset, PartPayload, ByteRemaning);
+					SimulateLatency(Size, PartPayload.GetSize());
+				});
+
+				Offset += Size;
+			}
+			SimulateLatency(0, 0);
+			return WorkItems;
+		}
+		return {};
+	}
+
+	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata");
+		ZEN_UNUSED(BuildId);
+
+		SimulateLatency(MetaData.GetSize(), 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		const std::filesystem::path BlockMetaDataPath = GetBlobMetadataPath(BlockRawHash);
+		CreateDirectories(BlockMetaDataPath.parent_path());
+		TemporaryFile::SafeWriteFile(BlockMetaDataPath, MetaData.GetView());
+		m_Stats.TotalBytesWritten += MetaData.GetSize();
+		WriteAsJson(BlockMetaDataPath, MetaData);
+		SimulateLatency(0, 0);
+	}
+
+	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::FindBlocks");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		DirectoryContent Content;
+		GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content);
+		std::vector<ChunkBlockDescription> Result;
+		for (const std::filesystem::path& MetaDataFile : Content.Files)
+		{
+			IoHash ChunkHash;
+			if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash))
+			{
+				std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash);
+				if (std::filesystem::is_regular_file(BlockPath))
+				{
+					IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
+
+					m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
+
+					CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
+					Result.emplace_back(ParseChunkBlockDescription(BlockObject));
+				}
+			}
+		}
+		SimulateLatency(0, sizeof(IoHash) * Result.size());
+		return Result;
+	}
+
+	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata");
+		ZEN_UNUSED(BuildId);
+		SimulateLatency(0, 0);
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		std::vector<ChunkBlockDescription> Result;
+		for (const IoHash& BlockHash : BlockHashes)
+		{
+			std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash);
+			if (std::filesystem::is_regular_file(MetaDataFile))
+			{
+				IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten();
+
+				m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize();
+
+				CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload));
+				Result.emplace_back(ParseChunkBlockDescription(BlockObject));
+			}
+		}
+		SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size());
+		return Result;
+	}
+
+protected:
+	std::filesystem::path GetBuildsFolder() const { return m_StoragePath / "builds"; }
+	std::filesystem::path GetBlobsFolder() const { return m_StoragePath / "blobs"; }
+	std::filesystem::path GetBlobsMetadataFolder() const { return m_StoragePath / "blocks"; }
+	std::filesystem::path GetBuildFolder(const Oid& BuildId) const { return GetBuildsFolder() / BuildId.ToString(); }
+
+	std::filesystem::path GetBuildPath(const Oid& BuildId) const { return GetBuildFolder(BuildId) / "metadata.cb"; }
+
+	std::filesystem::path GetBuildPartFolder(const Oid& BuildId, const Oid& BuildPartId) const
+	{
+		return GetBuildFolder(BuildId) / "parts" / BuildPartId.ToString();
+	}
+
+	std::filesystem::path GetBuildPartPath(const Oid& BuildId, const Oid& BuildPartId) const
+	{
+		return GetBuildPartFolder(BuildId, BuildPartId) / "metadata.cb";
+	}
+
+	std::filesystem::path GetBlobPayloadPath(const IoHash& RawHash) const { return GetBlobsFolder() / fmt::format("{}.cbz", RawHash); }
+
+	std::filesystem::path GetBlobMetadataPath(const IoHash& RawHash) const
+	{
+		return GetBlobsMetadataFolder() / fmt::format("{}.cb", RawHash);
+	}
+
+	void SimulateLatency(uint64_t ReceiveSize, uint64_t SendSize)
+	{
+		double SleepSec = m_LatencySec;
+		if (m_DelayPerKBSec > 0.0)
+		{
+			SleepSec += m_DelayPerKBSec * (double(SendSize + ReceiveSize) / 1024u);
+		}
+		if (SleepSec > 0)
+		{
+			Sleep(int(SleepSec * 1000));
+		}
+	}
+
+	void WriteAsJson(const std::filesystem::path& OriginalPath, CbObjectView Data) const
+	{
+		if (m_EnableJsonOutput)
+		{
+			ExtendableStringBuilder<128> SB;
+			CompactBinaryToJson(Data, SB);
+			std::filesystem::path JsonPath = OriginalPath;
+			JsonPath.replace_extension(".json");
+			std::string_view JsonMetaData = SB.ToView();
+			TemporaryFile::SafeWriteFile(JsonPath, MemoryView(JsonMetaData.data(), JsonMetaData.length()));
+		}
+	}
+
+	void WriteBuild(const Oid& BuildId, CbObjectView Data)
+	{
+		const std::filesystem::path BuildDataPath = GetBuildPath(BuildId);
+		CreateDirectories(BuildDataPath.parent_path());
+		TemporaryFile::SafeWriteFile(BuildDataPath, Data.GetView());
+		m_Stats.TotalBytesWritten += Data.GetSize();
+		WriteAsJson(BuildDataPath, Data);
+	}
+
+	CbObject ReadBuild(const Oid& BuildId)
+	{
+		const std::filesystem::path BuildDataPath = GetBuildPath(BuildId);
+		FileContents				Content		  = ReadFile(BuildDataPath);
+		if (Content.ErrorCode)
+		{
+			throw std::runtime_error(fmt::format("Failed reading build '{}' from '{}': {} ({})",
+												 BuildId,
+												 BuildDataPath,
+												 Content.ErrorCode.message(),
+												 Content.ErrorCode.value()));
+		}
+		IoBuffer Payload = Content.Flatten();
+		m_Stats.TotalBytesRead += Payload.GetSize();
+		ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None);
+		CbObject BuildObject = CbObject(SharedBuffer(Payload));
+		return BuildObject;
+	}
+
+	std::vector<IoHash> GetNeededAttachments(CbObjectView BuildPartObject)
+	{
+		std::vector<IoHash> NeededAttachments;
+		BuildPartObject.IterateAttachments([&](CbFieldView FieldView) {
+			const IoHash				AttachmentHash = FieldView.AsBinaryAttachment();
+			const std::filesystem::path BlockPath	   = GetBlobPayloadPath(AttachmentHash);
+			if (!std::filesystem::is_regular_file(BlockPath))
+			{
+				NeededAttachments.push_back(AttachmentHash);
+			}
+		});
+		return NeededAttachments;
+	}
+
+	bool ValidateCompressedBuffer(const IoHash& RawHash, const CompositeBuffer& Payload)
+	{
+		IoHash			 VerifyHash;
+		uint64_t		 VerifySize;
+		CompressedBuffer ValidateBuffer = CompressedBuffer::FromCompressed(Payload, VerifyHash, VerifySize);
+		if (!ValidateBuffer)
+		{
+			return false;
+		}
+		if (VerifyHash != RawHash)
+		{
+			return false;
+		}
+		CompositeBuffer Decompressed = ValidateBuffer.DecompressToComposite();
+		if (!Decompressed)
+		{
+			return false;
+		}
+		IoHash Hash = IoHash::HashBuffer(Decompressed);
+		if (Hash != RawHash)
+		{
+			return false;
+		}
+		return true;
+	}
+
+private:
+	const std::filesystem::path m_StoragePath;
+	BuildStorage::Statistics&	m_Stats;
+	const bool					m_EnableJsonOutput = false;
+	std::atomic<uint64_t>		m_WrittenBytes;
+
+	const double m_LatencySec	 = 0.0;
+	const double m_DelayPerKBSec = 0.0;
+};
+
+std::unique_ptr<BuildStorage>
+CreateFileBuildStorage(const std::filesystem::path& StoragePath,
+					   BuildStorage::Statistics&	Stats,
+					   bool							EnableJsonOutput,
+					   double						LatencySec,
+					   double						DelayPerKBSec)
+{
+	return std::make_unique<FileBuildStorage>(StoragePath, Stats, EnableJsonOutput, LatencySec, DelayPerKBSec);
+}
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
new file mode 100644
index 000000000..9d2bab170
--- /dev/null
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -0,0 +1,58 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zenutil/chunkblock.h>
+
+namespace zen {
+
+class BuildStorage
+{
+public:
+	struct Statistics
+	{
+		std::atomic<uint64_t> TotalBytesRead	   = 0;
+		std::atomic<uint64_t> TotalBytesWritten	   = 0;
+		std::atomic<uint64_t> TotalRequestCount	   = 0;
+		std::atomic<uint64_t> TotalRequestTimeUs   = 0;
+		std::atomic<uint64_t> TotalExecutionTimeUs = 0;
+	};
+
+	virtual ~BuildStorage() {}
+
+	virtual CbObject ListBuilds(CbObject Query)								= 0;
+	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0;
+	virtual CbObject GetBuild(const Oid& BuildId)							= 0;
+	virtual void	 FinalizeBuild(const Oid& BuildId)						= 0;
+
+	virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid&		 BuildId,
+																const Oid&		 BuildPartId,
+																std::string_view PartName,
+																const CbObject&	 MetaData)											 = 0;
+	virtual CbObject							   GetBuildPart(const Oid& BuildId, const Oid& BuildPartId)							 = 0;
+	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash)				 = 0;
+	virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0;
+	virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid&												   BuildId,
+																 const IoHash&											   RawHash,
+																 ZenContentType											   ContentType,
+																 uint64_t												   PayloadSize,
+																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
+																 std::function<void(uint64_t, bool)>&&					   OnSentBytes)					 = 0;
+
+	virtual IoBuffer						   GetBuildBlob(const Oid&	  BuildId,
+															const IoHash& RawHash,
+															uint64_t	  RangeOffset = 0,
+															uint64_t	  RangeBytes  = (uint64_t)-1) = 0;
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(
+		const Oid&																			   BuildId,
+		const IoHash&																		   RawHash,
+		uint64_t																			   ChunkSize,
+		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) = 0;
+
+	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData)				 = 0;
+	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId)											 = 0;
+	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/cache/cachekey.h b/src/zenutil/include/zenutil/cache/cachekey.h
index 741375946..0ab05f4f1 100644
--- a/src/zenutil/include/zenutil/cache/cachekey.h
+++ b/src/zenutil/include/zenutil/cache/cachekey.h
@@ -17,6 +17,12 @@ struct CacheKey
 
 	static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; }
 
+	// This should be used whenever the bucket name has already been validated to avoid redundant ToLower calls
+	static CacheKey CreateValidated(std::string&& BucketValidated, const IoHash& Hash)
+	{
+		return {.Bucket = std::move(BucketValidated), .Hash = Hash};
+	}
+
 	auto operator<=>(const CacheKey& that) const
 	{
 		if (auto b = caseSensitiveCompareStrings(Bucket, that.Bucket); b != std::strong_ordering::equal)
diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h
index 9b7414629..277580c74 100644
--- a/src/zenutil/include/zenutil/chunkblock.h
+++ b/src/zenutil/include/zenutil/chunkblock.h
@@ -12,21 +12,29 @@
 
 namespace zen {
 
-struct ChunkBlockDescription
+struct ThinChunkBlockDescription
 {
-	IoHash				  BlockHash;
-	std::vector<IoHash>	  ChunkHashes;
+	IoHash				BlockHash;
+	std::vector<IoHash> ChunkRawHashes;
+};
+
+struct ChunkBlockDescription : public ThinChunkBlockDescription
+{
+	uint64_t			  HeaderSize;
 	std::vector<uint32_t> ChunkRawLengths;
+	std::vector<uint32_t> ChunkCompressedLengths;
 };
 
 std::vector<ChunkBlockDescription> ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject);
 ChunkBlockDescription			   ParseChunkBlockDescription(const CbObjectView& BlockObject);
 CbObject						   BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData);
-
+ChunkBlockDescription			   GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash);
 typedef std::function<std::pair<uint64_t, CompressedBuffer>(const IoHash& RawHash)> FetchChunkFunc;
 
-CompressedBuffer GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock);
-bool			 IterateChunkBlock(const SharedBuffer&														   BlockPayload,
-								   std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor);
+CompressedBuffer	  GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock);
+bool				  IterateChunkBlock(const SharedBuffer&															BlockPayload,
+										std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
+										uint64_t&																	OutHeaderSize);
+std::vector<uint32_t> ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize);
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
new file mode 100644
index 000000000..57b55cb8e
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkedcontent.h
@@ -0,0 +1,283 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/iohash.h>
+
+#include <filesystem>
+#include <vector>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+class CbWriter;
+class ChunkingController;
+class WorkerThreadPool;
+
+enum class SourcePlatform
+{
+	Windows = 0,
+	Linux	= 1,
+	MacOS	= 2,
+	_Count
+};
+
+std::string_view ToString(SourcePlatform Platform);
+SourcePlatform	 FromString(std::string_view Platform, SourcePlatform Default);
+SourcePlatform	 GetSourceCurrentPlatform();
+
+struct FolderContent
+{
+	SourcePlatform					   Platform = GetSourceCurrentPlatform();
+	std::vector<std::filesystem::path> Paths;
+	std::vector<uint64_t>			   RawSizes;
+	std::vector<uint32_t>			   Attributes;
+	std::vector<uint64_t>			   ModificationTicks;
+
+	bool operator==(const FolderContent& Rhs) const;
+
+	bool		AreKnownFilesEqual(const FolderContent& Rhs) const;
+	void		UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& PathIndexesOufOfDate);
+	static bool AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs);
+};
+
+FolderContent GetUpdatedContent(const FolderContent&				Old,
+								const FolderContent&				New,
+								std::vector<std::filesystem::path>& OutDeletedPathIndexes);
+
+void		  SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output);
+FolderContent LoadFolderContentToCompactBinary(CbObjectView Input);
+
+struct GetFolderContentStatistics
+{
+	std::atomic<uint64_t> FoundFileCount		= 0;
+	std::atomic<uint64_t> FoundFileByteCount	= 0;
+	std::atomic<uint64_t> AcceptedFileCount		= 0;
+	std::atomic<uint64_t> AcceptedFileByteCount = 0;
+	uint64_t			  ElapsedWallTimeUS		= 0;
+};
+
+FolderContent GetFolderContent(GetFolderContentStatistics&																Stats,
+							   const std::filesystem::path&																RootPath,
+							   std::function<bool(const std::string_view& RelativePath)>&&								AcceptDirectory,
+							   std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile,
+							   WorkerThreadPool&																		WorkerPool,
+							   int32_t																					UpdateInteralMS,
+							   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						UpdateCallback,
+							   std::atomic<bool>&																		AbortFlag);
+
+struct ChunkedContentData
+{
+	// To describe one asset with a particular RawHash, find the index of the hash in SequenceRawHashes
+	// ChunkCounts for that index will be the number of indexes in ChunkOrders that describe
+	// the sequence of chunks required to reconstruct the asset.
+	// Offset into ChunkOrders is based on how many entries in ChunkOrders the previous [n - 1] SequenceRawHashes uses
+	std::vector<IoHash>	  SequenceRawHashes;  // Raw hash for Chunk sequence
+	std::vector<uint32_t> ChunkCounts;		  // Chunk count of ChunkOrder for SequenceRawHashes[n]
+	std::vector<uint32_t> ChunkOrders;		  // Chunk sequence indexed into ChunkHashes, ChunkCounts[n] indexes per SequenceRawHashes[n]
+	std::vector<IoHash>	  ChunkHashes;		  // Unique chunk hashes
+	std::vector<uint64_t> ChunkRawSizes;	  // Unique chunk raw size for ChunkHash[n]
+};
+
+struct ChunkedFolderContent
+{
+	SourcePlatform					   Platform = GetSourceCurrentPlatform();
+	std::vector<std::filesystem::path> Paths;
+	std::vector<uint64_t>			   RawSizes;
+	std::vector<uint32_t>			   Attributes;
+	std::vector<IoHash>				   RawHashes;
+	ChunkedContentData				   ChunkedContent;
+};
+
+void				 SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output);
+ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input);
+
+ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays);
+ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span<const std::filesystem::path> DeletedPaths);
+
+struct ChunkingStatistics
+{
+	std::atomic<uint64_t> FilesProcessed	   = 0;
+	std::atomic<uint64_t> FilesChunked		   = 0;
+	std::atomic<uint64_t> BytesHashed		   = 0;
+	std::atomic<uint64_t> UniqueChunksFound	   = 0;
+	std::atomic<uint64_t> UniqueSequencesFound = 0;
+	std::atomic<uint64_t> UniqueBytesFound	   = 0;
+	uint64_t			  ElapsedWallTimeUS	   = 0;
+};
+
+ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&												  Stats,
+										WorkerThreadPool&												  WorkerPool,
+										const std::filesystem::path&									  RootPath,
+										const FolderContent&											  Content,
+										const ChunkingController&										  InChunkingController,
+										int32_t															  UpdateInteralMS,
+										std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+										std::atomic<bool>&												  AbortFlag);
+
+struct ChunkedContentLookup
+{
+	struct ChunkSequenceLocation
+	{
+		uint32_t SequenceIndex = (uint32_t)-1;
+		uint64_t Offset		   = (uint64_t)-1;
+	};
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex;
+	std::vector<uint32_t>							 SequenceIndexChunkOrderOffset;
+	std::vector<ChunkSequenceLocation>				 ChunkSequenceLocations;
+	std::vector<size_t>
+		ChunkSequenceLocationOffset;  // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex
+	std::vector<uint32_t> ChunkSequenceLocationCounts;	// ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex
+	std::vector<uint32_t> SequenceIndexFirstPathIndex;	// SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash
+};
+
+ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content);
+
+inline std::pair<size_t, uint32_t>
+GetChunkSequenceLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
+{
+	return std::make_pair(Lookup.ChunkSequenceLocationOffset[ChunkIndex], Lookup.ChunkSequenceLocationCounts[ChunkIndex]);
+}
+
+inline std::span<const ChunkedContentLookup::ChunkSequenceLocation>
+GetChunkSequenceLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
+{
+	std::pair<size_t, uint32_t> Range = GetChunkSequenceLocationRange(Lookup, ChunkIndex);
+	return std::span<const ChunkedContentLookup::ChunkSequenceLocation>(Lookup.ChunkSequenceLocations).subspan(Range.first, Range.second);
+}
+
+inline uint32_t
+GetSequenceIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	return Lookup.RawHashToSequenceIndex.at(RawHash);
+}
+
+inline uint32_t
+GetChunkIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	return Lookup.RawHashToSequenceIndex.at(RawHash);
+}
+
+inline uint32_t
+GetFirstPathIndexForSeqeuenceIndex(const ChunkedContentLookup& Lookup, const uint32_t SequenceIndex)
+{
+	return Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
+}
+
+inline uint32_t
+GetFirstPathIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	const uint32_t SequenceIndex = GetSequenceIndexForRawHash(Lookup, RawHash);
+	return GetFirstPathIndexForSeqeuenceIndex(Lookup, SequenceIndex);
+}
+
+namespace compactbinary_helpers {
+	template<typename Type>
+	void WriteArray(std::span<const Type> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const Type Value : Values)
+		{
+			Output << Value;
+		}
+		Output.EndArray();
+	}
+
+	template<typename Type>
+	void WriteArray(const std::vector<Type>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const Type>(Values), ArrayName, Output);
+	}
+
+	template<>
+	inline void WriteArray(std::span<const std::filesystem::path> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const std::filesystem::path& Path : Values)
+		{
+			Output.AddString((const char*)Path.generic_u8string().c_str());
+		}
+		Output.EndArray();
+	}
+
+	template<>
+	inline void WriteArray(const std::vector<std::filesystem::path>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const std::filesystem::path>(Values), ArrayName, Output);
+	}
+
+	inline void WriteBinaryAttachmentArray(std::span<const IoHash> Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		Output.BeginArray(ArrayName);
+		for (const IoHash& Hash : Values)
+		{
+			Output.AddBinaryAttachment(Hash);
+		}
+		Output.EndArray();
+	}
+
+	inline void WriteBinaryAttachmentArray(const std::vector<IoHash>& Values, std::string_view ArrayName, CbWriter& Output)
+	{
+		WriteArray(std::span<const IoHash>(Values), ArrayName, Output);
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint32_t>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsUInt32());
+		}
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint64_t>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsUInt64());
+		}
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<std::filesystem::path>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			std::u8string_view U8Path = ItemView.AsU8String();
+			Result.push_back(std::filesystem::path(U8Path));
+		}
+	}
+
+	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsHash());
+		}
+	}
+
+	inline void ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result)
+	{
+		CbArrayView Array = Input[ArrayName].AsArrayView();
+		Result.reserve(Array.Num());
+		for (CbFieldView ItemView : Array)
+		{
+			Result.push_back(ItemView.AsBinaryAttachment());
+		}
+	}
+
+}  // namespace compactbinary_helpers
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h
index 7110ad317..4cec80fdb 100644
--- a/src/zenutil/include/zenutil/chunkedfile.h
+++ b/src/zenutil/include/zenutil/chunkedfile.h
@@ -47,7 +47,8 @@ ChunkedInfoWithSource ChunkData(BasicFile&			   RawData,
 								uint64_t			   Offset,
 								uint64_t			   Size,
 								ChunkedParams		   Params		  = {},
-								std::atomic<uint64_t>* BytesProcessed = nullptr);
+								std::atomic<uint64_t>* BytesProcessed = nullptr,
+								std::atomic<bool>*	   AbortFlag	  = nullptr);
 void				  Reconstruct(const ChunkedInfo&							   Info,
 								  const std::filesystem::path&					   TargetPath,
 								  std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk);
diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h
new file mode 100644
index 000000000..246f4498a
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkingcontroller.h
@@ -0,0 +1,56 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+
+#include <zenutil/chunkedfile.h>
+
+#include <atomic>
+#include <filesystem>
+
+namespace zen {
+
+const std::vector<std::string_view> DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"};
+
+const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u,
+											.MaxSize = 128u * 1024u,
+											.AvgSize = ((8u * 4u) * 1024u) + 128u};
+
+const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize;
+
+const uint32_t DefaultFixedChunkingChunkSize = 16u * 1024u * 1024u;
+
+struct ChunkedInfoWithSource;
+
+class ChunkingController
+{
+public:
+	virtual ~ChunkingController() {}
+
+	// Return true if the input file was processed. If true is returned OutChunked will contain the chunked info
+	virtual bool			 ProcessFile(const std::filesystem::path& InputPath,
+										 uint64_t					  RawSize,
+										 ChunkedInfoWithSource&		  OutChunked,
+										 std::atomic<uint64_t>&		  BytesProcessed,
+										 std::atomic<bool>&			  AbortFlag) const = 0;
+	virtual std::string_view GetName() const								 = 0;
+	virtual CbObject		 GetParameters() const							 = 0;
+};
+
+std::unique_ptr<ChunkingController> CreateBasicChunkingController(
+	std::span<const std::string_view> ExcludeExtensions	 = DefaultChunkingExcludeExtensions,
+	uint64_t						  ChunkFileSizeLimit = DefaultChunkingFileSizeLimit,
+	const ChunkedParams&			  ChunkingParams	 = DefaultChunkedParams);
+std::unique_ptr<ChunkingController> CreateBasicChunkingController(CbObjectView Parameters);
+
+std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(
+	std::span<const std::string_view> ExcludeExtensions		 = DefaultChunkingExcludeExtensions,
+	uint64_t						  ChunkFileSizeLimit	 = DefaultChunkingFileSizeLimit,
+	const ChunkedParams&			  ChunkingParams		 = DefaultChunkedParams,
+	uint32_t						  FixedChunkingChunkSize = DefaultFixedChunkingChunkSize);
+std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(CbObjectView Parameters);
+
+std::unique_ptr<ChunkingController> CreateChunkingController(std::string_view Name, CbObjectView Parameters);
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/filebuildstorage.h b/src/zenutil/include/zenutil/filebuildstorage.h
new file mode 100644
index 000000000..c95fb32e6
--- /dev/null
+++ b/src/zenutil/include/zenutil/filebuildstorage.h
@@ -0,0 +1,16 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+#include <zenutil/buildstorage.h>
+
+namespace zen {
+class HttpClient;
+
+std::unique_ptr<BuildStorage> CreateFileBuildStorage(const std::filesystem::path& StoragePath,
+													 BuildStorage::Statistics&	  Stats,
+													 bool						  EnableJsonOutput,
+													 double						  LatencySec	= 0.0,
+													 double						  DelayPerKBSec = 0.0);
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h
new file mode 100644
index 000000000..89fc70140
--- /dev/null
+++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h
@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+#include <zenutil/buildstorage.h>
+
+namespace zen {
+class HttpClient;
+
+std::unique_ptr<BuildStorage> CreateJupiterBuildStorage(LoggerRef					 InLog,
+														HttpClient&					 InHttpClient,
+														BuildStorage::Statistics&	 Stats,
+														std::string_view			 Namespace,
+														std::string_view			 Bucket,
+														const std::filesystem::path& TempFolderPath);
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupiterclient.h b/src/zenutil/include/zenutil/jupiter/jupiterclient.h
index defe50edc..8a51bd60a 100644
--- a/src/zenutil/include/zenutil/jupiter/jupiterclient.h
+++ b/src/zenutil/include/zenutil/jupiter/jupiterclient.h
@@ -44,12 +44,11 @@ public:
 	HttpClient& Client() { return m_HttpClient; }
 
 private:
-	LoggerRef							   m_Log;
-	const std::string					   m_DefaultDdcNamespace;
-	const std::string					   m_DefaultBlobStoreNamespace;
-	const std::string					   m_ComputeCluster;
-	std::function<HttpClientAccessToken()> m_TokenProvider;
-	HttpClient							   m_HttpClient;
+	LoggerRef		  m_Log;
+	const std::string m_DefaultDdcNamespace;
+	const std::string m_DefaultBlobStoreNamespace;
+	const std::string m_ComputeCluster;
+	HttpClient		  m_HttpClient;
 
 	friend class JupiterSession;
 };
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
index 075c35b40..2c5fc73b8 100644
--- a/src/zenutil/include/zenutil/jupiter/jupitersession.h
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -116,21 +116,20 @@ public:
 	JupiterResult	   PutBuildBlob(std::string_view	   Namespace,
 									std::string_view	   BucketId,
 									const Oid&			   BuildId,
-									const Oid&			   PartId,
 									const IoHash&		   Hash,
 									ZenContentType		   ContentType,
 									const CompositeBuffer& Payload);
 	JupiterResult	   GetBuildBlob(std::string_view	  Namespace,
 									std::string_view	  BucketId,
 									const Oid&			  BuildId,
-									const Oid&			  PartId,
 									const IoHash&		  Hash,
-									std::filesystem::path TempFolderPath);
+									std::filesystem::path TempFolderPath,
+									uint64_t			  Offset = 0,
+									uint64_t			  Size	 = (uint64_t)-1);
 
 	JupiterResult			PutMultipartBuildBlob(std::string_view												  Namespace,
 												  std::string_view												  BucketId,
 												  const Oid&													  BuildId,
-												  const Oid&													  PartId,
 												  const IoHash&													  Hash,
 												  ZenContentType												  ContentType,
 												  uint64_t														  PayloadSize,
@@ -139,7 +138,6 @@ public:
 	JupiterResult			GetMultipartBuildBlob(std::string_view																		 Namespace,
 												  std::string_view																		 BucketId,
 												  const Oid&																			 BuildId,
-												  const Oid&																			 PartId,
 												  const IoHash&																			 Hash,
 												  uint64_t																				 ChunkSize,
 												  std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver,
@@ -147,7 +145,6 @@ public:
 	JupiterResult			PutBlockMetadata(std::string_view Namespace,
 											 std::string_view BucketId,
 											 const Oid&		  BuildId,
-											 const Oid&		  PartId,
 											 const IoHash&	  Hash,
 											 const IoBuffer&  Payload);
 	FinalizeBuildPartResult FinalizeBuildPart(std::string_view Namespace,
@@ -155,12 +152,8 @@ public:
 											  const Oid&	   BuildId,
 											  const Oid&	   PartId,
 											  const IoHash&	   RawHash);
-	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
-	JupiterResult			GetBlockMetadata(std::string_view Namespace,
-											 std::string_view BucketId,
-											 const Oid&		  BuildId,
-											 const Oid&		  PartId,
-											 IoBuffer		  Payload);
+	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
+	JupiterResult			GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload);
 
 private:
 	inline LoggerRef Log() { return m_Log; }
diff --git a/src/zenutil/include/zenutil/logging.h b/src/zenutil/include/zenutil/logging.h
index ebf6372fc..d64eef207 100644
--- a/src/zenutil/include/zenutil/logging.h
+++ b/src/zenutil/include/zenutil/logging.h
@@ -32,6 +32,7 @@ struct LoggingOptions
 	bool				  IsDebug		  = false;
 	bool				  IsVerbose		  = false;
 	bool				  IsTest		  = false;
+	bool				  AllowAsync	  = true;
 	bool				  NoConsoleOutput = false;
 	std::filesystem::path AbsLogFile;  // Absolute path to main log file
 	std::string			  LogId;
diff --git a/src/zenutil/include/zenutil/logging/fullformatter.h b/src/zenutil/include/zenutil/logging/fullformatter.h
index 07ad408fa..0326870e5 100644
--- a/src/zenutil/include/zenutil/logging/fullformatter.h
+++ b/src/zenutil/include/zenutil/logging/fullformatter.h
@@ -45,6 +45,8 @@ public:
 
 		std::chrono::seconds TimestampSeconds;
 
+		std::chrono::milliseconds millis;
+
 		if (m_UseFullDate)
 		{
 			TimestampSeconds = std::chrono::duration_cast<std::chrono::seconds>(msg.time.time_since_epoch());
@@ -69,6 +71,8 @@ public:
 				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime);
 				m_CachedDatetime.push_back('.');
 			}
+
+			millis = spdlog::details::fmt_helper::time_fraction<std::chrono::milliseconds>(msg.time);
 		}
 		else
 		{
@@ -97,6 +101,8 @@ public:
 				spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime);
 				m_CachedDatetime.push_back('.');
 			}
+
+			millis = std::chrono::duration_cast<std::chrono::milliseconds>(ElapsedTime - TimestampSeconds);
 		}
 
 		{
@@ -104,7 +110,6 @@ public:
 			OutBuffer.append(m_CachedDatetime.begin(), m_CachedDatetime.end());
 		}
 
-		auto millis = spdlog::details::fmt_helper::time_fraction<std::chrono::milliseconds>(msg.time);
 		spdlog::details::fmt_helper::pad3(static_cast<uint32_t>(millis.count()), OutBuffer);
 		OutBuffer.push_back(']');
 		OutBuffer.push_back(' ');
diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h
new file mode 100644
index 000000000..79798fc8d
--- /dev/null
+++ b/src/zenutil/include/zenutil/parallellwork.h
@@ -0,0 +1,117 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/except.h>
+#include <zencore/fmtutils.h>
+#include <zencore/thread.h>
+#include <zencore/workthreadpool.h>
+
+#include <atomic>
+
+namespace zen {
+
+class ParallellWork
+{
+public:
+	ParallellWork(std::atomic<bool>& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) {}
+
+	~ParallellWork()
+	{
+		// Make sure to call Wait before destroying
+		ZEN_ASSERT(m_PendingWork.Remaining() == 0);
+	}
+
+	std::function<void(const std::exception& Ex, std::atomic<bool>& AbortFlag)> DefaultErrorFunction()
+	{
+		return [&](const std::exception& Ex, std::atomic<bool>& AbortFlag) {
+			m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex.what()); });
+			AbortFlag = true;
+		};
+	}
+
+	void ScheduleWork(WorkerThreadPool&																WorkerPool,
+					  std::function<void(std::atomic<bool>& AbortFlag)>&&							Work,
+					  std::function<void(const std::exception& Ex, std::atomic<bool>& AbortFlag)>&& OnError)
+	{
+		m_PendingWork.AddCount(1);
+		try
+		{
+			WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = std::move(OnError)] {
+				try
+				{
+					Work(m_AbortFlag);
+				}
+				catch (const AssertException& AssertEx)
+				{
+					OnError(
+						std::runtime_error(fmt::format("Caught assert exception while handling request: {}", AssertEx.FullDescription())),
+						m_AbortFlag);
+				}
+				catch (const std::system_error& SystemError)
+				{
+					if (IsOOM(SystemError.code()))
+					{
+						OnError(std::runtime_error(fmt::format("Out of memory. Reason: {}", SystemError.what())), m_AbortFlag);
+					}
+					else if (IsOOD(SystemError.code()))
+					{
+						OnError(std::runtime_error(fmt::format("Out of disk. Reason: {}", SystemError.what())), m_AbortFlag);
+					}
+					else
+					{
+						OnError(std::runtime_error(fmt::format("System error. Reason: {}", SystemError.what())), m_AbortFlag);
+					}
+				}
+				catch (const std::exception& Ex)
+				{
+					OnError(Ex, m_AbortFlag);
+				}
+				m_PendingWork.CountDown();
+			});
+		}
+		catch (const std::exception&)
+		{
+			m_PendingWork.CountDown();
+			throw;
+		}
+	}
+
+	void Abort() { m_AbortFlag = true; }
+
+	bool IsAborted() const { return m_AbortFlag.load(); }
+
+	void Wait(int32_t UpdateInteralMS, std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback)
+	{
+		ZEN_ASSERT(m_PendingWork.Remaining() > 0);
+		m_PendingWork.CountDown();
+		while (!m_PendingWork.Wait(UpdateInteralMS))
+		{
+			UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining());
+		}
+		if (m_Errors.size() == 1)
+		{
+			throw std::runtime_error(m_Errors.front());
+		}
+		else if (m_Errors.size() > 1)
+		{
+			ExtendableStringBuilder<128> SB;
+			SB.Append("Multiple errors:");
+			for (const std::string& Error : m_Errors)
+			{
+				SB.Append(fmt::format("\n    {}", Error));
+			}
+			throw std::runtime_error(SB.ToString());
+		}
+	}
+	Latch& PendingWork() { return m_PendingWork; }
+
+private:
+	std::atomic<bool>& m_AbortFlag;
+	Latch			   m_PendingWork;
+
+	RwLock					 m_ErrorLock;
+	std::vector<std::string> m_Errors;
+};
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/service.h b/src/zenutil/include/zenutil/service.h
new file mode 100644
index 000000000..2798bcb1f
--- /dev/null
+++ b/src/zenutil/include/zenutil/service.h
@@ -0,0 +1,56 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/zenbase.h>
+
+#include <filesystem>
+#include <optional>
+
+namespace zen {
+
+enum class ServiceLevel
+{
+	CurrentUser,
+	AllUsers,
+	SystemService
+};
+
+struct ServiceSpec
+{
+	std::filesystem::path ExecutablePath;
+	std::string			  CommandLineOptions;
+	std::string			  UserName;
+#if ZEN_PLATFORM_WINDOWS
+	std::string DisplayName;
+	std::string Description;
+#endif	// ZEN_PLATFORM_WINDOWS
+};
+
+enum class ServiceStatus
+{
+	NotInstalled,
+	Starting,
+	Running,
+	Stopping,
+	Stopped,
+	Pausing,
+	Paused,
+	Resuming
+};
+
+struct ServiceInfo
+{
+	ServiceStatus Status;
+	ServiceSpec	  Spec;
+};
+
+std::string_view ToString(ServiceStatus Status);
+
+std::error_code InstallService(std::string_view ServiceName, const ServiceSpec& Spec);
+std::error_code UninstallService(std::string_view ServiceName);
+std::error_code QueryInstalledService(std::string_view ServiceName, ServiceInfo& OutInfo);
+std::error_code StartService(std::string_view ServiceName);
+std::error_code StopService(std::string_view ServiceName);
+
+}  // namespace zen
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
new file mode 100644
index 000000000..bf89ce785
--- /dev/null
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -0,0 +1,404 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/jupiter/jupiterbuildstorage.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zenutil/jupiter/jupitersession.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+class JupiterBuildStorage : public BuildStorage
+{
+public:
+	JupiterBuildStorage(LoggerRef					 InLog,
+						HttpClient&					 InHttpClient,
+						Statistics&					 Stats,
+						std::string_view			 Namespace,
+						std::string_view			 Bucket,
+						const std::filesystem::path& TempFolderPath)
+	: m_Session(InLog, InHttpClient)
+	, m_Stats(Stats)
+	, m_Namespace(Namespace)
+	, m_Bucket(Bucket)
+	, m_TempFolderPath(TempFolderPath)
+	{
+	}
+	virtual ~JupiterBuildStorage() {}
+
+	virtual CbObject ListBuilds(CbObject Query) override
+	{
+		ZEN_TRACE_CPU("Jupiter::ListBuilds");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = Query.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult ListResult = m_Session.ListBuilds(m_Namespace, m_Bucket, Payload);
+		AddStatistic(ListResult);
+		if (!ListResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+		}
+		return PayloadToJson("Failed listing builds"sv, ListResult.Response);
+	}
+
+	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBuild");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult PutResult = m_Session.PutBuild(m_Namespace, m_Bucket, BuildId, Payload);
+		AddStatistic(PutResult);
+		if (!PutResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode));
+		}
+		return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
+	}
+
+	virtual CbObject GetBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBuild");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _				 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult GetBuildResult = m_Session.GetBuild(m_Namespace, m_Bucket, BuildId);
+		AddStatistic(GetBuildResult);
+		if (!GetBuildResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode));
+		}
+		return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
+	}
+
+	virtual void FinalizeBuild(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuild");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _					  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult FinalizeBuildResult = m_Session.FinalizeBuild(m_Namespace, m_Bucket, BuildId);
+		AddStatistic(FinalizeBuildResult);
+		if (!FinalizeBuildResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed finalizing build part: {} ({})", FinalizeBuildResult.Reason, FinalizeBuildResult.ErrorCode));
+		}
+	}
+
+	virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid&		 BuildId,
+																const Oid&		 BuildPartId,
+																std::string_view PartName,
+																const CbObject&	 MetaData) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildPart");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		PutBuildPartResult PutPartResult = m_Session.PutBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartName, Payload);
+		AddStatistic(PutPartResult);
+		if (!PutPartResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed creating build part: {} ({})", PutPartResult.Reason, PutPartResult.ErrorCode));
+		}
+		return std::make_pair(PutPartResult.RawHash, std::move(PutPartResult.Needs));
+	}
+
+	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildPart");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult GetBuildPartResult = m_Session.GetBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId);
+		AddStatistic(GetBuildPartResult);
+		if (!GetBuildPartResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed fetching build part {}: {} ({})",
+												 BuildPartId,
+												 GetBuildPartResult.Reason,
+												 GetBuildPartResult.ErrorCode));
+		}
+		return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
+	}
+
+	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
+	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuildPart");
+
+		Stopwatch				ExecutionTimer;
+		auto					_ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		FinalizeBuildPartResult FinalizePartResult = m_Session.FinalizeBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartHash);
+		AddStatistic(FinalizePartResult);
+		if (!FinalizePartResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed finalizing build part: {} ({})", FinalizePartResult.Reason, FinalizePartResult.ErrorCode));
+		}
+		return std::move(FinalizePartResult.Needs);
+	}
+
+	virtual void PutBuildBlob(const Oid&			 BuildId,
+							  const IoHash&			 RawHash,
+							  ZenContentType		 ContentType,
+							  const CompositeBuffer& Payload) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildBlob");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _				= MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult PutBlobResult = m_Session.PutBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ContentType, Payload);
+		AddStatistic(PutBlobResult);
+		if (!PutBlobResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PutBlobResult.Reason, PutBlobResult.ErrorCode));
+		}
+	}
+
+	virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid&												   BuildId,
+																 const IoHash&											   RawHash,
+																 ZenContentType											   ContentType,
+																 uint64_t												   PayloadSize,
+																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
+																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutLargeBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		std::vector<std::function<JupiterResult(bool&)>> WorkItems;
+		JupiterResult									 PutMultipartBlobResult = m_Session.PutMultipartBuildBlob(m_Namespace,
+																				  m_Bucket,
+																				  BuildId,
+																				  RawHash,
+																				  ContentType,
+																				  PayloadSize,
+																				  std::move(Transmitter),
+																				  WorkItems);
+		AddStatistic(PutMultipartBlobResult);
+		if (!PutMultipartBlobResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed putting build part: {} ({})", PutMultipartBlobResult.Reason, PutMultipartBlobResult.ErrorCode));
+		}
+		OnSentBytes(PutMultipartBlobResult.SentBytes, WorkItems.empty());
+
+		std::vector<std::function<void()>> WorkList;
+		for (auto& WorkItem : WorkItems)
+		{
+			WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes]() {
+				Stopwatch	  ExecutionTimer;
+				auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+				bool		  IsComplete = false;
+				JupiterResult PartResult = WorkItem(IsComplete);
+				AddStatistic(PartResult);
+				if (!PartResult.Success)
+				{
+					throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode));
+				}
+				OnSentBytes(PartResult.SentBytes, IsComplete);
+			});
+		}
+		return WorkList;
+	}
+
+	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		CreateDirectories(m_TempFolderPath);
+		JupiterResult GetBuildBlobResult =
+			m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, RangeOffset, RangeBytes);
+		AddStatistic(GetBuildBlobResult);
+		if (!GetBuildBlobResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed fetching build blob {}: {} ({})", RawHash, GetBuildBlobResult.Reason, GetBuildBlobResult.ErrorCode));
+		}
+		return std::move(GetBuildBlobResult.Response);
+	}
+
+	virtual std::vector<std::function<void()>> GetLargeBuildBlob(
+		const Oid&																			   BuildId,
+		const IoHash&																		   RawHash,
+		uint64_t																			   ChunkSize,
+		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob");
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		std::vector<std::function<JupiterResult()>> WorkItems;
+		JupiterResult								GetMultipartBlobResult =
+			m_Session.GetMultipartBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ChunkSize, std::move(Receiver), WorkItems);
+
+		AddStatistic(GetMultipartBlobResult);
+		if (!GetMultipartBlobResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed getting build part: {} ({})", GetMultipartBlobResult.Reason, GetMultipartBlobResult.ErrorCode));
+		}
+		std::vector<std::function<void()>> WorkList;
+		for (auto& WorkItem : WorkItems)
+		{
+			WorkList.emplace_back([this, WorkItem = std::move(WorkItem)]() {
+				Stopwatch	  ExecutionTimer;
+				auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+				JupiterResult PartResult = WorkItem();
+				AddStatistic(PartResult);
+				if (!PartResult.Success)
+				{
+					throw std::runtime_error(fmt::format("Failed getting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode));
+				}
+			});
+		}
+		return WorkList;
+	}
+
+	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	{
+		ZEN_TRACE_CPU("Jupiter::PutBlockMetadata");
+
+		Stopwatch ExecutionTimer;
+		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult PutMetaResult = m_Session.PutBlockMetadata(m_Namespace, m_Bucket, BuildId, BlockRawHash, Payload);
+		AddStatistic(PutMetaResult);
+		if (!PutMetaResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode));
+		}
+	}
+
+	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
+	{
+		ZEN_TRACE_CPU("Jupiter::FindBlocks");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId);
+		AddStatistic(FindResult);
+		if (!FindResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode));
+		}
+		return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response));
+	}
+
+	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+	{
+		ZEN_TRACE_CPU("Jupiter::GetBlockMetadata");
+
+		Stopwatch	   ExecutionTimer;
+		auto		   _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		CbObjectWriter Request;
+
+		Request.BeginArray("blocks"sv);
+		for (const IoHash& BlockHash : BlockHashes)
+		{
+			Request.AddHash(BlockHash);
+		}
+		Request.EndArray();
+
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult GetBlockMetadataResult = m_Session.GetBlockMetadata(m_Namespace, m_Bucket, BuildId, Payload);
+		AddStatistic(GetBlockMetadataResult);
+		if (!GetBlockMetadataResult.Success)
+		{
+			throw std::runtime_error(
+				fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode));
+		}
+		std::vector<ChunkBlockDescription> UnorderedList =
+			ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response));
+		tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup;
+		for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++)
+		{
+			const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex];
+			BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex);
+		}
+		std::vector<ChunkBlockDescription> SortedBlockDescriptions;
+		SortedBlockDescriptions.reserve(BlockDescriptionLookup.size());
+		for (const IoHash& BlockHash : BlockHashes)
+		{
+			if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end())
+			{
+				SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second]));
+			}
+		}
+		return SortedBlockDescriptions;
+	}
+
+private:
+	static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload)
+	{
+		if (Payload.GetContentType() == ZenContentType::kJSON)
+		{
+			std::string_view Json(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize());
+			return LoadCompactBinaryFromJson(Json).AsObject();
+		}
+		else if (Payload.GetContentType() == ZenContentType::kCbObject)
+		{
+			return LoadCompactBinaryObject(Payload);
+		}
+		else if (Payload.GetContentType() == ZenContentType::kCompressedBinary)
+		{
+			IoHash	 RawHash;
+			uint64_t RawSize;
+			return LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize));
+		}
+		else
+		{
+			throw std::runtime_error(
+				fmt::format("{}: {} ({})", "Unsupported response format", Context, ToString(Payload.GetContentType())));
+		}
+	}
+
+	void AddStatistic(const JupiterResult& Result)
+	{
+		m_Stats.TotalBytesWritten += Result.SentBytes;
+		m_Stats.TotalBytesRead += Result.ReceivedBytes;
+		m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0);
+		m_Stats.TotalRequestCount++;
+	}
+
+	JupiterSession				m_Session;
+	Statistics&					m_Stats;
+	const std::string			m_Namespace;
+	const std::string			m_Bucket;
+	const std::filesystem::path m_TempFolderPath;
+};
+
+std::unique_ptr<BuildStorage>
+CreateJupiterBuildStorage(LoggerRef					   InLog,
+						  HttpClient&				   InHttpClient,
+						  BuildStorage::Statistics&	   Stats,
+						  std::string_view			   Namespace,
+						  std::string_view			   Bucket,
+						  const std::filesystem::path& TempFolderPath)
+{
+	ZEN_TRACE_CPU("CreateJupiterBuildStorage");
+
+	return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath);
+}
+
+}  // namespace zen
diff --git a/src/zenutil/jupiter/jupiterclient.cpp b/src/zenutil/jupiter/jupiterclient.cpp
index 5e5da3750..dbac218a4 100644
--- a/src/zenutil/jupiter/jupiterclient.cpp
+++ b/src/zenutil/jupiter/jupiterclient.cpp
@@ -11,7 +11,6 @@ JupiterClient::JupiterClient(const JupiterClientOptions& Options, std::function<
 , m_DefaultDdcNamespace(Options.DdcNamespace)
 , m_DefaultBlobStoreNamespace(Options.BlobStoreNamespace)
 , m_ComputeCluster(Options.ComputeCluster)
-, m_TokenProvider(std::move(TokenProvider))
 , m_HttpClient(Options.ServiceUrl,
 			   HttpClientSettings{.ConnectTimeout	   = Options.ConnectTimeout,
 								  .Timeout			   = Options.Timeout,
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index d56927b44..68f214c06 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -436,15 +436,14 @@ JupiterResult
 JupiterSession::PutBuildBlob(std::string_view		Namespace,
 							 std::string_view		BucketId,
 							 const Oid&				BuildId,
-							 const Oid&				PartId,
 							 const IoHash&			Hash,
 							 ZenContentType			ContentType,
 							 const CompositeBuffer& Payload)
 {
-	HttpClient::Response Response = m_HttpClient.Upload(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		Payload,
-		ContentType);
+	HttpClient::Response Response =
+		m_HttpClient.Upload(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
+							Payload,
+							ContentType);
 	return detail::ConvertResponse(Response, "JupiterSession::PutBuildBlob"sv);
 }
 
@@ -452,7 +451,6 @@ JupiterResult
 JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 									  std::string_view												  BucketId,
 									  const Oid&													  BuildId,
-									  const Oid&													  PartId,
 									  const IoHash&													  Hash,
 									  ZenContentType												  ContentType,
 									  uint64_t														  PayloadSize,
@@ -498,12 +496,8 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 	StartMultipartPayloadWriter.AddInteger("blobLength"sv, PayloadSize);
 	CbObject StartMultipartPayload = StartMultipartPayloadWriter.Save();
 
-	std::string StartMultipartResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/startMultipartUpload",
-																  Namespace,
-																  BucketId,
-																  BuildId,
-																  PartId,
-																  Hash.ToHexString());
+	std::string StartMultipartResponseRequestString =
+		fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/startMultipartUpload", Namespace, BucketId, BuildId, Hash.ToHexString());
 	// ZEN_INFO("POST: {}", StartMultipartResponseRequestString);
 	HttpClient::Response StartMultipartResponse =
 		m_HttpClient.Post(StartMultipartResponseRequestString, StartMultipartPayload, HttpClient::Accept(ZenContentType::kCbObject));
@@ -529,15 +523,14 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 
 	for (size_t PartIndex = 0; PartIndex < Workload->PartDescription.Parts.size(); PartIndex++)
 	{
-		OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, PartId, Hash, ContentType, Workload, PartIndex](
+		OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, Hash, ContentType, Workload, PartIndex](
 									  bool& OutIsComplete) -> JupiterResult {
 			const MultipartUploadResponse::Part& Part		 = Workload->PartDescription.Parts[PartIndex];
 			IoBuffer							 PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte);
-			std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}",
+			std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
 																		   Namespace,
 																		   BucketId,
 																		   BuildId,
-																		   PartId,
 																		   Hash.ToHexString(),
 																		   Part.QueryString);
 			// ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString);
@@ -571,12 +564,7 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 					CbObject CompletePayload = CompletePayloadWriter.Save();
 
 					std::string MultipartEndResponseRequestString =
-						fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/completeMultipart",
-									Namespace,
-									BucketId,
-									BuildId,
-									PartId,
-									Hash.ToHexString());
+						fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/completeMultipart", Namespace, BucketId, BuildId, Hash.ToHexString());
 
 					MultipartEndResponse = m_HttpClient.Post(MultipartEndResponseRequestString,
 															 CompletePayload,
@@ -600,13 +588,12 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 								size_t								 RetryPartIndex = PartNameToIndex.at(RetryPartId);
 								const MultipartUploadResponse::Part& RetryPart		= Workload->PartDescription.Parts[RetryPartIndex];
 								IoBuffer							 RetryPartPayload =
-									Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte);
+									Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte - 1);
 								std::string RetryMultipartUploadResponseRequestString =
-									fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}",
+									fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
 												Namespace,
 												BucketId,
 												BuildId,
-												RetryPartId,
 												Hash.ToHexString(),
 												RetryPart.QueryString);
 
@@ -642,14 +629,12 @@ JupiterResult
 JupiterSession::GetMultipartBuildBlob(std::string_view																		 Namespace,
 									  std::string_view																		 BucketId,
 									  const Oid&																			 BuildId,
-									  const Oid&																			 PartId,
 									  const IoHash&																			 Hash,
 									  uint64_t																				 ChunkSize,
 									  std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver,
 									  std::vector<std::function<JupiterResult()>>&											 OutWorkItems)
 {
-	std::string RequestUrl =
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString());
+	std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
 	HttpClient::Response Response =
 		m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}}));
 	if (Response.IsSuccess())
@@ -683,17 +668,12 @@ JupiterSession::GetMultipartBuildBlob(std::string_view																		 Namespa
 						{
 							uint64_t PartSize = Min(ChunkSize, TotalSize - Offset);
 							OutWorkItems.emplace_back(
-								[this, Namespace, BucketId, BuildId, PartId, Hash, TotalSize, Workload, Offset, PartSize]()
-									-> JupiterResult {
-									std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}",
-																		  Namespace,
-																		  BucketId,
-																		  BuildId,
-																		  PartId,
-																		  Hash.ToHexString());
-									HttpClient::Response Response	= m_HttpClient.Get(
-										  RequestUrl,
-										  HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
+								[this, Namespace, BucketId, BuildId, Hash, TotalSize, Workload, Offset, PartSize]() -> JupiterResult {
+									std::string RequestUrl =
+										fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
+									HttpClient::Response Response = m_HttpClient.Get(
+										RequestUrl,
+										HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
 									if (Response.IsSuccess())
 									{
 										uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize());
@@ -717,13 +697,20 @@ JupiterResult
 JupiterSession::GetBuildBlob(std::string_view	   Namespace,
 							 std::string_view	   BucketId,
 							 const Oid&			   BuildId,
-							 const Oid&			   PartId,
 							 const IoHash&		   Hash,
-							 std::filesystem::path TempFolderPath)
+							 std::filesystem::path TempFolderPath,
+							 uint64_t			   Offset,
+							 uint64_t			   Size)
 {
-	HttpClient::Response Response = m_HttpClient.Download(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		TempFolderPath);
+	HttpClient::KeyValueMap Headers;
+	if (Offset != 0 || Size != (uint64_t)-1)
+	{
+		Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", Offset, Offset + Size - 1)});
+	}
+	HttpClient::Response Response =
+		m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
+							  TempFolderPath,
+							  Headers);
 
 	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
 }
@@ -732,14 +719,13 @@ JupiterResult
 JupiterSession::PutBlockMetadata(std::string_view Namespace,
 								 std::string_view BucketId,
 								 const Oid&		  BuildId,
-								 const Oid&		  PartId,
 								 const IoHash&	  Hash,
 								 const IoBuffer&  Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
-	HttpClient::Response Response = m_HttpClient.Put(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		Payload);
+	HttpClient::Response Response =
+		m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, Hash.ToHexString()),
+						 Payload);
 	return detail::ConvertResponse(Response, "JupiterSession::PutBlockMetadata"sv);
 }
 
@@ -772,24 +758,19 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace,
 }
 
 JupiterResult
-JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId)
+JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId)
 {
-	HttpClient::Response Response =
-		m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/listBlocks", Namespace, BucketId, BuildId, PartId),
-						 HttpClient::Accept(ZenContentType::kCbObject));
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId),
+													 HttpClient::Accept(ZenContentType::kCbObject));
 	return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv);
 }
 
 JupiterResult
-JupiterSession::GetBlockMetadata(std::string_view Namespace,
-								 std::string_view BucketId,
-								 const Oid&		  BuildId,
-								 const Oid&		  PartId,
-								 IoBuffer		  Payload)
+JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
 	HttpClient::Response Response =
-		m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId, PartId),
+		m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId),
 						  Payload,
 						  HttpClient::Accept(ZenContentType::kCbObject));
 	return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv);
diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp
index 6314c407f..cb0fd6679 100644
--- a/src/zenutil/logging.cpp
+++ b/src/zenutil/logging.cpp
@@ -10,11 +10,13 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <spdlog/spdlog.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#include <zencore/callstack.h>
 #include <zencore/compactbinary.h>
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
 #include <zencore/memory/llm.h>
 #include <zencore/string.h>
+#include <zencore/timer.h>
 #include <zenutil/logging/fullformatter.h>
 #include <zenutil/logging/jsonformatter.h>
 #include <zenutil/logging/rotatingfilesink.h>
@@ -48,7 +50,7 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
 	zen::logging::InitializeLogging();
 	zen::logging::EnableVTMode();
 
-	bool IsAsync = true;
+	bool IsAsync = LogOptions.AllowAsync;
 
 	if (LogOptions.IsDebug)
 	{
@@ -97,7 +99,13 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
 		}
 	}
 
-	std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); });
+	std::set_terminate([]() {
+		void*			 Frames[8];
+		uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+		CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+		ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, "  "));
+		FreeCallstack(Callstack);
+	});
 
 	// Default
 
@@ -174,7 +182,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 		LogLevel = logging::level::Debug;
 	}
 
-	if (LogOptions.IsTest)
+	if (LogOptions.IsTest || LogOptions.IsVerbose)
 	{
 		LogLevel = logging::level::Trace;
 	}
@@ -184,21 +192,25 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 	logging::RefreshLogLevels(LogLevel);
 	spdlog::flush_on(spdlog::level::err);
 	spdlog::flush_every(std::chrono::seconds{2});
-	spdlog::set_formatter(
-		std::make_unique<logging::full_formatter>(LogOptions.LogId, std::chrono::system_clock::now()));	 // default to duration prefix
+	spdlog::set_formatter(std::make_unique<logging::full_formatter>(
+		LogOptions.LogId,
+		std::chrono::system_clock::now() - std::chrono::milliseconds(GetTimeSinceProcessStart())));	 // default to duration prefix
 
-	if (LogOptions.AbsLogFile.extension() == ".json")
+	if (g_FileSink)
 	{
-		g_FileSink->set_formatter(std::make_unique<logging::json_formatter>(LogOptions.LogId));
-	}
-	else
-	{
-		g_FileSink->set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId));	 // this will have a date prefix
-	}
+		if (LogOptions.AbsLogFile.extension() == ".json")
+		{
+			g_FileSink->set_formatter(std::make_unique<logging::json_formatter>(LogOptions.LogId));
+		}
+		else
+		{
+			g_FileSink->set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId));	 // this will have a date prefix
+		}
 
-	const std::string StartLogTime = zen::DateTime::Now().ToIso8601();
+		const std::string StartLogTime = zen::DateTime::Now().ToIso8601();
 
-	spdlog::apply_all([&](auto Logger) { Logger->info("log starting at {}", StartLogTime); });
+		spdlog::apply_all([&](auto Logger) { Logger->info("log starting at {}", StartLogTime); });
+	}
 
 	g_IsLoggingInitialized = true;
 }
@@ -206,7 +218,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 void
 ShutdownLogging()
 {
-	if (g_IsLoggingInitialized)
+	if (g_IsLoggingInitialized && g_FileSink)
 	{
 		auto DefaultLogger = zen::logging::Default();
 		ZEN_LOG_INFO(DefaultLogger, "log ending at {}", zen::DateTime::Now().ToIso8601());
diff --git a/src/zenutil/service.cpp b/src/zenutil/service.cpp
new file mode 100644
index 000000000..6e68a01fa
--- /dev/null
+++ b/src/zenutil/service.cpp
@@ -0,0 +1,1077 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/service.h>
+
+#include <zencore/except.h>
+#include <zencore/process.h>
+#include <zencore/scopeguard.h>
+#include <zencore/zencore.h>
+#include <string_view>
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#endif
+#if ZEN_PLATFORM_MAC
+#	include <zencore/filesystem.h>
+#	include <zencore/fmtutils.h>
+
+#	include <unistd.h>
+#	include <sys/stat.h>
+#endif
+#if ZEN_PLATFORM_LINUX
+#	include <zencore/filesystem.h>
+#	include <zencore/fmtutils.h>
+
+#	include <unistd.h>
+#	include <sys/stat.h>
+#	include <regex>
+#endif
+
+namespace zen {
+using namespace std::literals;
+
+namespace {
+#if ZEN_PLATFORM_WINDOWS
+
+	bool SplitExecutableAndArgs(const std::wstring& ExeAndArgs, std::filesystem::path& OutExecutablePath, std::string& OutArguments)
+	{
+		if (ExeAndArgs.size())
+		{
+			if (ExeAndArgs[0] == '"')
+			{
+				std::wstring::size_type ExecutableEnd = ExeAndArgs.find('"', 1);
+				if (ExecutableEnd == std::wstring::npos)
+				{
+					OutExecutablePath = ExeAndArgs;
+					return true;
+				}
+				else
+				{
+					OutExecutablePath = ExeAndArgs.substr(0, ExecutableEnd + 1);
+					OutArguments	  = WideToUtf8(ExeAndArgs.substr(ExecutableEnd + 1 + ExeAndArgs[ExecutableEnd + 1] == ' ' ? 1 : 0));
+					return true;
+				}
+			}
+			else
+			{
+				std::wstring::size_type ExecutableEnd = ExeAndArgs.find(' ', 1);
+				if (ExecutableEnd == std::wstring::npos)
+				{
+					OutExecutablePath = ExeAndArgs;
+					return true;
+				}
+				else
+				{
+					OutExecutablePath = ExeAndArgs.substr(0, ExecutableEnd);
+					OutArguments	  = WideToUtf8(ExeAndArgs.substr(ExecutableEnd + 1));
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+#endif	// ZEN_PLATFORM_WINDOWS
+
+#if ZEN_PLATFORM_MAC
+	std::vector<std::string_view> SplitArguments(std::string_view Arguments)
+	{
+		bool						  IsQuote = false;
+		size_t						  Start	  = 0;
+		size_t						  Offset  = 0;
+		std::vector<std::string_view> Result;
+		for (; Offset < Arguments.length(); Offset++)
+		{
+			switch (Arguments[Offset])
+			{
+				case ' ':
+					if (IsQuote)
+					{
+						continue;
+					}
+					else if (Offset > Start)
+					{
+						Result.push_back(Arguments.substr(Start, Offset - Start));
+						Start = Offset + 1;
+					}
+					break;
+				case '"':
+					if (IsQuote)
+					{
+						IsQuote = false;
+						if (Offset - Start > 1)
+						{
+							Result.push_back(Arguments.substr(Start + 1, Offset - (Start + 1)));
+						}
+						Start = Offset + 1;
+					}
+					else
+					{
+						IsQuote = true;
+					}
+					break;
+				case '=':
+					if (IsQuote)
+					{
+						continue;
+					}
+					else if (Offset > Start)
+					{
+						Result.push_back(Arguments.substr(Start, Offset - Start));
+						Start = Offset + 1;
+					}
+					break;
+				default:
+					break;
+			}
+		}
+		ZEN_ASSERT(!IsQuote);
+		if (Offset > Start)
+		{
+			Result.push_back(Arguments.substr(Start, Offset - Start));
+		}
+		return Result;
+	}
+
+	// Needs special character escaping
+	void AppendEscaped(std::string_view String, StringBuilderBase& SB)
+	{
+		size_t Offset = 0;
+		while (Offset < String.length())
+		{
+			size_t NextEscapeCharacter = String.find_first_of("\"'<>&", Offset);
+			if (NextEscapeCharacter == std::string_view::npos)
+			{
+				break;
+			}
+			if (NextEscapeCharacter > Offset)
+			{
+				SB.Append(String.substr(Offset, NextEscapeCharacter - Offset));
+			}
+			switch (String[NextEscapeCharacter])
+			{
+				case '"':
+					SB.Append("&quot");
+					break;
+				case '\'':
+					SB.Append("&apos");
+					break;
+				case '<':
+					SB.Append("&lt");
+					break;
+				case '>':
+					SB.Append("&gt");
+					break;
+				case '&':
+					SB.Append("&amp");
+					break;
+				default:
+					ZEN_ASSERT(false);
+					break;
+			}
+			Offset = NextEscapeCharacter + 1;
+		}
+		if (Offset == 0)
+		{
+			SB.Append(String);
+		}
+		else if (String.length() > Offset)
+		{
+			SB.Append(String.substr(Offset));
+		}
+	}
+
+	std::string GetDaemonName(std::string_view ServiceName) { return fmt::format("com.epicgames.unreal.{}", ServiceName); }
+
+	std::filesystem::path GetPListPath(const std::string& DaemonName)
+	{
+		const std::filesystem::path PListFolder = "/Library/LaunchDaemons";
+		return PListFolder / (DaemonName + ".plist");
+	}
+
+	std::string BuildPlist(std::string_view				ServiceName,
+						   const std::filesystem::path& ExecutablePath,
+						   std::string_view				CommandLineOptions,
+						   std::string_view				DaemonName,
+						   bool							Debug)
+	{
+		std::vector<std::string_view> Arguments = SplitArguments(CommandLineOptions);
+		ExtendableStringBuilder<256>  ProgramArguments;
+		for (const std::string_view Argument : Arguments)
+		{
+			ProgramArguments.Append("    <string>");
+			AppendEscaped(Argument, ProgramArguments);
+			ProgramArguments.Append("</string>\n");
+		}
+
+		return fmt::format(
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+			"<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n"
+			"<plist version=\"1.0\">\n"
+			"<dict>\n"
+			"  <key>Label</key>\n"
+			"  <string>{}</string>\n"  // DaemonName
+			"  \n"
+			"  <key>ProgramArguments</key>\n"
+			"  <array>\n"
+			"    <string>{}</string>\n"	 // Program name
+			"{}"						 // "<string>arg</string>\n" * number of arguments
+			"  </array>\n"
+			"  \n"
+			"  <key>RunAtLoad</key>\n"
+			"  <true/>\n"
+			"  \n"
+			//			"  <key>KeepAlive</key>\n"
+			//			"  <true/>\n"
+			//			"  \n"
+			"  <key>StandardOutPath</key>\n"
+			"  <string>/var/log/{}.log</string>\n"
+			"  \n"
+			"  <key>StandardErrorPath</key>\n"
+			"  <string>/var/log/{}.err.log</string>\n"
+			"  \n"
+			"  <key>Debug</key>\n"
+			"  <{}/>\n"
+			"  \n"
+			"</dict>\n"
+			"</plist>\n",
+			DaemonName,
+			ExecutablePath,
+			ProgramArguments.ToView(),
+			ServiceName,
+			ServiceName,
+			Debug ? "true"sv : "false"sv);
+
+		//            "<key>Sockets</key>"
+		//            "<dict>"
+		//                "<key>Listeners</key>"
+		//                "<dict>"
+		//                    "<key>SockServiceName</key>"
+		//                    "<string>{}</string>"       // Listen socket
+		//                    "<key>SockType</key>"
+		//                    "<string>tcp</string>"
+		//                    "<key>SockFamily</key>"
+		//                    "<string>IPv4</string>"
+		//                "</dict>"
+		//            "</dict>"
+	}
+
+#endif	// ZEN_PLATFORM_MAC
+
+#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
+
+	std::pair<int, std::string> ExecuteProgram(std::string_view Cmd)
+	{
+		std::string Data;
+		const int	BufferSize = 256;
+		char		Buffer[BufferSize];
+		std::string Command(Cmd);
+		Command.append(" 2>&1");
+
+		ZEN_DEBUG("Running: '{}'", Command);
+
+		FILE* Stream = popen(Command.c_str(), "r");
+		if (Stream)
+		{
+			while (!feof(Stream))
+			{
+				if (fgets(Buffer, BufferSize, Stream) != NULL)
+				{
+					Data.append(Buffer);
+				}
+			}
+
+			while (!Data.empty() && isspace(Data[Data.length() - 1]))
+			{
+				Data.pop_back();
+			}
+
+			int Res	   = -1;
+			int Status = pclose(Stream);
+			if (Status < 0)
+			{
+				ZEN_DEBUG("Command {} returned {}, errno {}", Command, Status, errno);
+				return {Status, Data};
+			}
+			uint64_t WaitMS = 100;
+			if (WIFEXITED(Status))
+			{
+				Res = WEXITSTATUS(Status);
+			}
+			if (Res != 0 && Res != (128 + 13))
+			{
+				return {Res, Data};
+			}
+			return {0, Data};
+		}
+		return {errno, ""};
+	}
+
+#endif	// ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
+
+#if ZEN_PLATFORM_LINUX
+	std::string GetUnitName(std::string_view ServiceName) { return fmt::format("com.epicgames.unreal.{}", ServiceName); }
+
+	std::filesystem::path GetServiceUnitPath(const std::string& UnitName)
+	{
+		const std::filesystem::path SystemUnitFolder = "/etc/systemd/system/";
+		return SystemUnitFolder / (UnitName + ".service");
+	}
+
+	std::string BuildUnitFile(std::string_view			   ServiceName,
+							  const std::filesystem::path& ExecutablePath,
+							  std::string_view			   CommandLineOptions,
+							  std::string_view			   AliasName,
+							  std::string_view			   UserName)
+	{
+		return fmt::format(
+			"[Unit]\n"
+			"Description={}\n"
+			"Documentation=https://github.com/epicgames/zen\n"
+			"\n"
+			"DefaultDependencies=no\n"
+			"After=network.target\n"
+			"StartLimitIntervalSec=0\n"
+			"\n"
+			"[Service]\n"
+			"Type=notify\n"
+			"Restart=always\n"
+			"RestartSec=1\n"
+			"User={}\n"
+			"ExecStart={} {}\n"
+			"RuntimeDirectory={}\n"
+			"[Install]\n"
+			"Alias={}\n"
+			"WantedBy=multi-user.target",
+			ServiceName,
+			UserName,
+			ExecutablePath,
+			CommandLineOptions,
+			ExecutablePath.parent_path(),
+			AliasName);
+	}
+
+#endif	// ZEN_PLATFORM_LINUX
+}  // namespace
+
+std::string_view
+ToString(ServiceStatus Status)
+{
+	switch (Status)
+	{
+		case ServiceStatus::NotInstalled:
+			return "Not installed"sv;
+		case ServiceStatus::Starting:
+			return "Starting"sv;
+		case ServiceStatus::Running:
+			return "Running"sv;
+		case ServiceStatus::Stopping:
+			return "Stopping"sv;
+		case ServiceStatus::Stopped:
+			return "Stopped"sv;
+		case ServiceStatus::Pausing:
+			return "Pausing"sv;
+		case ServiceStatus::Paused:
+			return "Paused"sv;
+		case ServiceStatus::Resuming:
+			return "Resuming"sv;
+		default:
+			ZEN_ASSERT(false);
+			return ""sv;
+	}
+}
+
+#if ZEN_PLATFORM_WINDOWS
+
+std::error_code
+InstallService(std::string_view ServiceName, const ServiceSpec& Spec)
+{
+	// TODO: Is "LocalService account" the correct account?
+	// TODO: Is ther eother parameters/security settings that we need to set up properly?
+
+	// Get a handle to the SCM database.
+	SC_HANDLE schSCManager = OpenSCManager(NULL,					// local computer
+										   NULL,					// ServicesActive database
+										   SC_MANAGER_ALL_ACCESS);	// full access rights
+
+	if (NULL == schSCManager)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	auto _ = MakeGuard([schSCManager]() { CloseServiceHandle(schSCManager); });
+
+	// Create the service
+
+	ExtendableWideStringBuilder<128> Name;
+	Utf8ToWide(ServiceName, Name);
+
+	ExtendableWideStringBuilder<128> DisplayName;
+	Utf8ToWide(Spec.DisplayName, DisplayName);
+
+	ExtendableWideStringBuilder<128> Path;
+	Path.Append(Spec.ExecutablePath.c_str());
+	if (!Spec.CommandLineOptions.empty())
+	{
+		Path.AppendAscii(" ");
+		Utf8ToWide(Spec.CommandLineOptions, Path);
+	}
+
+	SC_HANDLE schService = CreateService(schSCManager,						  // SCM database
+										 Name.c_str(),						  // name of service
+										 DisplayName.c_str(),				  // service name to display
+										 SERVICE_ALL_ACCESS,				  // desired access
+										 SERVICE_WIN32_OWN_PROCESS,			  // service type
+										 SERVICE_AUTO_START,				  // start type
+										 SERVICE_ERROR_NORMAL,				  // error control type
+										 Path.c_str(),						  // path to service's binary
+										 NULL,								  // no load ordering group
+										 NULL,								  // no tag identifier
+										 NULL,								  // no dependencies
+										 TEXT("NT AUTHORITY\\LocalService"),  // LocalService account
+										 NULL);								  // no password
+
+	if (schService == NULL)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	auto __ = MakeGuard([schService]() { CloseServiceHandle(schService); });
+
+	if (!Spec.Description.empty())
+	{
+		ExtendableWideStringBuilder<128> DescriptionBuilder;
+		Utf8ToWide(Spec.Description, DescriptionBuilder);
+
+		SERVICE_DESCRIPTION Description;
+		Description.lpDescription = const_cast<wchar_t*>(DescriptionBuilder.c_str());
+		if (!ChangeServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, &Description))
+		{
+			return MakeErrorCodeFromLastError();
+		}
+	}
+
+	// Actions defining what the service manager should do in the event of a zenserver crash.
+	// Attempt an immediate restart twice. If both restarts fail, stop trying.
+	// The attempt count will be reset based on the timeout specified in SERVICE_FAILURE_ACTIONS.
+	// If the service manages to survive for the length of that timeout, the reset attempt count
+	// will be reset and we will try restarting if we fail again.
+	SC_ACTION Actions[] = {{SC_ACTION_RESTART, 0}, {SC_ACTION_RESTART, 0}, {SC_ACTION_NONE, 0}};
+
+	SERVICE_FAILURE_ACTIONS FailureAction = {
+		60,	   // if we haven't failed for one minute, assume the service is healthy and reset the failure count
+		NULL,  // no reboot message - we don't want to reboot the whole system if zen dies
+		NULL,  // no command to run on failure - just attempt restarting the service
+		ZEN_ARRAY_COUNT(Actions),
+		Actions};
+
+	if (!ChangeServiceConfig2(schService, SERVICE_CONFIG_FAILURE_ACTIONS, &FailureAction))
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	return {};
+}
+
+std::error_code
+UninstallService(std::string_view ServiceName)
+{
+	// Get a handle to the SCM database.
+	SC_HANDLE schSCManager = OpenSCManager(NULL,					// local computer
+										   NULL,					// ServicesActive database
+										   SC_MANAGER_ALL_ACCESS);	// full access rights
+
+	if (NULL == schSCManager)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	auto _ = MakeGuard([schSCManager]() { CloseServiceHandle(schSCManager); });
+
+	// Get a handle to the service.
+
+	ExtendableWideStringBuilder<128> Name;
+	Utf8ToWide(ServiceName, Name);
+
+	SC_HANDLE schService = OpenService(schSCManager,  // SCM database
+									   Name.c_str(),  // name of service
+									   DELETE);		  // need delete access
+
+	if (schService == NULL)
+	{
+		DWORD Error = ::GetLastError();
+		if (Error == ERROR_SERVICE_DOES_NOT_EXIST)
+		{
+			return {};
+		}
+		return MakeErrorCode(Error);
+	}
+	auto __ = MakeGuard([schService]() { CloseServiceHandle(schService); });
+
+	// Delete the service.
+
+	if (!DeleteService(schService))
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	return {};
+}
+
+std::error_code
+QueryInstalledService(std::string_view ServiceName, ServiceInfo& OutInfo)
+{
+	// Get a handle to the SCM database.
+	SC_HANDLE schSCManager = OpenSCManager(NULL,				 // local computer
+										   NULL,				 // ServicesActive database
+										   SC_MANAGER_CONNECT);	 // standard access rights
+
+	if (NULL == schSCManager)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	auto _ = MakeGuard([schSCManager]() { CloseServiceHandle(schSCManager); });
+
+	// Get a handle to the service.
+
+	ExtendableWideStringBuilder<128> Name;
+	Utf8ToWide(ServiceName, Name);
+
+	SC_HANDLE schService = OpenService(schSCManager,								  // SCM database
+									   Name.c_str(),								  // name of service
+									   SERVICE_QUERY_STATUS | SERVICE_QUERY_CONFIG);  // need delete access
+
+	if (schService == NULL)
+	{
+		DWORD Error = ::GetLastError();
+		if (Error == ERROR_SERVICE_DOES_NOT_EXIST)
+		{
+			OutInfo.Status = ServiceStatus::NotInstalled;
+			return {};
+		}
+		return MakeErrorCode(Error);
+	}
+	auto __ = MakeGuard([schService]() { CloseServiceHandle(schService); });
+
+	std::vector<std::uint8_t> Buffer(8192);
+	QUERY_SERVICE_CONFIG*	  ServiceConfig = reinterpret_cast<QUERY_SERVICE_CONFIG*>(Buffer.data());
+	DWORD					  BytesNeeded	= 0;
+	if (!QueryServiceConfig(schService, ServiceConfig, (DWORD)Buffer.size(), &BytesNeeded))
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	std::wstring BinaryWithArguments(ServiceConfig->lpBinaryPathName);
+	(void)SplitExecutableAndArgs(BinaryWithArguments, OutInfo.Spec.ExecutablePath, OutInfo.Spec.CommandLineOptions);
+	OutInfo.Spec.DisplayName = WideToUtf8(ServiceConfig->lpDisplayName);
+
+	SERVICE_STATUS ServiceStatus;
+	if (!::QueryServiceStatus(schService, &ServiceStatus))
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	switch (ServiceStatus.dwCurrentState)
+	{
+		case SERVICE_STOPPED:
+			OutInfo.Status = ServiceStatus::Stopped;
+			break;
+		case SERVICE_START_PENDING:
+			OutInfo.Status = ServiceStatus::Starting;
+			break;
+		case SERVICE_STOP_PENDING:
+			OutInfo.Status = ServiceStatus::Stopping;
+			break;
+		case SERVICE_RUNNING:
+			OutInfo.Status = ServiceStatus::Running;
+			break;
+		case SERVICE_CONTINUE_PENDING:
+			OutInfo.Status = ServiceStatus::Resuming;
+			break;
+		case SERVICE_PAUSE_PENDING:
+			OutInfo.Status = ServiceStatus::Pausing;
+			break;
+		case SERVICE_PAUSED:
+			OutInfo.Status = ServiceStatus::Paused;
+			break;
+		default:
+			throw std::runtime_error(fmt::format("Unknown service status for '{}': {}", ServiceName, ServiceStatus.dwCurrentState));
+	}
+
+	if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, Buffer.data(), (DWORD)Buffer.size(), &BytesNeeded))
+	{
+		DWORD Error = ::GetLastError();
+		if (Error == ERROR_INSUFFICIENT_BUFFER)
+		{
+			Buffer.resize((size_t)BytesNeeded);
+			if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, Buffer.data(), (DWORD)Buffer.size(), &BytesNeeded))
+			{
+				return MakeErrorCodeFromLastError();
+			}
+		}
+		else
+		{
+			return MakeErrorCode(Error);
+		}
+	}
+	SERVICE_DESCRIPTION* Description = (SERVICE_DESCRIPTION*)Buffer.data();
+	if (Description->lpDescription != NULL)
+	{
+		OutInfo.Spec.Description = WideToUtf8(std::wstring(Description->lpDescription));
+	}
+
+	return {};
+}
+
+std::error_code
+StartService(std::string_view ServiceName)
+{
+	// Get a handle to the SCM database.
+	SC_HANDLE schSCManager = OpenSCManager(NULL,				 // local computer
+										   NULL,				 // ServicesActive database
+										   SC_MANAGER_CONNECT);	 // default access rights
+
+	if (NULL == schSCManager)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	auto _ = MakeGuard([schSCManager]() { CloseServiceHandle(schSCManager); });
+
+	// Get a handle to the service.
+
+	ExtendableWideStringBuilder<128> Name;
+	Utf8ToWide(ServiceName, Name);
+
+	SC_HANDLE schService = OpenService(schSCManager,	// SCM database
+									   Name.c_str(),	// name of service
+									   SERVICE_START);	// need start access
+
+	if (schService == NULL)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+	auto __ = MakeGuard([schService]() { CloseServiceHandle(schService); });
+
+	// Start the service.
+
+	if (!::StartService(schService, 0, NULL))
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	return {};
+}
+
+std::error_code
+StopService(std::string_view ServiceName)
+{
+	// Get a handle to the SCM database.
+	SC_HANDLE schSCManager = OpenSCManager(NULL,				 // local computer
+										   NULL,				 // ServicesActive database
+										   SC_MANAGER_CONNECT);	 // default access rights
+
+	if (NULL == schSCManager)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	auto _ = MakeGuard([schSCManager]() { CloseServiceHandle(schSCManager); });
+
+	// Get a handle to the service.
+
+	ExtendableWideStringBuilder<128> Name;
+	Utf8ToWide(ServiceName, Name);
+
+	SC_HANDLE schService = OpenService(schSCManager,   // SCM database
+									   Name.c_str(),   // name of service
+									   SERVICE_STOP);  // need start access
+
+	if (schService == NULL)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+	auto __ = MakeGuard([schService]() { CloseServiceHandle(schService); });
+
+	// Stop the service.
+	SERVICE_STATUS ServiceStatus;
+	if (!::ControlService(schService, SERVICE_CONTROL_STOP, &ServiceStatus))
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	return {};
+}
+
+#endif
+
+#if ZEN_PLATFORM_MAC
+
+std::error_code
+InstallService(std::string_view ServiceName, const ServiceSpec& Spec)
+{
+	// TODO: Do we need to create a separate user for the service or is running as the default service user OK?
+	const std::string DaemonName = GetDaemonName(ServiceName);
+	std::string		  PList		 = BuildPlist(ServiceName, Spec.ExecutablePath, Spec.CommandLineOptions, DaemonName, true);
+
+	const std::filesystem::path PListPath = GetPListPath(DaemonName);
+	ZEN_INFO("Writing launchd plist to {}", PListPath.string());
+	try
+	{
+		zen::WriteFile(PListPath, IoBuffer(IoBuffer::Wrap, PList.data(), PList.size()));
+	}
+	catch (const std::system_error& Ex)
+	{
+		return MakeErrorCode(Ex.code().value());
+	}
+
+	ZEN_INFO("Changing permissions to 644 for {}", PListPath.string());
+	if (chmod(PListPath.string().c_str(), 0644) == -1)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+	return {};
+}
+
+std::error_code
+UninstallService(std::string_view ServiceName)
+{
+	const std::string			DaemonName = GetDaemonName(ServiceName);
+	const std::filesystem::path PListPath  = GetPListPath(DaemonName);
+	ZEN_INFO("Attempting to remove launchd plist from {}", PListPath.string());
+	std::error_code Ec;
+	std::filesystem::remove(PListPath, Ec);
+	return Ec;
+}
+
+std::error_code
+QueryInstalledService(std::string_view ServiceName, ServiceInfo& OutInfo)
+{
+	OutInfo.Status				 = ServiceStatus::NotInstalled;
+	const std::string DaemonName = GetDaemonName(ServiceName);
+
+	const std::filesystem::path PListPath = GetPListPath(DaemonName);
+	if (std::filesystem::is_regular_file(PListPath))
+	{
+		OutInfo.Status = ServiceStatus::Stopped;
+
+		{
+			// Parse plist :(
+			IoBuffer	Buffer = ReadFile(PListPath).Flatten();
+			MemoryView	Data   = Buffer.GetView();
+			std::string PList((const char*)Data.GetData(), Data.GetSize());
+
+			enum class ParseMode
+			{
+				None,
+				ExpectingProgramArgumentsArray,
+				ExpectingProgramExecutablePath,
+				ExpectingCommandLineOption
+			};
+
+			ParseMode Mode = ParseMode::None;
+
+			ForEachStrTok(PList, '\n', [&](std::string_view Line) {
+				switch (Mode)
+				{
+					case ParseMode::None:
+						{
+							if (Line.find("<key>ProgramArguments</key>") != std::string_view::npos)
+							{
+								Mode = ParseMode::ExpectingProgramArgumentsArray;
+								return true;
+							}
+						}
+						break;
+					case ParseMode::ExpectingProgramArgumentsArray:
+						{
+							if (Line.find("<array>") != std::string_view::npos)
+							{
+								Mode = ParseMode::ExpectingProgramExecutablePath;
+								return true;
+							}
+							Mode = ParseMode::None;
+						}
+						break;
+					case ParseMode::ExpectingProgramExecutablePath:
+						{
+							if (std::string_view::size_type ArgStart = Line.find("<string>"); ArgStart != std::string_view::npos)
+							{
+								ArgStart += 8;
+								if (std::string_view::size_type ArgEnd = Line.find("</string>", ArgStart); ArgEnd != std::string_view::npos)
+								{
+									std::string_view ProgramString = Line.substr(ArgStart, ArgEnd - ArgStart);
+									OutInfo.Spec.ExecutablePath	   = ProgramString;
+									Mode						   = ParseMode::ExpectingCommandLineOption;
+									return true;
+								}
+							}
+							Mode = ParseMode::None;
+						}
+						break;
+					case ParseMode::ExpectingCommandLineOption:
+						{
+							if (std::string_view::size_type ArgStart = Line.find("</array>"); ArgStart != std::string_view::npos)
+							{
+								Mode = ParseMode::None;
+								return true;
+							}
+							else if (std::string_view::size_type ArgStart = Line.find("<string>"); ArgStart != std::string_view::npos)
+							{
+								ArgStart += 8;
+								if (std::string_view::size_type ArgEnd = Line.find("</string>", ArgStart); ArgEnd != std::string_view::npos)
+								{
+									std::string_view ArgumentString = Line.substr(ArgStart, ArgEnd - ArgStart);
+									if (!OutInfo.Spec.CommandLineOptions.empty())
+									{
+										OutInfo.Spec.CommandLineOptions += " ";
+									}
+									OutInfo.Spec.CommandLineOptions += ArgumentString;
+									return true;
+								}
+							}
+							Mode = ParseMode::None;
+						}
+						break;
+				}
+				return true;
+			});
+		}
+		{
+			std::pair<int, std::string> Res = ExecuteProgram(fmt::format("launchctl list {}", DaemonName));
+			if (Res.first == 0 && !Res.second.empty())
+			{
+				ForEachStrTok(Res.second, '\n', [&](std::string_view Line) {
+					if (Line.find("\"PID\"") != std::string_view::npos)
+					{
+						std::string_view::size_type PidStart  = Line.find('=');
+						std::string_view::size_type PidEnd	  = Line.find(';');
+						std::string_view			PidString = Line.substr(PidStart + 2, PidEnd - (PidStart + 2));
+						if (ParseInt<int>(PidString).has_value())
+						{
+							OutInfo.Status = ServiceStatus::Running;
+						}
+						return false;
+					}
+					return true;
+				});
+				// Parse installed info
+			}
+		}
+	}
+
+	return {};
+}
+
+std::error_code
+StartService(std::string_view ServiceName)
+{
+	const std::string			DaemonName = GetDaemonName(ServiceName);
+	const std::filesystem::path PListPath  = GetPListPath(DaemonName);
+
+	std::pair<int, std::string> Res = ExecuteProgram(fmt::format("launchctl bootstrap system {}", PListPath));
+	if (Res.first != 0)
+	{
+		return MakeErrorCode(Res.first);
+	}
+
+	return {};
+}
+
+std::error_code
+StopService(std::string_view ServiceName)
+{
+	const std::string			DaemonName = GetDaemonName(ServiceName);
+	const std::filesystem::path PListPath  = GetPListPath(DaemonName);
+
+	std::pair<int, std::string> Res = ExecuteProgram(fmt::format("launchctl bootout system ", PListPath.));
+	if (Res.first != 0)
+	{
+		return MakeErrorCode(Res.first);
+	}
+
+	return {};
+}
+
+#endif	// ZEN_PLATFORM_MAC
+
+#if ZEN_PLATFORM_LINUX
+
+std::error_code
+InstallService(std::string_view ServiceName, const ServiceSpec& Spec)
+{
+	const std::string			UnitName		= GetUnitName(ServiceName);
+	const std::filesystem::path ServiceUnitPath = GetServiceUnitPath(UnitName);
+	std::string					UserName		= Spec.UserName;
+
+	if (UserName == "")
+	{
+		std::pair<int, std::string> UserResult = ExecuteProgram("echo $SUDO_USER");
+		if (UserResult.first != 0 || UserResult.second.empty())
+		{
+			ZEN_ERROR("Unable to determine current user");
+			return MakeErrorCode(UserResult.first);
+		}
+
+		UserName = UserResult.second;
+	}
+
+	std::string UnitFile = BuildUnitFile(ServiceName, Spec.ExecutablePath, Spec.CommandLineOptions, UnitName, UserName);
+	ZEN_DEBUG("Writing systemd unit file to {}", ServiceUnitPath.string());
+	try
+	{
+		zen::WriteFile(ServiceUnitPath, IoBuffer(IoBuffer::Wrap, UnitFile.data(), UnitFile.size()));
+	}
+	catch (const std::system_error& Ex)
+	{
+		return MakeErrorCode(Ex.code().value());
+	}
+
+	ZEN_DEBUG("Changing permissions to 644 for {}", ServiceUnitPath.string());
+	if (chmod(ServiceUnitPath.string().c_str(), 0644) == -1)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+
+	std::pair<int, std::string> Res = ExecuteProgram("systemctl daemon-reload");
+	if (Res.first != 0 && Res.first != -1)
+	{
+		ZEN_ERROR("systemctl daemon-reload failed with {}: '{}'", Res.first, Res.second);
+		return MakeErrorCode(Res.first);
+	}
+
+	Res = ExecuteProgram(fmt::format("systemctl enable {}", UnitName));
+	if (Res.first != 0 && Res.first != -1)
+	{
+		ZEN_ERROR("systemctl enable failed with {}: '{}'", Res.first, Res.second);
+		return MakeErrorCode(Res.first);
+	}
+
+	return {};
+}
+
+std::error_code
+UninstallService(std::string_view ServiceName)
+{
+	const std::string			UnitName		= GetUnitName(ServiceName);
+	const std::filesystem::path ServiceUnitPath = GetServiceUnitPath(UnitName);
+
+	std::pair<int, std::string> Res = ExecuteProgram(fmt::format("systemctl disable {}", UnitName));
+	if (Res.first != 0 && Res.first != -1)
+	{
+		ZEN_ERROR("systemctl disable failed with {}: '{}'", Res.first, Res.second);
+		return MakeErrorCode(Res.first);
+	}
+
+	ZEN_DEBUG("Attempting to remove systemd unit file from {}", ServiceUnitPath.string());
+	std::error_code Ec;
+	std::filesystem::remove(ServiceUnitPath, Ec);
+	if (Ec)
+	{
+		ZEN_ERROR("failed to remove {}: '{}'", ServiceUnitPath, Ec.message());
+		return Ec;
+	}
+
+	Res = ExecuteProgram("systemctl daemon-reload");
+	if (Res.first != 0 && Res.first != -1)
+	{
+		ZEN_ERROR("systemctl daemon-reload failed with {}: '{}'", Res.first, Res.second);
+		return MakeErrorCode(Res.first);
+	}
+
+	return {};
+}
+
+std::error_code
+QueryInstalledService(std::string_view ServiceName, ServiceInfo& OutInfo)
+{
+	const std::string			UnitName		= GetUnitName(ServiceName);
+	const std::filesystem::path ServiceUnitPath = GetServiceUnitPath(UnitName);
+
+	OutInfo.Status = ServiceStatus::NotInstalled;
+
+	if (std::filesystem::is_regular_file(ServiceUnitPath))
+	{
+		OutInfo.Status = ServiceStatus::Stopped;
+
+		std::pair<int, std::string> Res = ExecuteProgram(fmt::format("systemctl is-active --quiet {}", UnitName));
+		if (Res.first == 0)
+		{
+			OutInfo.Status = ServiceStatus::Running;
+
+			std::pair<int, std::string> ShowResult = ExecuteProgram(fmt::format("systemctl show -p ExecStart {}", UnitName));
+			if (ShowResult.first == 0)
+			{
+				std::regex	Regex(R"~(ExecStart=\{ path=(.*?) ; argv\[\]=(.*?) ;)~");
+				std::smatch Match;
+
+				if (std::regex_search(ShowResult.second, Match, Regex))
+				{
+					std::string Executable			= Match[1].str();
+					std::string CommandLine			= Match[2].str();
+					OutInfo.Spec.ExecutablePath		= Executable;
+					OutInfo.Spec.CommandLineOptions = CommandLine.substr(Executable.size(), CommandLine.size());
+				}
+				else
+				{
+					ZEN_WARN("Failed to parse output of systemctl show: {}", ShowResult.second);
+				}
+			}
+			else
+			{
+				ZEN_WARN("Failed to read start info from systemctl: error code {}", ShowResult.first);
+			}
+		}
+		else
+		{
+			ZEN_DEBUG("systemctl status failed with '{}'({})", Res.second, Res.first);
+		}
+	}
+
+	return {};
+}
+
+std::error_code
+StartService(std::string_view ServiceName)
+{
+	// TODO: Starting the service returns -1 from ExecuteProgram, so the service won't start
+	// TODO: Running start from command line gives no output but the service does not start - not sure what is wrong.
+	// Starting the same command line for the service using `sudo` *will* start the service sucessfully so I
+	// assume that the Unit file or some config is wrong/missing.
+
+	const std::string			UnitName		= GetUnitName(ServiceName);
+	const std::filesystem::path ServiceUnitPath = GetServiceUnitPath(UnitName);
+
+	std::pair<int, std::string> Res = ExecuteProgram(fmt::format("systemctl start {}", UnitName));
+	if (Res.first != 0)
+	{
+		ZEN_ERROR("service start failed with {}: '{}'", Res.first, Res.second);
+		return MakeErrorCode(Res.first);
+	}
+
+	return {};
+}
+
+std::error_code
+StopService(std::string_view ServiceName)
+{
+	// TODO: Stopping the service returns -1 from ExecuteProgram, maybe this ie expected as I have yet to successfully start the service
+	// using systemctl start
+
+	const std::string			UnitName		= GetUnitName(ServiceName);
+	const std::filesystem::path ServiceUnitPath = GetServiceUnitPath(UnitName);
+
+	std::pair<int, std::string> Res = ExecuteProgram(fmt::format("systemctl stop {}", UnitName));
+	if (Res.first != 0)
+	{
+		ZEN_ERROR("service stop failed with {}: '{}'", Res.first, Res.second);
+		return MakeErrorCode(Res.first);
+	}
+
+	return {};
+}
+
+#endif	// ZEN_PLATFORM_LINUX
+
+}  // namespace zen
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index b36f11741..0409cb976 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -167,10 +167,23 @@ ZenServerState::Initialize()
 		ThrowLastError("Could not map view of Zen server state");
 	}
 #else
-	int Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, 0666);
+	ZEN_INFO("{}", S_IRUSR | S_IWUSR | S_IXUSR);
+	ZEN_INFO("{}", geteuid());
+	int Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, geteuid() == 0 ? 0766 : 0666);
 	if (Fd < 0)
 	{
-		ThrowLastError("Could not open a shared memory object");
+		// Work around a potential issue if the service user is changed in certain configurations.
+		// If the sysctl 'fs.protected_regular' is set to 1 or 2 (default on many distros),
+		// we will be unable to open an existing shared memory object created by another user using O_CREAT,
+		// even if we have the correct permissions, or are running as root. If we destroy the existing
+		// shared memory object and retry, we'll be able to get past shm_open() so long as we have
+		// the appropriate permissions to create the shared memory object.
+		shm_unlink("/UnrealEngineZen");
+		Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, geteuid() == 0 ? 0766 : 0666);
+		if (Fd < 0)
+		{
+			ThrowLastError("Could not open a shared memory object");
+		}
 	}
 	fchmod(Fd, 0666);
 	void* hMap = (void*)intptr_t(Fd);