Add trace support for zen CLI command (#296)

- This change adds support for `--trace`, `--tracehost` and `--tracefile` command arguments to enable and control tracing to Insights
- It also adds profiling scopes primarily to build download command related code

4 files changed, 210 insertions, 37 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index fb9da021d..eb0650c4d 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -12,6 +12,7 @@
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/string.h>
+#include <zencore/trace.h>
 #include <zencore/uid.h>
 #include <zenhttp/formatters.h>
 #include <zenhttp/httpclient.h>
@@ -180,6 +181,8 @@ namespace {
 
 	void CleanDirectory(const std::filesystem::path& Path, std::span<const std::string_view> ExcludeDirectories)
 	{
+		ZEN_TRACE_CPU("CleanDirectory");
+
 		DirectoryContent LocalDirectoryContent;
 		GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent);
 		for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files)
@@ -352,6 +355,8 @@ namespace {
 		std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
 		ChunkingController&																		 ChunkController)
 	{
+		ZEN_TRACE_CPU("ScanAndChunkFolder");
+
 		FolderContent Content = GetFolderContent(
 			GetFolderContentStats,
 			Path,
@@ -506,6 +511,8 @@ namespace {
 													   const std::span<const uint32_t>&						   LooseChunkIndexes,
 													   const std::span<const ChunkBlockDescription>&		   BlockDescriptions)
 	{
+		ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders");
+
 #if EXTRA_VERIFY
 		std::vector<IoHash> TmpAbsoluteChunkHashes;
 		TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size());
@@ -572,6 +579,8 @@ namespace {
 								   std::vector<uint64_t>&						 OutLocalChunkRawSizes,
 								   std::vector<uint32_t>&						 OutLocalChunkOrders)
 	{
+		ZEN_TRACE_CPU("CalculateLocalChunkOrders");
+
 		std::vector<IoHash>	  AbsoluteChunkHashes;
 		std::vector<uint64_t> AbsoluteChunkRawSizes;
 		AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), LooseChunkHashes.begin(), LooseChunkHashes.end());
@@ -955,6 +964,8 @@ namespace {
 		const uint64_t	BlockSize = 256u * 1024u;
 		CompositeBuffer GetRange(uint64_t Offset, uint64_t Size)
 		{
+			ZEN_TRACE_CPU("BufferedOpenFile::GetRange");
+
 			ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache);
 			auto _ = MakeGuard([&]() { ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); });
 
@@ -1022,6 +1033,8 @@ namespace {
 
 		CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size)
 		{
+			ZEN_TRACE_CPU("ReadFileCache::GetRange");
+
 			auto CacheIt = std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) {
 				return Lhs.first == SequenceIndex;
 			});
@@ -1069,6 +1082,8 @@ namespace {
 
 	CompositeBuffer ValidateBlob(IoBuffer&& Payload, const IoHash& BlobHash, uint64_t& OutCompressedSize, uint64_t& OutDecompressedSize)
 	{
+		ZEN_TRACE_CPU("ValidateBlob");
+
 		if (Payload.GetContentType() != ZenContentType::kCompressedBinary)
 		{
 			throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'",
@@ -3087,6 +3102,8 @@ namespace {
 
 	void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path, bool VerifyFileHash)
 	{
+		ZEN_TRACE_CPU("VerifyFolder");
+
 		ProgressBar			  ProgressBar(UsePlainProgress);
 		std::atomic<uint64_t> FilesVerified(0);
 		std::atomic<uint64_t> FilesFailed(0);
@@ -3133,6 +3150,8 @@ namespace {
 				[&, PathIndex](std::atomic<bool>&) {
 					if (!AbortFlag)
 					{
+						ZEN_TRACE_CPU("VerifyFile_work");
+
 						// TODO: Convert ScheduleWork body to function
 
 						const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred();
@@ -3348,6 +3367,8 @@ namespace {
 						  std::atomic<uint32_t>&		   OutChunksComplete,
 						  std::atomic<uint64_t>&		   OutBytesWritten)
 	{
+		ZEN_TRACE_CPU("WriteBlockToDisk");
+
 		std::vector<CompositeBuffer> ChunkBuffers;
 		struct WriteOpData
 		{
@@ -3438,6 +3459,8 @@ namespace {
 					}
 					if (!AbortFlag)
 					{
+						ZEN_TRACE_CPU("WriteBlockToDisk_VerifyHash");
+
 						// Write tracking, updating this must be done without any files open (WriteFileCache)
 						for (const WriteOpData& WriteOp : WriteOps)
 						{
@@ -3449,7 +3472,7 @@ namespace {
 									 IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)));
 								if (VerifyChunkHash != SequenceRawHash)
 								{
-									throw std::runtime_error(fmt::format("Written hunk sequence {} hash does not match expected hash {}",
+									throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}",
 																		 VerifyChunkHash,
 																		 SequenceRawHash));
 								}
@@ -3468,6 +3491,8 @@ namespace {
 
 	SharedBuffer Decompress(const CompositeBuffer& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize)
 	{
+		ZEN_TRACE_CPU("Decompress");
+
 		IoHash			 RawHash;
 		uint64_t		 RawSize;
 		CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedChunk, RawHash, RawSize);
@@ -3506,6 +3531,8 @@ namespace {
 						  WriteFileCache&													  OpenFileCache,
 						  std::atomic<uint64_t>&											  OutBytesWritten)
 	{
+		ZEN_TRACE_CPU("WriteChunkToDisk");
+
 		for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargets)
 		{
 			const auto&	   Target		 = *TargetPtr;
@@ -3611,6 +3638,8 @@ namespace {
 						   std::atomic<uint32_t>&												  ChunksComplete,
 						   std::atomic<uint64_t>&												  MultipartAttachmentCount)
 	{
+		ZEN_TRACE_CPU("DownloadLargeBlob");
+
 		struct WorkloadData
 		{
 			TemporaryFile TempFile;
@@ -3666,6 +3695,8 @@ namespace {
 							 &WriteToDiskBytes,
 							 SequenceIndexChunksLeftToWriteCounters,
 							 ChunkTargetPtrs](std::atomic<bool>&) {
+								ZEN_TRACE_CPU("DownloadLargeBlob_Work");
+
 								if (!AbortFlag)
 								{
 									uint64_t CompressedSize = Workload->TempFile.FileSize();
@@ -3729,6 +3760,8 @@ namespace {
 											const uint32_t RemoteSequenceIndex = Location->SequenceIndex;
 											if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1)
 											{
+												ZEN_TRACE_CPU("VerifyChunkHash");
+
 												const IoHash& SequenceRawHash =
 													RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
 												if (NeedHashVerify)
@@ -3743,6 +3776,8 @@ namespace {
 																		ChunkHash));
 													}
 												}
+
+												ZEN_TRACE_CPU("VerifyChunkHashes_rename");
 												std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash),
 																		GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash));
 											}
@@ -3784,6 +3819,8 @@ namespace {
 					  bool										WipeTargetFolder,
 					  FolderContent&							OutLocalFolderState)
 	{
+		ZEN_TRACE_CPU("UpdateFolder");
+
 		ZEN_UNUSED(WipeTargetFolder);
 		std::atomic<uint64_t> DownloadedBlocks		   = 0;
 		std::atomic<uint64_t> BlockBytes			   = 0;
@@ -3933,6 +3970,8 @@ namespace {
 		std::atomic<uint32_t> ChunkCountWritten = 0;
 
 		{
+			ZEN_TRACE_CPU("HandleChunks");
+
 			FilteredRate FilteredDownloadedBytesPerSecond;
 			FilteredRate FilteredWrittenBytesPerSecond;
 
@@ -3974,6 +4013,8 @@ namespace {
 						Work.ScheduleWork(
 							NetworkPool,  // GetSyncWorkerPool(),//
 							[&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic<bool>&) {
+								ZEN_TRACE_CPU("UpdateFolder_LooseChunk");
+
 								if (!AbortFlag)
 								{
 									FilteredDownloadedBytesPerSecond.Start();
@@ -4019,6 +4060,8 @@ namespace {
 												WritePool,
 												[&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs, CompressedPart = std::move(Payload)](
 													std::atomic<bool>&) {
+													ZEN_TRACE_CPU("UpdateFolder_WriteBlob");
+
 													if (!AbortFlag)
 													{
 														FilteredWrittenBytesPerSecond.Start();
@@ -4066,6 +4109,8 @@ namespace {
 																if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(
 																		1) == 1)
 																{
+																	ZEN_TRACE_CPU("UpdateFolder_VerifyHash");
+
 																	const IoHash& SequenceRawHash =
 																		RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
 																	if (NeedHashVerify)
@@ -4077,12 +4122,14 @@ namespace {
 																		if (VerifyChunkHash != SequenceRawHash)
 																		{
 																			throw std::runtime_error(
-																				fmt::format("Written hunk sequence {} hash does not match "
+																				fmt::format("Written chunk sequence {} hash does not match "
 																							"expected hash {}",
 																							VerifyChunkHash,
 																							SequenceRawHash));
 																		}
 																	}
+
+																	ZEN_TRACE_CPU("UpdateFolder_rename");
 																	std::filesystem::rename(
 																		GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash),
 																		GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash));
@@ -4113,6 +4160,8 @@ namespace {
 					[&, CopyDataIndex](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
+							ZEN_TRACE_CPU("UpdateFolder_Copy");
+
 							FilteredWrittenBytesPerSecond.Start();
 							const CacheCopyData& CopyData		= CacheCopyDatas[CopyDataIndex];
 							const uint32_t		 LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex];
@@ -4205,6 +4254,8 @@ namespace {
 										// Write tracking, updating this must be done without any files open (WriteFileCache)
 										for (const WriteOp& Op : WriteOps)
 										{
+											ZEN_TRACE_CPU("UpdateFolder_Copy_VerifyHash");
+
 											const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
 											if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1)
 											{
@@ -4215,10 +4266,12 @@ namespace {
 												if (VerifyChunkHash != SequenceRawHash)
 												{
 													throw std::runtime_error(
-														fmt::format("Written hunk sequence {} hash does not match expected hash {}",
+														fmt::format("Written chunk sequence {} hash does not match expected hash {}",
 																	VerifyChunkHash,
 																	SequenceRawHash));
 												}
+
+												ZEN_TRACE_CPU("UpdateFolder_Copy_rename");
 												std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash),
 																		GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash));
 											}
@@ -4268,6 +4321,8 @@ namespace {
 						[&, BlockIndex](std::atomic<bool>&) {
 							if (!AbortFlag)
 							{
+								ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Read");
+
 								FilteredDownloadedBytesPerSecond.Start();
 								IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash);
 								if (!BlockBuffer)
@@ -4288,6 +4343,8 @@ namespace {
 										[&, BlockIndex, BlockBuffer = std::move(Payload)](std::atomic<bool>&) {
 											if (!AbortFlag)
 											{
+												ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Write");
+
 												FilteredWrittenBytesPerSecond.Start();
 												IoHash			 BlockRawHash;
 												uint64_t		 BlockRawSize;
@@ -4348,6 +4405,8 @@ namespace {
 				}
 			}
 
+			ZEN_TRACE_CPU("HandleChunks_Wait");
+
 			Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
 				ZEN_UNUSED(IsAborted, PendingWork);
 				ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load());
@@ -4411,12 +4470,16 @@ namespace {
 
 		if (WipeTargetFolder)
 		{
+			ZEN_TRACE_CPU("UpdateFolder_WipeTarget");
+
 			// Clean target folder
 			ZEN_CONSOLE("Wiping {}", Path);
 			CleanDirectory(Path, DefaultExcludeFolders);
 		}
 		else
 		{
+			ZEN_TRACE_CPU("UpdateFolder_RemoveUnused");
+
 			// Remove unused tracked files
 			tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex;
 			RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size());
@@ -4468,6 +4531,8 @@ namespace {
 					break;
 				}
 
+				ZEN_TRACE_CPU("UpdateFolder_FinalizeTree");
+
 				size_t		  TargetCount = 1;
 				const IoHash& RawHash	  = Targets[TargetOffset].first;
 				while (Targets[TargetOffset + TargetCount].first == RawHash)
@@ -4480,6 +4545,8 @@ namespace {
 					[&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic<bool>&) {
 						if (!AbortFlag)
 						{
+							ZEN_TRACE_CPU("FinalizeTree_Work");
+
 							size_t						 TargetOffset		   = BaseTargetOffset;
 							const IoHash&				 RawHash			   = Targets[TargetOffset].first;
 							const uint32_t				 FirstTargetPathIndex  = Targets[TargetOffset].second;
@@ -4501,6 +4568,8 @@ namespace {
 							}
 							else
 							{
+								ZEN_TRACE_CPU("FinalizeTree_MoveIntoPlace");
+
 								const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
 								ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath));
 								CreateDirectories(FirstTargetFilePath.parent_path());
@@ -4522,6 +4591,8 @@ namespace {
 							TargetsComplete++;
 							while (TargetOffset < (BaseTargetOffset + TargetCount))
 							{
+								ZEN_TRACE_CPU("FinalizeTree_Copy");
+
 								ZEN_ASSERT(Targets[TargetOffset].first == RawHash);
 								ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath));
 								const uint32_t				 ExtraTargetPathIndex  = Targets[TargetOffset].second;
@@ -4555,6 +4626,8 @@ namespace {
 				TargetOffset += TargetCount;
 			}
 
+			ZEN_TRACE_CPU("FinalizeTree_Wait");
+
 			Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
 				ZEN_UNUSED(IsAborted, PendingWork);
 				std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size());
@@ -4668,6 +4741,8 @@ namespace {
 										  std::vector<ChunkBlockDescription>&			  OutBlockDescriptions,
 										  std::vector<IoHash>&							  OutLooseChunkHashes)
 	{
+		ZEN_TRACE_CPU("GetRemoteContent");
+
 		Stopwatch GetBuildPartTimer;
 		CbObject  BuildPartManifest = Storage.GetBuildPart(BuildId, BuildParts[0].first);
 		ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
@@ -5114,6 +5189,8 @@ namespace {
 						bool						 WipeTargetFolder,
 						bool						 PostDownloadVerify)
 	{
+		ZEN_TRACE_CPU("DownloadFolder");
+
 		Stopwatch DownloadTimer;
 
 		const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName;
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index bee1fd676..713a0d66d 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -31,9 +31,16 @@
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/string.h>
+#include <zencore/trace.h>
 #include <zenhttp/httpcommon.h>
 #include <zenutil/zenserverprocess.h>
 
+#include <zencore/memory/fmalloc.h>
+#include <zencore/memory/llm.h>
+#include <zencore/memory/memory.h>
+#include <zencore/memory/memorytrace.h>
+#include <zencore/memory/newdelete.h>
+
 #if ZEN_WITH_TESTS
 #	define ZEN_TEST_WITH_RUNNER 1
 #	include <zencore/testing.h>
@@ -603,6 +610,29 @@ main(int argc, char** argv)
 	Options.add_options()("help", "Show command line help");
 	Options.add_options()("c, command", "Sub command", cxxopts::value<std::string>(SubCommand));
 
+#if ZEN_WITH_TRACE
+	std::string TraceChannels;
+	std::string TraceHost;
+	std::string TraceFile;
+
+	Options.add_option("ue-trace",
+					   "",
+					   "trace",
+					   "Specify which trace channels should be enabled",
+					   cxxopts::value<std::string>(TraceChannels)->default_value(""),
+					   "");
+
+	Options.add_option("ue-trace",
+					   "",
+					   "tracehost",
+					   "Hostname to send the trace to",
+					   cxxopts::value<std::string>(TraceHost)->default_value(""),
+					   "");
+
+	Options
+		.add_option("ue-trace", "", "tracefile", "Path to write a trace to", cxxopts::value<std::string>(TraceFile)->default_value(""), "");
+#endif	// ZEN_WITH_TRACE
+
 	Options.parse_positional({"command"});
 
 	const bool IsNullInvoke = (argc == 1);	// If no arguments are passed we want to print usage information
@@ -653,6 +683,30 @@ main(int argc, char** argv)
 			logging::SetLogLevel(logging::level::Trace);
 		}
 
+#if ZEN_WITH_TRACE
+		if (TraceHost.size())
+		{
+			TraceStart("zen", TraceHost.c_str(), TraceType::Network);
+		}
+		else if (TraceFile.size())
+		{
+			TraceStart("zen", TraceFile.c_str(), TraceType::File);
+		}
+		else
+		{
+			TraceInit("zen");
+		}
+#endif	// ZEN_WITH_TRACE
+
+#if ZEN_WITH_MEMTRACK
+		FMalloc* TraceMalloc = MemoryTrace_Create(GMalloc);
+		if (TraceMalloc != GMalloc)
+		{
+			GMalloc = TraceMalloc;
+			MemoryTrace_Initialize();
+		}
+#endif
+
 		for (const CommandInfo& CmdInfo : Commands)
 		{
 			if (StrCaseCompare(SubCommand.c_str(), CmdInfo.CmdName) == 0)
diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
index 1552ea823..4ca89d996 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenutil/chunkedcontent.cpp
@@ -7,6 +7,7 @@
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 
 #include <zenutil/chunkedfile.h>
 #include <zenutil/chunkingcontroller.h>
@@ -23,13 +24,13 @@ namespace zen {
 using namespace std::literals;
 
 namespace {
-	void AddCunkSequence(ChunkingStatistics&							   Stats,
-						 ChunkedContentData&							   InOutChunkedContent,
-						 tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
-						 const IoHash&									   RawHash,
-						 std::span<const uint32_t>						   ChunkSequence,
-						 std::span<const IoHash>						   ChunkHashes,
-						 std::span<const uint64_t>						   ChunkRawSizes)
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  std::span<const uint32_t>							ChunkSequence,
+						  std::span<const IoHash>							ChunkHashes,
+						  std::span<const uint64_t>							ChunkRawSizes)
 	{
 		ZEN_ASSERT(ChunkHashes.size() == ChunkRawSizes.size());
 		InOutChunkedContent.ChunkCounts.push_back(gsl::narrow<uint32_t>(ChunkSequence.size()));
@@ -58,11 +59,11 @@ namespace {
 		Stats.UniqueSequencesFound++;
 	}
 
-	void AddCunkSequence(ChunkingStatistics&							   Stats,
-						 ChunkedContentData&							   InOutChunkedContent,
-						 tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
-						 const IoHash&									   RawHash,
-						 const uint64_t									   RawSize)
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  const uint64_t									RawSize)
 	{
 		InOutChunkedContent.ChunkCounts.push_back(1);
 
@@ -120,13 +121,13 @@ namespace {
 						{
 							ChunkSizes.push_back(Source.Size);
 						}
-						AddCunkSequence(Stats,
-										OutChunkedContent.ChunkedContent,
-										ChunkHashToChunkIndex,
-										Chunked.Info.RawHash,
-										Chunked.Info.ChunkSequence,
-										Chunked.Info.ChunkHashes,
-										ChunkSizes);
+						AddChunkSequence(Stats,
+										 OutChunkedContent.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 Chunked.Info.RawHash,
+										 Chunked.Info.ChunkSequence,
+										 Chunked.Info.ChunkHashes,
+										 ChunkSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				});
@@ -143,7 +144,7 @@ namespace {
 					{
 						RawHashToSequenceRawHashIndex.insert(
 							{Hash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())});
-						AddCunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
+						AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
 						Stats.UniqueSequencesFound++;
 					}
 				});
@@ -360,6 +361,8 @@ GetFolderContent(GetFolderContentStatistics&															  Stats,
 				 std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						  UpdateCallback,
 				 std::atomic<bool>&																		  AbortFlag)
 {
+	ZEN_TRACE_CPU("GetFolderContent");
+
 	Stopwatch Timer;
 	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
 
@@ -528,6 +531,8 @@ LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
 ChunkedFolderContent
 MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays)
 {
+	ZEN_TRACE_CPU("MergeChunkedFolderContents");
+
 	ZEN_ASSERT(!Overlays.empty());
 
 	ChunkedFolderContent Result;
@@ -605,13 +610,13 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const Chu
 						ChunkingStatistics		  Stats;
 						std::span<const uint32_t> OriginalChunkOrder =
 							std::span<const uint32_t>(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
-						AddCunkSequence(Stats,
-										Result.ChunkedContent,
-										ChunkHashToChunkIndex,
-										RawHash,
-										OriginalChunkOrder,
-										OverlayContent.ChunkedContent.ChunkHashes,
-										OverlayContent.ChunkedContent.ChunkRawSizes);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 OverlayContent.ChunkedContent.ChunkHashes,
+										 OverlayContent.ChunkedContent.ChunkRawSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				}
@@ -636,6 +641,8 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const Chu
 ChunkedFolderContent
 DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span<const std::filesystem::path> DeletedPaths)
 {
+	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
+
 	ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size());
 	ChunkedFolderContent Result = {.Platform = BaseContent.Platform};
 	if (DeletedPaths.size() < BaseContent.Paths.size())
@@ -673,13 +680,13 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span
 						ChunkingStatistics		  Stats;
 						std::span<const uint32_t> OriginalChunkOrder =
 							std::span<const uint32_t>(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
-						AddCunkSequence(Stats,
-										Result.ChunkedContent,
-										ChunkHashToChunkIndex,
-										RawHash,
-										OriginalChunkOrder,
-										BaseContent.ChunkedContent.ChunkHashes,
-										BaseContent.ChunkedContent.ChunkRawSizes);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 BaseContent.ChunkedContent.ChunkHashes,
+										 BaseContent.ChunkedContent.ChunkRawSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				}
@@ -699,6 +706,8 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 				   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
 				   std::atomic<bool>&												 AbortFlag)
 {
+	ZEN_TRACE_CPU("ChunkFolderContent");
+
 	Stopwatch Timer;
 	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
 
@@ -775,6 +784,8 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 ChunkedContentLookup
 BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 {
+	ZEN_TRACE_CPU("BuildChunkedContentLookup");
+
 	struct ChunkLocationReference
 	{
 		uint32_t									ChunkIndex;
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 481e9146f..309885b05 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -6,6 +6,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 #include <zenutil/jupiter/jupitersession.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -36,6 +37,8 @@ public:
 
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
+		ZEN_TRACE_CPU("Jupiter::ListBuilds");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = Query.GetBuffer().AsIoBuffer();
@@ -51,6 +54,8 @@ public:
 
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBuild");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
@@ -66,6 +71,8 @@ public:
 
 	virtual CbObject GetBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBuild");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _				 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult GetBuildResult = m_Session.GetBuild(m_Namespace, m_Bucket, BuildId);
@@ -79,6 +86,8 @@ public:
 
 	virtual void FinalizeBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuild");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _					  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult FinalizeBuildResult = m_Session.FinalizeBuild(m_Namespace, m_Bucket, BuildId);
@@ -95,6 +104,8 @@ public:
 																std::string_view PartName,
 																const CbObject&	 MetaData) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildPart");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
@@ -110,6 +121,8 @@ public:
 
 	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildPart");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult GetBuildPartResult = m_Session.GetBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId);
@@ -126,6 +139,8 @@ public:
 
 	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
 	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuildPart");
+
 		Stopwatch				ExecutionTimer;
 		auto					_ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		FinalizeBuildPartResult FinalizePartResult = m_Session.FinalizeBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartHash);
@@ -143,6 +158,8 @@ public:
 							  ZenContentType		 ContentType,
 							  const CompositeBuffer& Payload) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildBlob");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _				= MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult PutBlobResult = m_Session.PutBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ContentType, Payload);
@@ -160,6 +177,8 @@ public:
 																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
 																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutLargeBuildBlob");
+
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		std::vector<std::function<JupiterResult(bool&)>> WorkItems;
@@ -200,6 +219,8 @@ public:
 
 	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildBlob");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath);
@@ -218,6 +239,8 @@ public:
 		uint64_t																			   ChunkSize,
 		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob");
+
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		std::vector<std::function<JupiterResult()>> WorkItems;
@@ -249,6 +272,8 @@ public:
 
 	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBlockMetadata");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
@@ -264,6 +289,8 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::FindBlocks");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId);
@@ -277,6 +304,8 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBlockMetadata");
+
 		Stopwatch	   ExecutionTimer;
 		auto		   _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		CbObjectWriter Request;
@@ -365,6 +394,8 @@ CreateJupiterBuildStorage(LoggerRef					   InLog,
 						  std::string_view			   Bucket,
 						  const std::filesystem::path& TempFolderPath)
 {
+	ZEN_TRACE_CPU("CreateJupiterBuildStorage");
+
 	return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath);
 }