Add trace support for zen CLI command (#296)

- This change adds support for `--trace`, `--tracehost` and `--tracefile` command arguments to enable and control tracing to Insights
- It also adds profiling scopes primarily to build download command related code

2 files changed, 76 insertions, 34 deletions

diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
index 1552ea823..4ca89d996 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenutil/chunkedcontent.cpp
@@ -7,6 +7,7 @@
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 
 #include <zenutil/chunkedfile.h>
 #include <zenutil/chunkingcontroller.h>
@@ -23,13 +24,13 @@ namespace zen {
 using namespace std::literals;
 
 namespace {
-	void AddCunkSequence(ChunkingStatistics&							   Stats,
-						 ChunkedContentData&							   InOutChunkedContent,
-						 tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
-						 const IoHash&									   RawHash,
-						 std::span<const uint32_t>						   ChunkSequence,
-						 std::span<const IoHash>						   ChunkHashes,
-						 std::span<const uint64_t>						   ChunkRawSizes)
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  std::span<const uint32_t>							ChunkSequence,
+						  std::span<const IoHash>							ChunkHashes,
+						  std::span<const uint64_t>							ChunkRawSizes)
 	{
 		ZEN_ASSERT(ChunkHashes.size() == ChunkRawSizes.size());
 		InOutChunkedContent.ChunkCounts.push_back(gsl::narrow<uint32_t>(ChunkSequence.size()));
@@ -58,11 +59,11 @@ namespace {
 		Stats.UniqueSequencesFound++;
 	}
 
-	void AddCunkSequence(ChunkingStatistics&							   Stats,
-						 ChunkedContentData&							   InOutChunkedContent,
-						 tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
-						 const IoHash&									   RawHash,
-						 const uint64_t									   RawSize)
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  const uint64_t									RawSize)
 	{
 		InOutChunkedContent.ChunkCounts.push_back(1);
 
@@ -120,13 +121,13 @@ namespace {
 						{
 							ChunkSizes.push_back(Source.Size);
 						}
-						AddCunkSequence(Stats,
-										OutChunkedContent.ChunkedContent,
-										ChunkHashToChunkIndex,
-										Chunked.Info.RawHash,
-										Chunked.Info.ChunkSequence,
-										Chunked.Info.ChunkHashes,
-										ChunkSizes);
+						AddChunkSequence(Stats,
+										 OutChunkedContent.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 Chunked.Info.RawHash,
+										 Chunked.Info.ChunkSequence,
+										 Chunked.Info.ChunkHashes,
+										 ChunkSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				});
@@ -143,7 +144,7 @@ namespace {
 					{
 						RawHashToSequenceRawHashIndex.insert(
 							{Hash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())});
-						AddCunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
+						AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
 						Stats.UniqueSequencesFound++;
 					}
 				});
@@ -360,6 +361,8 @@ GetFolderContent(GetFolderContentStatistics&															  Stats,
 				 std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						  UpdateCallback,
 				 std::atomic<bool>&																		  AbortFlag)
 {
+	ZEN_TRACE_CPU("GetFolderContent");
+
 	Stopwatch Timer;
 	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
 
@@ -528,6 +531,8 @@ LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
 ChunkedFolderContent
 MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays)
 {
+	ZEN_TRACE_CPU("MergeChunkedFolderContents");
+
 	ZEN_ASSERT(!Overlays.empty());
 
 	ChunkedFolderContent Result;
@@ -605,13 +610,13 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const Chu
 						ChunkingStatistics		  Stats;
 						std::span<const uint32_t> OriginalChunkOrder =
 							std::span<const uint32_t>(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
-						AddCunkSequence(Stats,
-										Result.ChunkedContent,
-										ChunkHashToChunkIndex,
-										RawHash,
-										OriginalChunkOrder,
-										OverlayContent.ChunkedContent.ChunkHashes,
-										OverlayContent.ChunkedContent.ChunkRawSizes);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 OverlayContent.ChunkedContent.ChunkHashes,
+										 OverlayContent.ChunkedContent.ChunkRawSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				}
@@ -636,6 +641,8 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const Chu
 ChunkedFolderContent
 DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span<const std::filesystem::path> DeletedPaths)
 {
+	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
+
 	ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size());
 	ChunkedFolderContent Result = {.Platform = BaseContent.Platform};
 	if (DeletedPaths.size() < BaseContent.Paths.size())
@@ -673,13 +680,13 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span
 						ChunkingStatistics		  Stats;
 						std::span<const uint32_t> OriginalChunkOrder =
 							std::span<const uint32_t>(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
-						AddCunkSequence(Stats,
-										Result.ChunkedContent,
-										ChunkHashToChunkIndex,
-										RawHash,
-										OriginalChunkOrder,
-										BaseContent.ChunkedContent.ChunkHashes,
-										BaseContent.ChunkedContent.ChunkRawSizes);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 BaseContent.ChunkedContent.ChunkHashes,
+										 BaseContent.ChunkedContent.ChunkRawSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				}
@@ -699,6 +706,8 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 				   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
 				   std::atomic<bool>&												 AbortFlag)
 {
+	ZEN_TRACE_CPU("ChunkFolderContent");
+
 	Stopwatch Timer;
 	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
 
@@ -775,6 +784,8 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 ChunkedContentLookup
 BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 {
+	ZEN_TRACE_CPU("BuildChunkedContentLookup");
+
 	struct ChunkLocationReference
 	{
 		uint32_t									ChunkIndex;
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 481e9146f..309885b05 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -6,6 +6,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 #include <zenutil/jupiter/jupitersession.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -36,6 +37,8 @@ public:
 
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
+		ZEN_TRACE_CPU("Jupiter::ListBuilds");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = Query.GetBuffer().AsIoBuffer();
@@ -51,6 +54,8 @@ public:
 
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBuild");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
@@ -66,6 +71,8 @@ public:
 
 	virtual CbObject GetBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBuild");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _				 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult GetBuildResult = m_Session.GetBuild(m_Namespace, m_Bucket, BuildId);
@@ -79,6 +86,8 @@ public:
 
 	virtual void FinalizeBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuild");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _					  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult FinalizeBuildResult = m_Session.FinalizeBuild(m_Namespace, m_Bucket, BuildId);
@@ -95,6 +104,8 @@ public:
 																std::string_view PartName,
 																const CbObject&	 MetaData) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildPart");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
@@ -110,6 +121,8 @@ public:
 
 	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildPart");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult GetBuildPartResult = m_Session.GetBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId);
@@ -126,6 +139,8 @@ public:
 
 	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
 	{
+		ZEN_TRACE_CPU("Jupiter::FinalizeBuildPart");
+
 		Stopwatch				ExecutionTimer;
 		auto					_ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		FinalizeBuildPartResult FinalizePartResult = m_Session.FinalizeBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartHash);
@@ -143,6 +158,8 @@ public:
 							  ZenContentType		 ContentType,
 							  const CompositeBuffer& Payload) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBuildBlob");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _				= MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult PutBlobResult = m_Session.PutBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ContentType, Payload);
@@ -160,6 +177,8 @@ public:
 																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
 																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutLargeBuildBlob");
+
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		std::vector<std::function<JupiterResult(bool&)>> WorkItems;
@@ -200,6 +219,8 @@ public:
 
 	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBuildBlob");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _					 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath);
@@ -218,6 +239,8 @@ public:
 		uint64_t																			   ChunkSize,
 		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob");
+
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		std::vector<std::function<JupiterResult()>> WorkItems;
@@ -249,6 +272,8 @@ public:
 
 	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("Jupiter::PutBlockMetadata");
+
 		Stopwatch ExecutionTimer;
 		auto	  _		  = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		IoBuffer  Payload = MetaData.GetBuffer().AsIoBuffer();
@@ -264,6 +289,8 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("Jupiter::FindBlocks");
+
 		Stopwatch	  ExecutionTimer;
 		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId);
@@ -277,6 +304,8 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
+		ZEN_TRACE_CPU("Jupiter::GetBlockMetadata");
+
 		Stopwatch	   ExecutionTimer;
 		auto		   _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
 		CbObjectWriter Request;
@@ -365,6 +394,8 @@ CreateJupiterBuildStorage(LoggerRef					   InLog,
 						  std::string_view			   Bucket,
 						  const std::filesystem::path& TempFolderPath)
 {
+	ZEN_TRACE_CPU("CreateJupiterBuildStorage");
+
 	return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath);
 }