Add trace support for zen CLI command (#296)

- This change adds support for `--trace`, `--tracehost` and `--tracefile` command arguments to enable and control tracing to Insights
- It also adds profiling scopes primarily to build download command related code

1 files changed, 45 insertions, 34 deletions

diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
index 1552ea823..4ca89d996 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenutil/chunkedcontent.cpp
@@ -7,6 +7,7 @@
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 
 #include <zenutil/chunkedfile.h>
 #include <zenutil/chunkingcontroller.h>
@@ -23,13 +24,13 @@ namespace zen {
 using namespace std::literals;
 
 namespace {
-	void AddCunkSequence(ChunkingStatistics&							   Stats,
-						 ChunkedContentData&							   InOutChunkedContent,
-						 tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
-						 const IoHash&									   RawHash,
-						 std::span<const uint32_t>						   ChunkSequence,
-						 std::span<const IoHash>						   ChunkHashes,
-						 std::span<const uint64_t>						   ChunkRawSizes)
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  std::span<const uint32_t>							ChunkSequence,
+						  std::span<const IoHash>							ChunkHashes,
+						  std::span<const uint64_t>							ChunkRawSizes)
 	{
 		ZEN_ASSERT(ChunkHashes.size() == ChunkRawSizes.size());
 		InOutChunkedContent.ChunkCounts.push_back(gsl::narrow<uint32_t>(ChunkSequence.size()));
@@ -58,11 +59,11 @@ namespace {
 		Stats.UniqueSequencesFound++;
 	}
 
-	void AddCunkSequence(ChunkingStatistics&							   Stats,
-						 ChunkedContentData&							   InOutChunkedContent,
-						 tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
-						 const IoHash&									   RawHash,
-						 const uint64_t									   RawSize)
+	void AddChunkSequence(ChunkingStatistics&								Stats,
+						  ChunkedContentData&								InOutChunkedContent,
+						  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+						  const IoHash&										RawHash,
+						  const uint64_t									RawSize)
 	{
 		InOutChunkedContent.ChunkCounts.push_back(1);
 
@@ -120,13 +121,13 @@ namespace {
 						{
 							ChunkSizes.push_back(Source.Size);
 						}
-						AddCunkSequence(Stats,
-										OutChunkedContent.ChunkedContent,
-										ChunkHashToChunkIndex,
-										Chunked.Info.RawHash,
-										Chunked.Info.ChunkSequence,
-										Chunked.Info.ChunkHashes,
-										ChunkSizes);
+						AddChunkSequence(Stats,
+										 OutChunkedContent.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 Chunked.Info.RawHash,
+										 Chunked.Info.ChunkSequence,
+										 Chunked.Info.ChunkHashes,
+										 ChunkSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				});
@@ -143,7 +144,7 @@ namespace {
 					{
 						RawHashToSequenceRawHashIndex.insert(
 							{Hash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())});
-						AddCunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
+						AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize);
 						Stats.UniqueSequencesFound++;
 					}
 				});
@@ -360,6 +361,8 @@ GetFolderContent(GetFolderContentStatistics&															  Stats,
 				 std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						  UpdateCallback,
 				 std::atomic<bool>&																		  AbortFlag)
 {
+	ZEN_TRACE_CPU("GetFolderContent");
+
 	Stopwatch Timer;
 	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
 
@@ -528,6 +531,8 @@ LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
 ChunkedFolderContent
 MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays)
 {
+	ZEN_TRACE_CPU("MergeChunkedFolderContents");
+
 	ZEN_ASSERT(!Overlays.empty());
 
 	ChunkedFolderContent Result;
@@ -605,13 +610,13 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const Chu
 						ChunkingStatistics		  Stats;
 						std::span<const uint32_t> OriginalChunkOrder =
 							std::span<const uint32_t>(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
-						AddCunkSequence(Stats,
-										Result.ChunkedContent,
-										ChunkHashToChunkIndex,
-										RawHash,
-										OriginalChunkOrder,
-										OverlayContent.ChunkedContent.ChunkHashes,
-										OverlayContent.ChunkedContent.ChunkRawSizes);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 OverlayContent.ChunkedContent.ChunkHashes,
+										 OverlayContent.ChunkedContent.ChunkRawSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				}
@@ -636,6 +641,8 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const Chu
 ChunkedFolderContent
 DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span<const std::filesystem::path> DeletedPaths)
 {
+	ZEN_TRACE_CPU("DeletePathsFromChunkedContent");
+
 	ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size());
 	ChunkedFolderContent Result = {.Platform = BaseContent.Platform};
 	if (DeletedPaths.size() < BaseContent.Paths.size())
@@ -673,13 +680,13 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span
 						ChunkingStatistics		  Stats;
 						std::span<const uint32_t> OriginalChunkOrder =
 							std::span<const uint32_t>(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount);
-						AddCunkSequence(Stats,
-										Result.ChunkedContent,
-										ChunkHashToChunkIndex,
-										RawHash,
-										OriginalChunkOrder,
-										BaseContent.ChunkedContent.ChunkHashes,
-										BaseContent.ChunkedContent.ChunkRawSizes);
+						AddChunkSequence(Stats,
+										 Result.ChunkedContent,
+										 ChunkHashToChunkIndex,
+										 RawHash,
+										 OriginalChunkOrder,
+										 BaseContent.ChunkedContent.ChunkHashes,
+										 BaseContent.ChunkedContent.ChunkRawSizes);
 						Stats.UniqueSequencesFound++;
 					}
 				}
@@ -699,6 +706,8 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 				   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
 				   std::atomic<bool>&												 AbortFlag)
 {
+	ZEN_TRACE_CPU("ChunkFolderContent");
+
 	Stopwatch Timer;
 	auto	  _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); });
 
@@ -775,6 +784,8 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 ChunkedContentLookup
 BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 {
+	ZEN_TRACE_CPU("BuildChunkedContentLookup");
+
 	struct ChunkLocationReference
 	{
 		uint32_t									ChunkIndex;