path: root/src/zen/trace/trace_model.h

// Copyright Epic Games, Inc. All Rights Reserved.

#pragma once

#include "trace_memory.h"
#include "zen.h"

#include <zencore/workthreadpool.h>

ZEN_THIRD_PARTY_INCLUDES_START
#include <EASTL/vector.h>
ZEN_THIRD_PARTY_INCLUDES_END

#include <cstdint>
#include <filesystem>
#include <functional>
#include <string>

namespace zen::trace_detail {

// Shared trace timing state. Tourist's Dispatcher only allows one subscriber
// per event type, so only one analyzer can own the `$Trace.NewTrace`
// subscription. Other analyzers that need to convert absolute Cycle64 values
// read from this shared struct, which the owning analyzer fills in during its
// OnNewTrace callback.
struct TraceTiming
{
	uint64_t Freq  = 0;
	uint64_t Base  = 0;
	uint64_t UsDiv = 1;

	uint32_t CycleToTimeUs(uint64_t Cycle) const
	{
		uint64_t CycleFromStart = (Cycle >= Base) ? (Cycle - Base) : 0;
		uint64_t D				= (UsDiv > 0) ? UsDiv : 1;
		return uint32_t((CycleFromStart + (D >> 1)) / D);
	}
};

// Safely convert a tourist FieldStr to std::string, stripping trailing NULs
// and returning an empty string on failure.
std::string SafeFieldStr(class FieldStr&& Field);

struct SessionInfo
{
	std::string Platform;
	std::string AppName;
	std::string ProjectName;
	std::string CommandLine;
	std::string Branch;
	std::string BuildVersion;
	uint32_t	Changelist		  = 0;
	uint8_t		ConfigurationType = 0;
	bool		HasSession		  = false;
};

struct ThreadInfoEntry
{
	uint32_t	ThreadId = 0;
	std::string Name;
	std::string GroupName;	// from $Trace.ThreadGroupBegin/End bracketing, or synthesized by stripping a numeric suffix from Name
	uint32_t	SystemId = 0;
	int32_t		SortHint = 0;
};

struct ChannelInfo
{
	std::string Name;
	bool		Enabled	 = false;
	bool		ReadOnly = false;
};

// A DLL / shared library that was loaded (or seen already loaded) during the
// capture. Populated from the Diagnostics.Module{Init,Load,Unload} events
// which are all marked NoSync|Important, so they survive reconnects and our
// own trim filter. Load/unload timestamps aren't available because the events
// don't carry a Cycle field.
struct ModuleInfo
{
	std::string			   Name;	  // basename of FullPath
	std::string			   FullPath;  // full path as reported by the engine
	uint64_t			   Base		= 0;
	uint32_t			   Size		= 0;
	bool				   Unloaded = false;  // set when we see a matching ModuleUnload
	eastl::vector<uint8_t> ImageId;			  // PDB GUID + Age, opaque -- for later symbol lookup
};

// UE verbosity values mirror ELogVerbosity::Type. We expose the raw integer
// so the frontend can map it to a label / color.
struct LogCategoryInfo
{
	std::string Name;
	uint8_t		DefaultVerbosity = 0;
};

struct LogEntry
{
	uint32_t	TimeUs;			// microseconds from the start of the trace
	uint32_t	CategoryIndex;	// index into TraceModel::LogCategories (or ~0u)
	uint8_t		Verbosity;
	int32_t		Line;
	std::string File;
	std::string Message;
};

// Point-in-time marker emitted via TRACE_BOOKMARK / UE_TRACE_BOOKMARK.
// Each entry's Text has already been formatted (FormatString + FormatArgs
// substituted) during parsing.
struct Bookmark
{
	uint32_t	TimeUs;
	int32_t		Line;
	std::string File;
	std::string Text;
};

// A named time range announced via Misc.RegionBegin / Misc.RegionEnd
// (or the newer *WithId variants). Depth is the lane index assigned by
// the analyzer's greedy overlap-avoidance pass.
struct RegionEntry
{
	uint32_t	BeginUs;
	uint32_t	EndUs;	// == TraceEndUs if still open at trace end
	uint16_t	Depth;
	uint16_t	Reserved;
	std::string Name;
	std::string Category;
};

// A group of regions sharing the same category label. Each category has its
// own lane namespace so depths are assigned independently.
struct RegionCategory
{
	std::string				   Name;  // display name; empty categories get "Uncategorized"
	uint32_t				   LaneCount = 0;
	eastl::vector<RegionEntry> Regions;	 // sorted by BeginUs, Depth is per-category
};

struct CpuScopeStat
{
	std::string Name;
	uint64_t	Count	 = 0;
	uint32_t	MinUs	 = 0;
	uint32_t	MaxUs	 = 0;
	double		MeanUs	 = 0.0;
	double		StdDevUs = 0.0;
};

// Single CPU scope interval captured by TimelineAnalyzer. Packed for size:
// timelines can easily contain millions of entries.
struct TimelineScope
{
	uint32_t BeginUs;	  // microseconds from the start of the trace
	uint32_t DurationUs;  // scope duration in microseconds
	uint32_t NameId;	  // index into TraceModel::ScopeNames
	uint16_t Depth;		  // call-stack depth (0 == outermost)
	uint16_t MergeCount;  // 0 = raw (LOD 0), N>0 = N scopes merged (LOD 1+)
};

// Pre-computed detail level for a thread timeline. Each level merges scopes
// shorter than ResolutionUs into "macro scopes" carrying the dominant name
// (the name of the longest contributing scope). The merge count is stored in
// TimelineScope::MergeCount.
struct TimelineDetailLevel
{
	uint32_t					 ResolutionUs = 0;
	eastl::vector<TimelineScope> Scopes;  // sorted by BeginUs
};

// LOD resolutions in microseconds (geometric spacing inspired by Unreal Insights).
// LOD 0 is the raw ThreadTimeline::Scopes; these are LOD 1-5.
inline constexpr uint32_t kTimelineLodResolutions[] = {100, 1000, 8000, 40000, 200000};
inline constexpr size_t	  kTimelineLodCount			= sizeof(kTimelineLodResolutions) / sizeof(kTimelineLodResolutions[0]);

struct ThreadTimeline
{
	uint32_t					 ThreadId = 0;
	std::string					 Name;
	int32_t						 SortHint = 0;
	eastl::vector<TimelineScope> Scopes;  // LOD 0 -- full resolution, sorted by BeginUs

	TimelineDetailLevel DetailLevels[kTimelineLodCount];  // LOD 1-5
};

// Build pre-computed LOD levels for a ThreadTimeline whose Scopes vector is
// already sorted by BeginUs. Called from BuildTraceModel after populating the
// raw scopes.
void BuildTimelineLods(ThreadTimeline& Timeline);

// Complete in-memory view of a parsed .utrace file, produced by BuildTraceModel
// and consumed by the `zen trace serve` subcommand.
struct TraceModel
{
	std::filesystem::path FilePath;
	uint64_t			  FileSize	   = 0;
	uint64_t			  TotalEvents  = 0;
	uint64_t			  ParseTimeMs  = 0;
	uint32_t			  TraceStartUs = 0;
	uint32_t			  TraceEndUs   = 0;

	SessionInfo					   Session;
	eastl::vector<ThreadInfoEntry> Threads;	  // sorted by SortHint
	eastl::vector<ChannelInfo>	   Channels;  // sorted by name
	eastl::vector<ModuleInfo>	   Modules;	  // sorted by Name

	eastl::vector<std::string>	  ScopeNames;  // referenced by TimelineScope::NameId
	eastl::vector<CpuScopeStat>	  ScopeStats;  // sorted by Count descending
	eastl::vector<ThreadTimeline> Timelines;   // one entry per thread that produced scopes

	eastl::vector<LogCategoryInfo> LogCategories;  // referenced by LogEntry::CategoryIndex
	eastl::vector<LogEntry>		   LogEntries;	   // sorted by TimeUs

	eastl::vector<Bookmark>		  Bookmarks;		 // sorted by TimeUs
	eastl::vector<RegionCategory> RegionCategories;	 // sorted: uncategorized first, then alpha

	// -- CsvProfiler --
	struct CsvCategory
	{
		int32_t		Index = 0;
		std::string Name;
	};

	struct CsvStatDef
	{
		uint64_t	StatId		  = 0;
		int32_t		CategoryIndex = 0;
		std::string Name;
	};

	struct CsvSample
	{
		uint32_t TimeUs;
		float	 Value;
	};

	// Time series for one stat on one thread.
	struct CsvSeries
	{
		uint64_t				 StatId	  = 0;
		uint32_t				 ThreadId = 0;
		eastl::vector<CsvSample> Samples;  // sorted by TimeUs
	};

	struct CsvEvent
	{
		uint32_t	TimeUs;
		int32_t		CategoryIndex;
		std::string Text;
	};

	struct CsvMeta
	{
		std::string Key;
		std::string Value;
	};

	eastl::vector<CsvCategory> CsvCategories;
	eastl::vector<CsvStatDef>  CsvStatDefs;
	eastl::vector<CsvSeries>   CsvTimeSeries;  // per stat+thread
	eastl::vector<CsvEvent>	   CsvEvents;	   // sorted by TimeUs
	eastl::vector<CsvMeta>	   CsvMetadata;

	// -- Counters (TRACE_INT_VALUE / TRACE_FLOAT_VALUE / TRACE_MEMORY_VALUE) --
	// One CounterDef per registered counter (Counters.Spec event), and one
	// CounterSeries per counter that produced any samples (Counters.SetValueInt
	// / SetValueFloat events).
	struct CounterDef
	{
		uint16_t	Id			= 0;
		uint8_t		Type		= 0;  // 0 = Int, 1 = Float
		uint8_t		DisplayHint = 0;  // 0 = None, 1 = Memory
		std::string Name;
	};

	struct CounterSample
	{
		uint32_t TimeUs;
		double	 Value;	 // int counters are widened to double for transport;
						 // exact int values up to 2^53 round-trip losslessly.
	};

	struct CounterSeries
	{
		uint16_t					 Id	   = 0;
		uint8_t						 Type  = 0;	 // mirrors CounterDef::Type
		uint8_t						 Pad   = 0;
		uint32_t					 Count = 0;
		double						 Min   = 0.0;
		double						 Max   = 0.0;
		eastl::vector<CounterSample> Samples;  // sorted by TimeUs
	};

	eastl::vector<CounterDef>	 CounterDefs;		 // sorted by Id
	eastl::vector<CounterSeries> CounterTimeSeries;	 // one per counter that produced samples, sorted by Id

	// -- Event type counts (sorted by count descending) --
	struct EventTypeCount
	{
		std::string Name;
		uint64_t	Count = 0;
	};
	eastl::vector<EventTypeCount> EventTypeCounts;

	// -- Memory allocations --
	AllocationSummary					AllocSummary;
	eastl::vector<HeapInfo>				Heaps;				 // sorted by Id
	eastl::vector<TagInfo>				Tags;				 // sorted by Tag
	eastl::vector<MemoryTimelineSample> MemoryTimeline;		 // sorted by TimeUs
	eastl::vector<HeapStat>				HeapStats;			 // sorted by HeapId
	eastl::vector<CallstackEntry>		Callstacks;			 // sorted by Id
	eastl::vector<CallstackAllocStat>	CallstackStats;		 // sorted by LiveBytes desc
	eastl::vector<CallstackChurnStat>	ChurnStats;			 // sorted by TotalAllocs desc
	eastl::vector<AllocSizeBucket>		AllocSizeHistogram;	 // sorted by MinSize asc, populated buckets only
};

// Resolve and validate a .utrace file path. Throws OptionParseException when
// the path is empty and runtime_error when the file does not exist.
std::filesystem::path ResolveTraceFile(const std::filesystem::path& Input, cxxopts::Options& HelpOptions);

// Parse a .utrace file and print the event-schema inspect report to the console.
void RunInspect(const std::filesystem::path& FilePath);

// Progress callback invoked once per bundle during trace iteration.
// Arguments: BytesProcessed (estimated), TotalFileBytes, EventsSoFar.
using ProgressCallback = std::function<void(uint64_t, uint64_t, uint64_t)>;

// Parse a .utrace file into an in-memory TraceModel suitable for serving via
// the trace viewer. A single pass runs the session, CPU-stats and timeline
// analyzers. The optional progress callback is invoked once per bundle.
TraceModel BuildTraceModel(const std::filesystem::path& FilePath, WorkerThreadPool& ThreadPool, const ProgressCallback& OnProgress = {});

struct TraceTrimArgs
{
	std::filesystem::path InputPath;
	std::filesystem::path OutputPath;
	double				  StartSec = 0.0;
	double				  EndSec   = 0.0;
};

// Produce a trimmed .utrace file containing all type-definition and "important"
// packets from the input, plus any regular thread packets whose events overlap
// the [StartSec, EndSec] window. The output remains a valid .utrace that can be
// read by Unreal Insights and zen's own trace tooling. Trimming is coarse at
// the packet level: a packet that straddles the window boundary is kept in full.
void RunTraceTrim(const TraceTrimArgs& Args);

}  // namespace zen::trace_detail