aboutsummaryrefslogtreecommitdiff
path: root/src/zen/trace/trace_cache.h
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-04-20 21:50:41 +0200
committerGitHub Enterprise <[email protected]>2026-04-20 21:50:41 +0200
commit2dfb5da16b97a6c12e01977af5b5188522178a4e (patch)
tree428aa0aa8e6079c64438931e0fd4f828c613c94d /src/zen/trace/trace_cache.h
parentAdd CompactString utility type (#990) (diff)
downloadarchived-zen-2dfb5da16b97a6c12e01977af5b5188522178a4e.tar.xz
archived-zen-2dfb5da16b97a6c12e01977af5b5188522178a4e.zip
zen trace analysis support (#945)
Integrates the **tourist** trace analysis library and builds a full `zen trace` command suite for working with Unreal Engine `.utrace` files. ### Trace analysis library (`thirdparty/tourist/`) - Adds the tourist library as a third-party dependency with three modules: **foundation** (platform primitives, memory, scheduling), **trace** (UE Trace protocol decoding), and **analysis** (event dispatching and analyzer framework). - Cross-platform support for Windows, Linux, and macOS. ### `zen trace` CLI commands (`src/zen/cmds/`, `src/zen/trace/`) - **`zen trace analyze`** — Summarize a `.utrace` file: session metadata, thread inventory, command line + build configuration, CPU profiling scopes, timing, event rates, log messages, and (with symbols) memory allocation metrics including live-allocs dumps, callstack-keyed aggregation, and allocation churn. Optional HTML output for memory reports. - **`zen trace inspect`** — Dump the event schema (declared types, fields, sizes) from a trace file. - **`zen trace trim`** — Extract a time-window from a trace into a new `.utrace` file. - **`zen trace serve`** — Launch a local HTTP server hosting an interactive trace viewer; opens in the default browser. ### Symbolication (`src/zen/trace/symbol_resolver.*`, `thirdparty/raw_pdb/`) - Pluggable resolver with multiple backends: `pdb` (in-tree raw_pdb), `dbghelp` (Windows), `llvm-symbolizer` (all platforms), `atos` (macOS). An `auto` backend picks the best available tool per platform. - Microsoft Symbol Server support: downloads PDBs on demand using a redirect-aware HTTP client. - Local PDB cache keyed by image GUID preserves symbols across binary recompilation. - Callstack trimming heuristic strips UE internal noise from reports. - Binary analysis cache (`.ucache_z`) avoids re-resolving the same trace. ### Interactive trace viewer (`src/zen/frontend/html/`, `src/zen/trace/trace_viewer_service.*`) - Timeline: scope-level detail, horizontal zoom/pan, vertical scrolling, viewport-driven loading with pre-computed LOD for responsive navigation of large traces. - Thread grouping (collapsible sidebar sections) synthesized from name suffixes, natural sort order, visual distinction between lane threads and OS threads. - Bookmark and region annotations; region categories with per-category toggles; bookmark marker toggle in the toolbar. - Filterable Logs tab showing captured `UE_LOG` output. - Stats tab with per-scope aggregate statistics. - Memory tab with interactive allocation analysis and an allocation size histogram. - CsvProfiler event parsing and chart UI. ### Other in-branch supporting changes - **Cross-platform browser launcher** (`browser_launcher.{h,cpp}`) used by `trace serve`. - **`ReciprocalU64`** fast 64-bit integer division (zencore/intmath) for trace analyzers. - **`parallelsort`** cross-platform parallel sort helper (zenutil). - Frontend zip build rule so the viewer's HTML assets are bundled into `zen.exe`. - `/Zo` flag for better optimized debug info on Windows release builds. - `trace-tests.cpp` in the `zen-test` harness (harness itself landed on main via #985).
Diffstat (limited to 'src/zen/trace/trace_cache.h')
-rw-r--r--src/zen/trace/trace_cache.h253
1 files changed, 253 insertions, 0 deletions
diff --git a/src/zen/trace/trace_cache.h b/src/zen/trace/trace_cache.h
new file mode 100644
index 000000000..88778a020
--- /dev/null
+++ b/src/zen/trace/trace_cache.h
@@ -0,0 +1,253 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "symbol_resolver.h"
+#include "trace_model.h"
+
+#include <zencore/sharedbuffer.h>
+
+#include <cstdint>
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <string_view>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <EASTL/hash_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen::trace_detail {
+
+// ---------------------------------------------------------------------------
+// File format constants
+// ---------------------------------------------------------------------------
+
+static constexpr uint32_t kCacheMagic = 0x005A4355; // "UCZ\0"
+static constexpr uint32_t kCacheVersion = 1;
+
+enum class CacheSectionId : uint32_t
+{
+ StringTable = 0,
+ Metadata = 1,
+ Memory = 2,
+ Callstacks = 3,
+ Symbols = 4,
+ Count
+};
+
+// ---------------------------------------------------------------------------
+// On-disk header structures (naturally aligned, no packing)
+//
+// Fields are ordered so natural alignment matches across compilers without
+// needing pragma pack. static_asserts at the bottom of this block pin the
+// layout so a reordering or added field cannot silently break cached files.
+// ---------------------------------------------------------------------------
+
+struct CacheFileHeader
+{
+ uint32_t Magic;
+ uint32_t Version;
+ uint64_t SourceFileSize;
+ int64_t SourceModTimeNs; // last_write_time as nanoseconds since epoch
+ uint64_t Reserved;
+};
+
+struct SectionDirectoryEntry
+{
+ uint32_t SectionId;
+ uint32_t Reserved;
+ uint64_t FileOffset; // byte offset from start of file
+ uint64_t CompressedSize; // size of the CompressedBuffer blob on disk
+};
+
+// ---------------------------------------------------------------------------
+// POD types for memory-mappable section content
+// ---------------------------------------------------------------------------
+
+struct MetadataPod
+{
+ uint64_t FileSize;
+ uint64_t TotalEvents;
+ uint64_t ParseTimeMs;
+ uint32_t TraceStartUs;
+ uint32_t TraceEndUs;
+ // SessionInfo string indices
+ uint32_t SessionPlatform;
+ uint32_t SessionAppName;
+ uint32_t SessionProjectName;
+ uint32_t SessionCommandLine;
+ uint32_t SessionBranch;
+ uint32_t SessionBuildVersion;
+ uint32_t SessionChangelist;
+ uint8_t SessionConfigType;
+ uint8_t SessionHasSession;
+ uint8_t Padding[2];
+};
+
+struct ThreadInfoPod
+{
+ uint32_t ThreadId;
+ uint32_t Name; // string index
+ uint32_t GroupName; // string index
+ uint32_t SystemId;
+ int32_t SortHint;
+ uint32_t Pad;
+};
+
+struct ChannelInfoPod
+{
+ uint32_t Name; // string index
+ uint8_t Enabled;
+ uint8_t ReadOnly;
+ uint8_t Pad[2];
+};
+
+struct ModuleInfoPod
+{
+ uint32_t Name; // string index
+ uint32_t FullPath; // string index
+ uint64_t Base;
+ uint32_t Size;
+ uint32_t ImageIdSize; // byte count in the ImageId blob area
+ uint32_t ImageIdOffset; // byte offset into the ImageId blob area
+ uint32_t Pad;
+};
+
+struct EventTypeCountPod
+{
+ uint32_t Name; // string index
+ uint32_t Pad;
+ uint64_t Count;
+};
+
+struct CpuScopeStatPod
+{
+ uint32_t Name; // string index
+ uint32_t MinUs;
+ uint32_t MaxUs;
+ uint32_t Pad;
+ uint64_t Count;
+ double MeanUs;
+ double StdDevUs;
+};
+
+struct AllocSummaryPod
+{
+ uint8_t HasMemoryData;
+ uint8_t Pad0[3];
+ uint32_t PeakTimeUs;
+ uint32_t LiveAllocations;
+ uint32_t Pad1;
+ uint64_t TotalAllocs;
+ uint64_t TotalFrees;
+ uint64_t TotalReallocAllocs;
+ uint64_t TotalReallocFrees;
+ int64_t PeakBytes;
+ int64_t EndBytes;
+};
+
+struct HeapInfoPod
+{
+ uint32_t Id;
+ uint32_t ParentId;
+ uint16_t Flags;
+ uint16_t Pad0;
+ uint32_t Name; // string index
+};
+
+struct HeapStatPod
+{
+ uint32_t HeapId;
+ uint32_t Pad;
+ int64_t CurrentBytes;
+ int64_t PeakBytes;
+ uint64_t AllocCount;
+ uint64_t FreeCount;
+};
+
+struct CallstackAllocStatPod
+{
+ uint32_t CallstackId;
+ uint32_t LiveCount;
+ int64_t LiveBytes;
+ uint32_t ThreadIdCount;
+ uint32_t ThreadIds[4];
+ uint32_t Pad;
+ uint32_t Pad2;
+};
+
+struct CallstackChurnStatPod
+{
+ uint32_t CallstackId;
+ uint32_t Pad;
+ uint64_t ChurnAllocs;
+ uint64_t ChurnBytes;
+ uint64_t TotalAllocs;
+ uint64_t TotalBytes;
+ double MeanDistance;
+};
+
+struct CallstackHeaderPod
+{
+ uint32_t Id;
+ uint32_t FrameCount;
+ uint32_t FrameOffset; // index into the frames array
+ uint32_t Pad;
+};
+
+struct ResolvedFramePod
+{
+ uint64_t Address;
+ uint32_t ModuleIndex;
+ uint32_t Pad;
+ uint64_t Offset;
+};
+
+struct SymbolEntryPod
+{
+ uint64_t Address;
+ uint32_t StringIdx; // index into the string table
+ uint32_t Pad;
+};
+
+// Pin the on-disk layout. Any change here is a cache format change and must
+// bump kCacheVersion.
+static_assert(sizeof(CacheFileHeader) == 32);
+static_assert(sizeof(SectionDirectoryEntry) == 24);
+static_assert(sizeof(MetadataPod) == 64);
+static_assert(sizeof(ThreadInfoPod) == 24);
+static_assert(sizeof(ChannelInfoPod) == 8);
+static_assert(sizeof(ModuleInfoPod) == 32);
+static_assert(sizeof(EventTypeCountPod) == 16);
+static_assert(sizeof(CpuScopeStatPod) == 40);
+static_assert(sizeof(AllocSummaryPod) == 64);
+static_assert(sizeof(HeapInfoPod) == 16);
+static_assert(sizeof(HeapStatPod) == 40);
+static_assert(sizeof(CallstackAllocStatPod) == 48);
+static_assert(sizeof(CallstackChurnStatPod) == 48);
+static_assert(sizeof(CallstackHeaderPod) == 16);
+static_assert(sizeof(ResolvedFramePod) == 24);
+static_assert(sizeof(SymbolEntryPod) == 16);
+
+// ---------------------------------------------------------------------------
+// Cache read / write API
+// ---------------------------------------------------------------------------
+
+struct CachedAnalysis
+{
+ TraceModel Model;
+ std::unique_ptr<SymbolResolver> Symbols;
+};
+
+// Try to load a cached analysis from the .ucache_z file next to a .utrace.
+// Returns nullopt on any failure (missing, stale, corrupt, version mismatch).
+std::optional<CachedAnalysis> TryLoadAnalyzeCache(const std::filesystem::path& CachePath, const std::filesystem::path& SourcePath);
+
+// Write the analysis cache for future reuse.
+void WriteAnalyzeCache(const std::filesystem::path& CachePath,
+ const std::filesystem::path& SourcePath,
+ const TraceModel& Model,
+ const eastl::hash_map<uint64_t, std::string>& ResolvedSymbols);
+
+} // namespace zen::trace_detail