From 2dfb5da16b97a6c12e01977af5b5188522178a4e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 20 Apr 2026 21:50:41 +0200 Subject: zen trace analysis support (#945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrates the **tourist** trace analysis library and builds a full `zen trace` command suite for working with Unreal Engine `.utrace` files. ### Trace analysis library (`thirdparty/tourist/`) - Adds the tourist library as a third-party dependency with three modules: **foundation** (platform primitives, memory, scheduling), **trace** (UE Trace protocol decoding), and **analysis** (event dispatching and analyzer framework). - Cross-platform support for Windows, Linux, and macOS. ### `zen trace` CLI commands (`src/zen/cmds/`, `src/zen/trace/`) - **`zen trace analyze`** — Summarize a `.utrace` file: session metadata, thread inventory, command line + build configuration, CPU profiling scopes, timing, event rates, log messages, and (with symbols) memory allocation metrics including live-allocs dumps, callstack-keyed aggregation, and allocation churn. Optional HTML output for memory reports. - **`zen trace inspect`** — Dump the event schema (declared types, fields, sizes) from a trace file. - **`zen trace trim`** — Extract a time-window from a trace into a new `.utrace` file. - **`zen trace serve`** — Launch a local HTTP server hosting an interactive trace viewer; opens in the default browser. ### Symbolication (`src/zen/trace/symbol_resolver.*`, `thirdparty/raw_pdb/`) - Pluggable resolver with multiple backends: `pdb` (in-tree raw_pdb), `dbghelp` (Windows), `llvm-symbolizer` (all platforms), `atos` (macOS). An `auto` backend picks the best available tool per platform. - Microsoft Symbol Server support: downloads PDBs on demand using a redirect-aware HTTP client. - Local PDB cache keyed by image GUID preserves symbols across binary recompilation. - Callstack trimming heuristic strips UE internal noise from reports. - Binary analysis cache (`.ucache_z`) avoids re-resolving the same trace. ### Interactive trace viewer (`src/zen/frontend/html/`, `src/zen/trace/trace_viewer_service.*`) - Timeline: scope-level detail, horizontal zoom/pan, vertical scrolling, viewport-driven loading with pre-computed LOD for responsive navigation of large traces. - Thread grouping (collapsible sidebar sections) synthesized from name suffixes, natural sort order, visual distinction between lane threads and OS threads. - Bookmark and region annotations; region categories with per-category toggles; bookmark marker toggle in the toolbar. - Filterable Logs tab showing captured `UE_LOG` output. - Stats tab with per-scope aggregate statistics. - Memory tab with interactive allocation analysis and an allocation size histogram. - CsvProfiler event parsing and chart UI. ### Other in-branch supporting changes - **Cross-platform browser launcher** (`browser_launcher.{h,cpp}`) used by `trace serve`. - **`ReciprocalU64`** fast 64-bit integer division (zencore/intmath) for trace analyzers. - **`parallelsort`** cross-platform parallel sort helper (zenutil). - Frontend zip build rule so the viewer's HTML assets are bundled into `zen.exe`. - `/Zo` flag for better optimized debug info on Windows release builds. - `trace-tests.cpp` in the `zen-test` harness (harness itself landed on main via #985). --- thirdparty/raw_pdb/src/PDB_ModuleInfoStream.cpp | 184 ++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 thirdparty/raw_pdb/src/PDB_ModuleInfoStream.cpp (limited to 'thirdparty/raw_pdb/src/PDB_ModuleInfoStream.cpp') diff --git a/thirdparty/raw_pdb/src/PDB_ModuleInfoStream.cpp b/thirdparty/raw_pdb/src/PDB_ModuleInfoStream.cpp new file mode 100644 index 000000000..86040fffb --- /dev/null +++ b/thirdparty/raw_pdb/src/PDB_ModuleInfoStream.cpp @@ -0,0 +1,184 @@ +// Copyright 2011-2022, Molecular Matters GmbH +// See LICENSE.txt for licensing details (2-clause BSD License: https://opensource.org/licenses/BSD-2-Clause) + +#include "PDB_PCH.h" +#include "PDB_ModuleInfoStream.h" +#include "Foundation/PDB_Memory.h" +#include "Foundation/PDB_CRT.h" + +namespace +{ + static constexpr const char* LinkerSymbolName("* Linker *"); + + + // ------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------ + PDB_NO_DISCARD static inline size_t EstimateModuleCount(size_t streamSize) PDB_NO_EXCEPT + { + // work out how many modules are stored in the stream at most. + // the module info is stored in variable-length records, so we can't determine the exact number without walking the stream. + return streamSize / sizeof(PDB::DBI::ModuleInfo); + } +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::ModuleInfoStream::Module::Module(void) PDB_NO_EXCEPT + : m_info(nullptr) + , m_name(nullptr) + , m_nameLength(0u) + , m_objectName(nullptr) + , m_objectNameLength(0u) +{ +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::ModuleInfoStream::Module::Module(const DBI::ModuleInfo* info, const char* name, size_t nameLength, const char* objectName, size_t objectNameLength) PDB_NO_EXCEPT + : m_info(info) + , m_name(name) + , m_nameLength(nameLength) + , m_objectName(objectName) + , m_objectNameLength(objectNameLength) +{ +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB_NO_DISCARD bool PDB::ModuleInfoStream::Module::HasSymbolStream(void) const PDB_NO_EXCEPT +{ + const uint16_t streamIndex = m_info->moduleSymbolStreamIndex; + + // some modules don't have a symbol stream, i.e. no additional debug information is present. + // this usually happens when private symbols are stripped from a PDB. + return (streamIndex != 0xFFFFu); +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB_NO_DISCARD bool PDB::ModuleInfoStream::Module::HasLineStream(void) const PDB_NO_EXCEPT +{ + return (m_info->c13Size > 0); +} + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB_NO_DISCARD PDB::ModuleSymbolStream PDB::ModuleInfoStream::Module::CreateSymbolStream(const RawFile& file) const PDB_NO_EXCEPT +{ + PDB_ASSERT(HasSymbolStream(), "Module symbol stream index is invalid."); + + return ModuleSymbolStream(file, m_info->moduleSymbolStreamIndex, m_info->symbolSize); +} + +PDB_NO_DISCARD PDB::ModuleLineStream PDB::ModuleInfoStream::Module::CreateLineStream(const RawFile& file) const PDB_NO_EXCEPT +{ + PDB_ASSERT(HasLineStream(), "Module line stream is not present."); + + return ModuleLineStream(file, m_info->moduleSymbolStreamIndex, m_info->symbolSize + m_info->c11Size + m_info->c13Size, m_info->symbolSize + m_info->c11Size); +} + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::ModuleInfoStream::ModuleInfoStream(void) PDB_NO_EXCEPT + : m_stream() + , m_modules(nullptr) + , m_moduleCount(0u) +{ +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::ModuleInfoStream::ModuleInfoStream(ModuleInfoStream&& other) PDB_NO_EXCEPT + : m_stream(PDB_MOVE(other.m_stream)) + , m_modules(PDB_MOVE(other.m_modules)) + , m_moduleCount(PDB_MOVE(other.m_moduleCount)) +{ + other.m_modules = nullptr; + other.m_moduleCount = 0u; +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::ModuleInfoStream& PDB::ModuleInfoStream::operator=(ModuleInfoStream&& other) PDB_NO_EXCEPT +{ + if (this != &other) + { + PDB_DELETE_ARRAY(m_modules); + + m_stream = PDB_MOVE(other.m_stream); + m_modules = PDB_MOVE(other.m_modules); + m_moduleCount = PDB_MOVE(other.m_moduleCount); + + other.m_modules = nullptr; + other.m_moduleCount = 0u; + } + + return *this; +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::ModuleInfoStream::ModuleInfoStream(const DirectMSFStream& directStream, uint32_t size, uint32_t offset) PDB_NO_EXCEPT + : m_stream(directStream, size, offset) + , m_modules(nullptr) + , m_moduleCount(0u) +{ + m_modules = PDB_NEW_ARRAY(Module, EstimateModuleCount(size)); + + size_t streamOffset = 0u; + while (streamOffset < size) + { + const DBI::ModuleInfo* moduleInfo = m_stream.GetDataAtOffset(streamOffset); + streamOffset += sizeof(DBI::ModuleInfo); + + const char* name = m_stream.GetDataAtOffset(streamOffset); + const size_t nameLength = strlen(name); + streamOffset += nameLength + 1u; + + const char* objectName = m_stream.GetDataAtOffset(streamOffset); + const size_t objectNameLength = strlen(objectName); + streamOffset += objectNameLength + 1u; + + // the stream is aligned to 4 bytes + streamOffset = BitUtil::RoundUpToMultiple(streamOffset, 4ul); + + m_modules[m_moduleCount] = Module(moduleInfo, name, nameLength, objectName, objectNameLength); + ++m_moduleCount; + } +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::ModuleInfoStream::~ModuleInfoStream(void) PDB_NO_EXCEPT +{ + PDB_DELETE_ARRAY(m_modules); +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB_NO_DISCARD const PDB::ModuleInfoStream::Module* PDB::ModuleInfoStream::FindLinkerModule(void) const PDB_NO_EXCEPT +{ + const size_t count = m_moduleCount; + for (size_t i = 0u; i < count; ++i) + { + // with both MSVC cl.exe and Clang, the linker symbol is the last one to be stored, so start searching from the end + const Module& module = m_modules[count - i - 1u]; + + // check if this is the linker symbol + if (strcmp(module.GetName().Decay(), LinkerSymbolName) == 0) + { + return &module; + } + } + + return nullptr; +} -- cgit v1.2.3