From 2dfb5da16b97a6c12e01977af5b5188522178a4e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 20 Apr 2026 21:50:41 +0200 Subject: zen trace analysis support (#945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrates the **tourist** trace analysis library and builds a full `zen trace` command suite for working with Unreal Engine `.utrace` files. ### Trace analysis library (`thirdparty/tourist/`) - Adds the tourist library as a third-party dependency with three modules: **foundation** (platform primitives, memory, scheduling), **trace** (UE Trace protocol decoding), and **analysis** (event dispatching and analyzer framework). - Cross-platform support for Windows, Linux, and macOS. ### `zen trace` CLI commands (`src/zen/cmds/`, `src/zen/trace/`) - **`zen trace analyze`** — Summarize a `.utrace` file: session metadata, thread inventory, command line + build configuration, CPU profiling scopes, timing, event rates, log messages, and (with symbols) memory allocation metrics including live-allocs dumps, callstack-keyed aggregation, and allocation churn. Optional HTML output for memory reports. - **`zen trace inspect`** — Dump the event schema (declared types, fields, sizes) from a trace file. - **`zen trace trim`** — Extract a time-window from a trace into a new `.utrace` file. - **`zen trace serve`** — Launch a local HTTP server hosting an interactive trace viewer; opens in the default browser. ### Symbolication (`src/zen/trace/symbol_resolver.*`, `thirdparty/raw_pdb/`) - Pluggable resolver with multiple backends: `pdb` (in-tree raw_pdb), `dbghelp` (Windows), `llvm-symbolizer` (all platforms), `atos` (macOS). An `auto` backend picks the best available tool per platform. - Microsoft Symbol Server support: downloads PDBs on demand using a redirect-aware HTTP client. - Local PDB cache keyed by image GUID preserves symbols across binary recompilation. - Callstack trimming heuristic strips UE internal noise from reports. - Binary analysis cache (`.ucache_z`) avoids re-resolving the same trace. ### Interactive trace viewer (`src/zen/frontend/html/`, `src/zen/trace/trace_viewer_service.*`) - Timeline: scope-level detail, horizontal zoom/pan, vertical scrolling, viewport-driven loading with pre-computed LOD for responsive navigation of large traces. - Thread grouping (collapsible sidebar sections) synthesized from name suffixes, natural sort order, visual distinction between lane threads and OS threads. - Bookmark and region annotations; region categories with per-category toggles; bookmark marker toggle in the toolbar. - Filterable Logs tab showing captured `UE_LOG` output. - Stats tab with per-scope aggregate statistics. - Memory tab with interactive allocation analysis and an allocation size histogram. - CsvProfiler event parsing and chart UI. ### Other in-branch supporting changes - **Cross-platform browser launcher** (`browser_launcher.{h,cpp}`) used by `trace serve`. - **`ReciprocalU64`** fast 64-bit integer division (zencore/intmath) for trace analyzers. - **`parallelsort`** cross-platform parallel sort helper (zenutil). - Frontend zip build rule so the viewer's HTML assets are bundled into `zen.exe`. - `/Zo` flag for better optimized debug info on Windows release builds. - `trace-tests.cpp` in the `zen-test` harness (harness itself landed on main via #985). --- thirdparty/raw_pdb/src/PDB_RawFile.cpp | 147 +++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 thirdparty/raw_pdb/src/PDB_RawFile.cpp (limited to 'thirdparty/raw_pdb/src/PDB_RawFile.cpp') diff --git a/thirdparty/raw_pdb/src/PDB_RawFile.cpp b/thirdparty/raw_pdb/src/PDB_RawFile.cpp new file mode 100644 index 000000000..f135efe66 --- /dev/null +++ b/thirdparty/raw_pdb/src/PDB_RawFile.cpp @@ -0,0 +1,147 @@ +// Copyright 2011-2022, Molecular Matters GmbH +// See LICENSE.txt for licensing details (2-clause BSD License: https://opensource.org/licenses/BSD-2-Clause) + +#include "PDB_PCH.h" +#include "PDB_RawFile.h" +#include "PDB_Types.h" +#include "PDB_Util.h" +#include "PDB_DirectMSFStream.h" +#include "Foundation/PDB_PointerUtil.h" +#include "Foundation/PDB_Memory.h" +#include "Foundation/PDB_Assert.h" + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::RawFile::RawFile(RawFile&& other) PDB_NO_EXCEPT + : m_data(PDB_MOVE(other.m_data)) + , m_superBlock(PDB_MOVE(other.m_superBlock)) + , m_directoryStream(PDB_MOVE(other.m_directoryStream)) + , m_streamCount(PDB_MOVE(other.m_streamCount)) + , m_streamSizes(PDB_MOVE(other.m_streamSizes)) + , m_streamBlocks(PDB_MOVE(other.m_streamBlocks)) +{ + other.m_data = nullptr; + other.m_superBlock = nullptr; + other.m_streamCount = 0u; + other.m_streamSizes = nullptr; + other.m_streamBlocks = nullptr; +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::RawFile& PDB::RawFile::operator=(RawFile&& other) PDB_NO_EXCEPT +{ + if (this != &other) + { + PDB_DELETE_ARRAY(m_streamBlocks); + + m_data = PDB_MOVE(other.m_data); + m_superBlock = PDB_MOVE(other.m_superBlock); + m_directoryStream = PDB_MOVE(other.m_directoryStream); + m_streamCount = PDB_MOVE(other.m_streamCount); + m_streamSizes = PDB_MOVE(other.m_streamSizes); + m_streamBlocks = PDB_MOVE(other.m_streamBlocks); + + other.m_data = nullptr; + other.m_superBlock = nullptr; + other.m_streamCount = 0u; + other.m_streamSizes = nullptr; + other.m_streamBlocks = nullptr; + } + + return *this; +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::RawFile::RawFile(const void* data) PDB_NO_EXCEPT + : m_data(data) + , m_superBlock(Pointer::Offset(data, 0u)) + , m_directoryStream() + , m_streamCount(0u) + , m_streamSizes(nullptr) + , m_streamBlocks(nullptr) +{ + // the SuperBlock stores an array of indices of blocks that make up the indices of directory blocks, which need to be stitched together to form the directory. + // the blocks holding the indices of directory blocks are not necessarily contiguous, so they need to be coalesced first. + const uint32_t directoryBlockCount = PDB::ConvertSizeToBlockCount(m_superBlock->directorySize, m_superBlock->blockSize); + + // the directory is made up of directoryBlockCount blocks, so we need that many indices to be read from the blocks that make up the indices + CoalescedMSFStream directoryIndicesStream(data, m_superBlock->blockSize, m_superBlock->directoryBlockIndices, directoryBlockCount * sizeof(uint32_t)); + + // these are the indices of blocks making up the directory stream, now guaranteed to be contiguous + const uint32_t* directoryIndices = directoryIndicesStream.GetDataAtOffset(0u); + + m_directoryStream = CoalescedMSFStream(data, m_superBlock->blockSize, directoryIndices, m_superBlock->directorySize); + + // https://llvm.org/docs/PDB/MsfFile.html#the-stream-directory + // parse the directory from its contiguous version. the directory matches the following struct: + // struct StreamDirectory + // { + // uint32_t streamCount; + // uint32_t streamSizes[streamCount]; + // uint32_t streamBlocks[streamCount][]; + // }; + m_streamCount = *m_directoryStream.GetDataAtOffset(0u); + + // we can assign pointers into the stream directly, since the RawFile keeps ownership of the directory stream + m_streamSizes = m_directoryStream.GetDataAtOffset(sizeof(uint32_t)); + const uint32_t* directoryStreamBlocks = m_directoryStream.GetDataAtOffset(sizeof(uint32_t) + sizeof(uint32_t) * m_streamCount); + + // prepare indices for directly accessing individual streams + m_streamBlocks = PDB_NEW_ARRAY(const uint32_t*, m_streamCount); + + const uint32_t* indicesForCurrentBlock = directoryStreamBlocks; + for (uint32_t i = 0u; i < m_streamCount; ++i) + { + const uint32_t sizeInBytes = GetStreamSize(i); + const uint32_t blockCount = ConvertSizeToBlockCount(sizeInBytes, m_superBlock->blockSize); + m_streamBlocks[i] = indicesForCurrentBlock; + + indicesForCurrentBlock += blockCount; + } +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +PDB::RawFile::~RawFile(void) PDB_NO_EXCEPT +{ + PDB_DELETE_ARRAY(m_streamBlocks); +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +template +PDB_NO_DISCARD T PDB::RawFile::CreateMSFStream(uint32_t streamIndex) const PDB_NO_EXCEPT +{ + PDB_ASSERT(streamIndex != PDB::NilStreamIndex, "Invalid stream index."); + PDB_ASSERT(streamIndex < m_streamCount, "Invalid stream index."); + + return T(m_data, m_superBlock->blockSize, m_streamBlocks[streamIndex], GetStreamSize(streamIndex)); +} + + +// ------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------ +template +PDB_NO_DISCARD T PDB::RawFile::CreateMSFStream(uint32_t streamIndex, uint32_t streamSize) const PDB_NO_EXCEPT +{ + PDB_ASSERT(streamIndex != PDB::NilStreamIndex, "Invalid stream index."); + PDB_ASSERT(streamIndex < m_streamCount, "Invalid stream index."); + PDB_ASSERT(streamSize <= GetStreamSize(streamIndex), "Invalid stream size."); + + return T(m_data, m_superBlock->blockSize, m_streamBlocks[streamIndex], streamSize); +} + + +// explicit template instantiation +template PDB::CoalescedMSFStream PDB::RawFile::CreateMSFStream(uint32_t streamIndex) const PDB_NO_EXCEPT; +template PDB::DirectMSFStream PDB::RawFile::CreateMSFStream(uint32_t streamIndex) const PDB_NO_EXCEPT; + +template PDB::CoalescedMSFStream PDB::RawFile::CreateMSFStream(uint32_t streamIndex, uint32_t streamSize) const PDB_NO_EXCEPT; +template PDB::DirectMSFStream PDB::RawFile::CreateMSFStream(uint32_t streamIndex, uint32_t streamSize) const PDB_NO_EXCEPT; -- cgit v1.2.3